transcript_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cdubi\\anaconda3\\lib\\site-packages\\fuzzywuzzy\\fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from urllib.request import FancyURLopener  # This is library that helps us create the headless browser\n",
    "from random import choice #This library helps pick a random item from a list\n",
    "import bs4 as bs\n",
    "import datetime\n",
    "from datetime import timedelta, date\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "import sys\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "from splinter import Browser\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_browser():\n",
    "    executable_path = {'executable_path': 'chromedriver.exe'}\n",
    "    return Browser(\"chrome\", **executable_path, headless=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_agents = [\n",
    "    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',\n",
    "    'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',\n",
    "    'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',\n",
    "    'Mozilla/5.0 (Macintosh; Intel MaPc OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A',\n",
    "    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'\n",
    "]\n",
    "\n",
    "class MyOpener(FancyURLopener, object):\n",
    "    version = choice(user_agents)\n",
    "\n",
    "# Get each article and their link\n",
    "def getArticleMeta(infotable):\n",
    "    for i in range(0,len(infotable)):\n",
    "        CallDesc.append(infotable[i].getText().encode('utf-8'))\n",
    "        CallLink.append(infotable[i]['href'])\n",
    "        \n",
    "def getDate(i):\n",
    "    date = i.find(\"span\", class_= \"date\")\n",
    "    date = date.getText()\n",
    "    try:\n",
    "        date = pd.to_datetime(date,infer_datetime_format=True)\n",
    "    except ValueError:\n",
    "        if \"Yesterday\" in date:\n",
    "            Date = datetime.date.today() - timedelta(1)\n",
    "            date = date[10:]\n",
    "            date = str(Date) + date\n",
    "        else:\n",
    "            date = date.split(',')[1]\n",
    "            date = date + ' ' + str(2017)\n",
    "        date = pd.to_datetime(date,infer_datetime_format=True)\n",
    "    return date\n",
    "\n",
    "#def getTicker(companyName):\n",
    "#    df = pd.read_csv('company.txt', sep = '\\t')\n",
    "#    df = df[df['Country'] == 'USA']\n",
    "#    dfD = df.set_index('Name')['Ticker'].to_dict()\n",
    "#    del df\n",
    "#    return dfD[process.extractOne(companyName, dfD.keys())[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This uses the search tool on Seeking Alpha to get pull specific company's transcript.\n",
    "def getTranscriptBySearch(ticker):\n",
    "    CallDesc = []\n",
    "    CallLink = []\n",
    "    browser = init_browser()\n",
    "    try:\n",
    "        ticker = ticker\n",
    "        url = 'https://seekingalpha.com/search/transcripts?term='+ ticker\n",
    "        browser.visit(url) # Start the browser and open 'url'\n",
    "    except KeyError:\n",
    "        print(\"That company doesn't exist in my masterlist\")\n",
    "        raise\n",
    "\n",
    "    assert 'Seeking Alpha' in browser.title # Wait for the page to load\n",
    "    html = driver.page_source # Get the html of the page\n",
    "    driver.quit() # Close the browser\n",
    "\n",
    "    soup = bs.BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "    infotable = soup.find_all(\"div\", class_= \"transcript_link\")\n",
    "    for i in infotable:\n",
    "        date = getDate(i)\n",
    "        if (date >= datetime.datetime(2016,9,1)) & (ticker in i.find('a').getText()):\n",
    "            CallDesc.append(i.find('a').getText().encode('utf-8'))\n",
    "            CallLink.append(i.find('a')['href'])\n",
    "    CreateTextFiles(CallLink)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "# get list of S&P 100 companies and tickers from wikipedia\n",
    "\n",
    "import pandas as pd # library for data analysis\n",
    "import requests # library to handle requests\n",
    "from bs4 import BeautifulSoup # library to parse HTML documents\n",
    "\n",
    "# get the response in the form of html\n",
    "wikiurl=\"https://en.wikipedia.org/wiki/S%26P_100\"\n",
    "table_class=\"wikitable sortable jquery-tablesorter\"\n",
    "response=requests.get(wikiurl)\n",
    "print(response.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Symbol                          Name\n",
      "0    AAPL                    Apple Inc.\n",
      "1    ABBV                   AbbVie Inc.\n",
      "2     ABT           Abbott Laboratories\n",
      "3     ACN                     Accenture\n",
      "4    ADBE                    Adobe Inc.\n",
      "5     AIG  American International Group\n",
      "6     ALL                      Allstate\n",
      "7    AMGN                    Amgen Inc.\n",
      "8     AMT                American Tower\n",
      "9    AMZN                    Amazon.com\n",
      "10    AXP              American Express\n",
      "11     BA                    Boeing Co.\n",
      "12    BAC          Bank of America Corp\n",
      "13   BIIB                        Biogen\n",
      "14     BK   The Bank of New York Mellon\n",
      "15   BKNG              Booking Holdings\n",
      "16    BLK                 BlackRock Inc\n",
      "17    BMY          Bristol-Myers Squibb\n",
      "18  BRK.B            Berkshire Hathaway\n",
      "19      C                 Citigroup Inc\n",
      "20    CAT              Caterpillar Inc.\n",
      "21   CHTR        Charter Communications\n",
      "22     CL             Colgate-Palmolive\n",
      "23  CMCSA                 Comcast Corp.\n",
      "24    COF   Capital One Financial Corp.\n"
     ]
    }
   ],
   "source": [
    "# parse data from the html into a beautifulsoup object\n",
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "sandp100=soup.find_all('table')\n",
    "df=pd.read_html(str(sandp100))\n",
    "\n",
    "#convert list to dataframe\n",
    "df=pd.DataFrame(df[2])\n",
    "print(df.head(25))\n",
    "df.to_csv('sandp100.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This goes into the transcript link and extracts the text from each link\n",
    "def CreateTextFiles(CallLink):\n",
    "    for link in range(0,len(CallLink)):\n",
    "        myopener = MyOpener()\n",
    "        page=myopener.open('https://seekingalpha.com' + CallLink[link])\n",
    "\n",
    "        html = page.read()\n",
    "        soup = bs.BeautifulSoup(html, 'lxml')\n",
    "\n",
    "        infotable = soup.find_all(\"p\", class_= \"p\")\n",
    "        try:\n",
    "            Output_File = open(str(infotable[0].find(\"a\")['title']) + \n",
    "                               str(infotable[2].getText()) + \".txt\",'w')\n",
    "        except IndexError as exc:\n",
    "            continue\n",
    "        except TypeError as typ:\n",
    "            continue\n",
    "\n",
    "        for i in range(0,len(infotable)):\n",
    "            if(infotable[i].getText().encode('utf-8').startswith('Copyright policy:')):\n",
    "                break\n",
    "            if(len(infotable[i].getText()) == 0):\n",
    "                time.sleep(600)\n",
    "                continue\n",
    "            Output_File.write(infotable[i].getText().encode('utf-8'))\n",
    "            Output_File.write(\"\\n\")\n",
    "        Output_File.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'init_browser' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-189302ed92b8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0murl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'https://seekingalpha.com/symbol/AAPL/transcripts'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mbrowser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minit_browser\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvisit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mhtml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhtml\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'init_browser' is not defined"
     ]
    }
   ],
   "source": [
    "url = 'https://seekingalpha.com/symbol/AAPL/transcripts'\n",
    "browser = init_browser()\n",
    "browser.visit(url)\n",
    "time.sleep(10)\n",
    "html = browser.html\n",
    "soup = bs(html, \"html.parser\")\n",
    "results = soup.find_all(\"article\")\n",
    "for result in results:\n",
    "    title = result.find('h3').text\n",
    "    url = result.find('a')['href']\n",
    "    date = result.find('span').text\n",
    "    date = pd.to_datetime(date,infer_datetime_format=True)\n",
    "    if (date > datetime.datetime(2016, 1, 1)):\n",
    "        print(title)\n",
    "        print(url)\n",
    "        print(date)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTranscriptInfo(ticker, browser_):   \n",
    "    url = 'https://seekingalpha.com/symbol/' + ticker + '/transcripts'\n",
    "    browser = browser_\n",
    "    browser.visit(url)\n",
    "    time.sleep(10)\n",
    "    html = browser.html\n",
    "    soup = bs(html, \"html.parser\")\n",
    "    results = soup.find_all(\"article\")\n",
    "    search_results = []\n",
    "    for result in results:\n",
    "        try:\n",
    "            title = result.find('h3').text\n",
    "            url = result.find('a')['href']\n",
    "            date = result.find('span').text\n",
    "            date = pd.to_datetime(date,infer_datetime_format=True)\n",
    "            if (date > datetime.datetime(2015, 1, 1)):\n",
    "                print(title)\n",
    "                print(url)\n",
    "                print(date)\n",
    "                print('----------------------------------------')\n",
    "                results_dict = {\n",
    "                    \"title\": title,\n",
    "                    \"url\": url,\n",
    "                    \"date\": date,\n",
    "                    \"ticker\": ticker\n",
    "                }\n",
    "                search_results.append(results_dict)\n",
    "        except:\n",
    "            continue\n",
    "    return search_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Results - Earnings Call Transcript\n",
      "/article/4382943-apple-inc-aapl-ceo-tim-cook-on-q4-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A1\n",
      "2020-10-29 00:00:00\n",
      "----------------------------------------\n",
      "Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Results - Earnings Call Transcript\n",
      "/article/4362707-apple-inc-aapl-ceo-tim-cook-on-q3-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A2\n",
      "2020-07-30 00:00:00\n",
      "----------------------------------------\n",
      "Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Results - Earnings Call Transcript\n",
      "/article/4341792-apple-inc-aapl-ceo-tim-cook-on-q2-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A3\n",
      "2020-04-30 00:00:00\n",
      "----------------------------------------\n",
      "Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Results - Earnings Call Transcript\n",
      "/article/4319666-apple-inc-aapl-ceo-tim-cook-on-q1-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A4\n",
      "2020-01-28 00:00:00\n",
      "----------------------------------------\n",
      "Apple Inc. (AAPL) CEO Tim Cook on Q4 2019 Results - Earnings Call Transcript\n",
      "/article/4300690-apple-inc-aapl-ceo-tim-cook-on-q4-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A5\n",
      "2019-10-30 00:00:00\n",
      "----------------------------------------\n",
      "Apple, Inc. (AAPL) CEO Tim Cook On Q3 2019 Results - Earnings Call Transcript\n",
      "/article/4279301-apple-inc-aapl-ceo-tim-cook-on-q3-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A6\n",
      "2019-07-30 00:00:00\n",
      "----------------------------------------\n",
      "Apple, Inc. (AAPL) CEO Tim Cook on Q2 2019 Results - Earnings Call Transcript\n",
      "/article/4258275-apple-inc-aapl-ceo-tim-cook-on-q2-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A7\n",
      "2019-04-30 00:00:00\n",
      "----------------------------------------\n",
      "Apple Inc. (AAPL) CEO Tim Cook on Q1 2019 Results - Earnings Call Transcript\n",
      "/article/4236503-apple-inc-aapl-ceo-tim-cook-on-q1-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A8\n",
      "2019-01-29 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q4 2018 Results - Earnings Call Transcript\n",
      "/article/4217359-apple-aapl-q4-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A9\n",
      "2018-11-01 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q3 2018 Results - Earnings Call Transcript\n",
      "/article/4192790-apple-aapl-q3-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A10\n",
      "2018-07-31 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q2 2018 Results - Earnings Call Transcript\n",
      "/article/4168271-apple-aapl-q2-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A11\n",
      "2018-05-01 00:00:00\n",
      "----------------------------------------\n",
      "Apple's (AAPL) CEO Tim Cook On Q1 2018 Results - Earnings Call Transcript\n",
      "/article/4142447-apples-aapl-ceo-tim-cook-on-q1-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A12\n",
      "2018-02-01 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q4 2017 Results - Earnings Call Transcript\n",
      "/article/4119877-apple-aapl-q4-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A13\n",
      "2017-11-02 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q3 2017 Results - Earnings Call Transcript\n",
      "/article/4093437-apple-aapl-q3-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A14\n",
      "2017-08-01 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q2 2017 Results - Earnings Call Transcript\n",
      "/article/4068153-apple-aapl-q2-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A15\n",
      "2017-05-02 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q1 2017 Results - Earnings Call Transcript\n",
      "/article/4041266-apple-aapl-q1-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A16\n",
      "2017-01-31 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q4 2016 Results - Earnings Call Transcript\n",
      "/article/4014922-apple-aapl-q4-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A17\n",
      "2016-10-25 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Q3 2016 Results - Earnings Call Transcript\n",
      "/article/3991811-apple-aapl-q3-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A18\n",
      "2016-07-26 00:00:00\n",
      "----------------------------------------\n",
      "Apple (AAPL) Timothy Donald Cook on Q2 2016 Results - Earnings Call Transcript\n",
      "/article/3968300-apple-aapl-timothy-donald-cook-on-q2-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A19\n",
      "2016-04-26 00:00:00\n",
      "----------------------------------------\n",
      "Apple's (AAPL) CEO Tim Cook On Q1 2016 Results - Earnings Call Transcript\n",
      "/article/3836826-apples-aapl-ceo-tim-cook-on-q1-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A20\n",
      "2016-01-26 00:00:00\n",
      "----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4382943-apple-inc-aapl-ceo-tim-cook-on-q4-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A1',\n",
       "  'date': Timestamp('2020-10-29 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4362707-apple-inc-aapl-ceo-tim-cook-on-q3-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A2',\n",
       "  'date': Timestamp('2020-07-30 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4341792-apple-inc-aapl-ceo-tim-cook-on-q2-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A3',\n",
       "  'date': Timestamp('2020-04-30 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4319666-apple-inc-aapl-ceo-tim-cook-on-q1-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A4',\n",
       "  'date': Timestamp('2020-01-28 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q4 2019 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4300690-apple-inc-aapl-ceo-tim-cook-on-q4-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A5',\n",
       "  'date': Timestamp('2019-10-30 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple, Inc. (AAPL) CEO Tim Cook On Q3 2019 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4279301-apple-inc-aapl-ceo-tim-cook-on-q3-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A6',\n",
       "  'date': Timestamp('2019-07-30 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple, Inc. (AAPL) CEO Tim Cook on Q2 2019 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4258275-apple-inc-aapl-ceo-tim-cook-on-q2-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A7',\n",
       "  'date': Timestamp('2019-04-30 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple Inc. (AAPL) CEO Tim Cook on Q1 2019 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4236503-apple-inc-aapl-ceo-tim-cook-on-q1-2019-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A8',\n",
       "  'date': Timestamp('2019-01-29 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q4 2018 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4217359-apple-aapl-q4-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A9',\n",
       "  'date': Timestamp('2018-11-01 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q3 2018 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4192790-apple-aapl-q3-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A10',\n",
       "  'date': Timestamp('2018-07-31 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q2 2018 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4168271-apple-aapl-q2-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A11',\n",
       "  'date': Timestamp('2018-05-01 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': \"Apple's (AAPL) CEO Tim Cook On Q1 2018 Results - Earnings Call Transcript\",\n",
       "  'url': '/article/4142447-apples-aapl-ceo-tim-cook-on-q1-2018-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A12',\n",
       "  'date': Timestamp('2018-02-01 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q4 2017 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4119877-apple-aapl-q4-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A13',\n",
       "  'date': Timestamp('2017-11-02 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q3 2017 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4093437-apple-aapl-q3-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A14',\n",
       "  'date': Timestamp('2017-08-01 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q2 2017 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4068153-apple-aapl-q2-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A15',\n",
       "  'date': Timestamp('2017-05-02 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q1 2017 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4041266-apple-aapl-q1-2017-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A16',\n",
       "  'date': Timestamp('2017-01-31 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q4 2016 Results - Earnings Call Transcript',\n",
       "  'url': '/article/4014922-apple-aapl-q4-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A17',\n",
       "  'date': Timestamp('2016-10-25 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Q3 2016 Results - Earnings Call Transcript',\n",
       "  'url': '/article/3991811-apple-aapl-q3-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A18',\n",
       "  'date': Timestamp('2016-07-26 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': 'Apple (AAPL) Timothy Donald Cook on Q2 2016 Results - Earnings Call Transcript',\n",
       "  'url': '/article/3968300-apple-aapl-timothy-donald-cook-on-q2-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A19',\n",
       "  'date': Timestamp('2016-04-26 00:00:00'),\n",
       "  'ticker': 'AAPL'},\n",
       " {'title': \"Apple's (AAPL) CEO Tim Cook On Q1 2016 Results - Earnings Call Transcript\",\n",
       "  'url': '/article/3836826-apples-aapl-ceo-tim-cook-on-q1-2016-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A20',\n",
       "  'date': Timestamp('2016-01-26 00:00:00'),\n",
       "  'ticker': 'AAPL'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "browser = init_browser()\n",
    "getTranscriptInfo('AAPL', browser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = []\n",
    "df = pd.read_csv('sandp100.csv', usecols = ['Symbol'])\n",
    "tickers = df['Symbol'].tolist()\n",
    "browser = init_browser()\n",
    "for ticker in tickers:\n",
    "    counter = 0\n",
    "    results = getTranscriptInfo(ticker, browser)\n",
    "    all_results = all_results + results\n",
    "    time.sleep(5)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = pd.DataFrame(all_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all.to_csv('transcript_list.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTextFromArticle(url, browser_):\n",
    "    url_begin = 'https://seekingalpha.com/'\n",
    "    url_end = url\n",
    "    browser = browser_\n",
    "    browser.visit(url_begin+url_end)\n",
    "    time.sleep(10)\n",
    "    html = browser.html\n",
    "    soup = bs(html, \"html.parser\")\n",
    "    texts = soup.find_all('p')\n",
    "    total_doc = ''\n",
    "    for text in texts:\n",
    "        total_doc += text.text + '\\n'\n",
    "    return total_doc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = soup.find_all('p')\n",
    "total_doc = ''\n",
    "for text in texts:\n",
    "    total_doc += text.text + '\\n'\n",
    "print(total_doc)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cdubi\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Resu...</td>\n",
       "      <td>/article/4382943-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-10-29</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Resu...</td>\n",
       "      <td>/article/4362707-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-07-30</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Resu...</td>\n",
       "      <td>/article/4341792-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Resu...</td>\n",
       "      <td>/article/4319666-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AbbVie Inc.'s (ABBV) CEO Rick Gonzalez on Q3 2...</td>\n",
       "      <td>/article/4383381-abbvie-inc-s-abbv-ceo-rick-go...</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1031</th>\n",
       "      <td>Walmart, Inc. (WMT) CEO Doug McMillon on Q1 20...</td>\n",
       "      <td>/article/4348814-walmart-inc-wmt-ceo-doug-mcmi...</td>\n",
       "      <td>2020-05-19</td>\n",
       "      <td>WMT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1035</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) Q3 2020 Results ...</td>\n",
       "      <td>/article/4383411-exxon-mobil-corporation-xom-q...</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1037</th>\n",
       "      <td>Exxon Mobil Corp (XOM) Q2 2020 Results - Earni...</td>\n",
       "      <td>/article/4363013-exxon-mobil-corp-xom-q2-2020-...</td>\n",
       "      <td>2020-07-31</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) CEO Darren Woods...</td>\n",
       "      <td>/article/4342148-exxon-mobil-corporation-xom-c...</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) CEO Darren Woods...</td>\n",
       "      <td>/article/4320707-exxon-mobil-corporation-xom-c...</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>402 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Resu...   \n",
       "1     Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Resu...   \n",
       "2     Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Resu...   \n",
       "3     Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Resu...   \n",
       "8     AbbVie Inc.'s (ABBV) CEO Rick Gonzalez on Q3 2...   \n",
       "...                                                 ...   \n",
       "1031  Walmart, Inc. (WMT) CEO Doug McMillon on Q1 20...   \n",
       "1035  Exxon Mobil Corporation (XOM) Q3 2020 Results ...   \n",
       "1037  Exxon Mobil Corp (XOM) Q2 2020 Results - Earni...   \n",
       "1039  Exxon Mobil Corporation (XOM) CEO Darren Woods...   \n",
       "1041  Exxon Mobil Corporation (XOM) CEO Darren Woods...   \n",
       "\n",
       "                                                    url        date ticker  \\\n",
       "0     /article/4382943-apple-inc-aapl-ceo-tim-cook-o...  2020-10-29   AAPL   \n",
       "1     /article/4362707-apple-inc-aapl-ceo-tim-cook-o...  2020-07-30   AAPL   \n",
       "2     /article/4341792-apple-inc-aapl-ceo-tim-cook-o...  2020-04-30   AAPL   \n",
       "3     /article/4319666-apple-inc-aapl-ceo-tim-cook-o...  2020-01-28   AAPL   \n",
       "8     /article/4383381-abbvie-inc-s-abbv-ceo-rick-go...  2020-10-30   ABBV   \n",
       "...                                                 ...         ...    ...   \n",
       "1031  /article/4348814-walmart-inc-wmt-ceo-doug-mcmi...  2020-05-19    WMT   \n",
       "1035  /article/4383411-exxon-mobil-corporation-xom-q...  2020-10-30    XOM   \n",
       "1037  /article/4363013-exxon-mobil-corp-xom-q2-2020-...  2020-07-31    XOM   \n",
       "1039  /article/4342148-exxon-mobil-corporation-xom-c...  2020-05-01    XOM   \n",
       "1041  /article/4320707-exxon-mobil-corporation-xom-c...  2020-01-31    XOM   \n",
       "\n",
       "      text  \n",
       "0      NaN  \n",
       "1      NaN  \n",
       "2      NaN  \n",
       "3      NaN  \n",
       "8      NaN  \n",
       "...    ...  \n",
       "1031   NaN  \n",
       "1035   NaN  \n",
       "1037   NaN  \n",
       "1039   NaN  \n",
       "1041   NaN  \n",
       "\n",
       "[402 rows x 5 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = pd.read_csv(\"transcript_list.csv\")\n",
    "df_ECT = df_all[df_all['title'].str.contains(\"Earnings Call Transcript\")]\n",
    "df_ECT[\"text\"] = np.nan\n",
    "df_ECT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/article/4382943-apple-inc-aapl-ceo-tim-cook-on-q4-2020-results-earnings-call-transcript?source=content_type%3Areact%7Csection%3ATranscripts%7CsectionAsset%3ATranscripts%7Cfirst_level_url%3Asymbol%7Cbutton%3AAuthor%7Clock_status%3ANo%7Cline%3A1'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT.iloc[0]['url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser = init_browser()\n",
    "text = getTextFromArticle(df_ECT.iloc[0]['url'],browser)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cdubi\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df_ECT.iloc[0]['text'] = getTextFromArticle(df_ECT.iloc[0]['url'],browser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Resu...</td>\n",
       "      <td>/article/4382943-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-10-29</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Resu...</td>\n",
       "      <td>/article/4362707-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-07-30</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Resu...</td>\n",
       "      <td>/article/4341792-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Resu...</td>\n",
       "      <td>/article/4319666-apple-inc-aapl-ceo-tim-cook-o...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AbbVie Inc.'s (ABBV) CEO Rick Gonzalez on Q3 2...</td>\n",
       "      <td>/article/4383381-abbvie-inc-s-abbv-ceo-rick-go...</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q2 202...</td>\n",
       "      <td>/article/4363134-abbvie-inc-abbv-ceo-rick-gonz...</td>\n",
       "      <td>2020-07-31</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q1 202...</td>\n",
       "      <td>/article/4342124-abbvie-inc-abbv-ceo-rick-gonz...</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q4 201...</td>\n",
       "      <td>/article/4322499-abbvie-inc-abbv-ceo-rick-gonz...</td>\n",
       "      <td>2020-02-07</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Abbott Laboratories (ABT) CEO Robert Ford on Q...</td>\n",
       "      <td>/article/4380361-abbott-laboratories-abt-ceo-r...</td>\n",
       "      <td>2020-10-21</td>\n",
       "      <td>ABT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Abbott Laboratories (ABT) CEO, Robert Ford on ...</td>\n",
       "      <td>/article/4358872-abbott-laboratories-abt-ceo-r...</td>\n",
       "      <td>2020-07-16</td>\n",
       "      <td>ABT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Abbott Laboratories (ABT) CEO, Robert Ford on ...</td>\n",
       "      <td>/article/4338017-abbott-laboratories-abt-ceo-r...</td>\n",
       "      <td>2020-04-16</td>\n",
       "      <td>ABT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Abbott Laboratories (ABT) CEO Miles White on Q...</td>\n",
       "      <td>/article/4318424-abbott-laboratories-abt-ceo-m...</td>\n",
       "      <td>2020-01-22</td>\n",
       "      <td>ABT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Accenture plc's (ACN) CEO Julie Sweet on Q1 20...</td>\n",
       "      <td>/article/4395192-accenture-plcs-acn-ceo-julie-...</td>\n",
       "      <td>2020-12-17</td>\n",
       "      <td>ACN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Accenture PLC (ACN) CEO Julie Sweet on Q4 2020...</td>\n",
       "      <td>/article/4376150-accenture-plc-acn-ceo-julie-s...</td>\n",
       "      <td>2020-09-24</td>\n",
       "      <td>ACN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Accenture plc's (ACN) CEO Julie Sweet on Q3 20...</td>\n",
       "      <td>/article/4355660-accenture-plcs-acn-ceo-julie-...</td>\n",
       "      <td>2020-06-25</td>\n",
       "      <td>ACN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Accenture plc (ACN) CEO Julie Sweet on Q2 2020...</td>\n",
       "      <td>/article/4333092-accenture-plc-acn-ceo-julie-s...</td>\n",
       "      <td>2020-03-19</td>\n",
       "      <td>ACN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Adobe Inc. (ADBE) CEO Shantanu Narayen on Q4 2...</td>\n",
       "      <td>/article/4394124-adobe-inc-adbe-ceo-shantanu-n...</td>\n",
       "      <td>2020-12-10</td>\n",
       "      <td>ADBE</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Adobe Inc. (ADBE) CEO Shantanu Narayen on Q3 2...</td>\n",
       "      <td>/article/4374554-adobe-inc-adbe-ceo-shantanu-n...</td>\n",
       "      <td>2020-09-15</td>\n",
       "      <td>ADBE</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Adobe Inc. (ADBE) CEO Shantanu Narayen on Q2 2...</td>\n",
       "      <td>/article/4353449-adobe-inc-adbe-ceo-shantanu-n...</td>\n",
       "      <td>2020-06-11</td>\n",
       "      <td>ADBE</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Adobe Inc.'s (ADBE) CEO Shantanu Narayen on Q1...</td>\n",
       "      <td>/article/4331715-adobe-inc-s-adbe-ceo-shantanu...</td>\n",
       "      <td>2020-03-12</td>\n",
       "      <td>ADBE</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                title  \\\n",
       "0   Apple Inc. (AAPL) CEO Tim Cook on Q4 2020 Resu...   \n",
       "1   Apple Inc. (AAPL) CEO Tim Cook on Q3 2020 Resu...   \n",
       "2   Apple Inc. (AAPL) CEO Tim Cook on Q2 2020 Resu...   \n",
       "3   Apple Inc. (AAPL) CEO Tim Cook on Q1 2020 Resu...   \n",
       "8   AbbVie Inc.'s (ABBV) CEO Rick Gonzalez on Q3 2...   \n",
       "10  AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q2 202...   \n",
       "11  AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q1 202...   \n",
       "12  AbbVie Inc. (ABBV) CEO Rick Gonzalez on Q4 201...   \n",
       "14  Abbott Laboratories (ABT) CEO Robert Ford on Q...   \n",
       "16  Abbott Laboratories (ABT) CEO, Robert Ford on ...   \n",
       "18  Abbott Laboratories (ABT) CEO, Robert Ford on ...   \n",
       "20  Abbott Laboratories (ABT) CEO Miles White on Q...   \n",
       "22  Accenture plc's (ACN) CEO Julie Sweet on Q1 20...   \n",
       "24  Accenture PLC (ACN) CEO Julie Sweet on Q4 2020...   \n",
       "26  Accenture plc's (ACN) CEO Julie Sweet on Q3 20...   \n",
       "28  Accenture plc (ACN) CEO Julie Sweet on Q2 2020...   \n",
       "30  Adobe Inc. (ADBE) CEO Shantanu Narayen on Q4 2...   \n",
       "32  Adobe Inc. (ADBE) CEO Shantanu Narayen on Q3 2...   \n",
       "34  Adobe Inc. (ADBE) CEO Shantanu Narayen on Q2 2...   \n",
       "35  Adobe Inc.'s (ADBE) CEO Shantanu Narayen on Q1...   \n",
       "\n",
       "                                                  url        date ticker  text  \n",
       "0   /article/4382943-apple-inc-aapl-ceo-tim-cook-o...  2020-10-29   AAPL   NaN  \n",
       "1   /article/4362707-apple-inc-aapl-ceo-tim-cook-o...  2020-07-30   AAPL   NaN  \n",
       "2   /article/4341792-apple-inc-aapl-ceo-tim-cook-o...  2020-04-30   AAPL   NaN  \n",
       "3   /article/4319666-apple-inc-aapl-ceo-tim-cook-o...  2020-01-28   AAPL   NaN  \n",
       "8   /article/4383381-abbvie-inc-s-abbv-ceo-rick-go...  2020-10-30   ABBV   NaN  \n",
       "10  /article/4363134-abbvie-inc-abbv-ceo-rick-gonz...  2020-07-31   ABBV   NaN  \n",
       "11  /article/4342124-abbvie-inc-abbv-ceo-rick-gonz...  2020-05-01   ABBV   NaN  \n",
       "12  /article/4322499-abbvie-inc-abbv-ceo-rick-gonz...  2020-02-07   ABBV   NaN  \n",
       "14  /article/4380361-abbott-laboratories-abt-ceo-r...  2020-10-21    ABT   NaN  \n",
       "16  /article/4358872-abbott-laboratories-abt-ceo-r...  2020-07-16    ABT   NaN  \n",
       "18  /article/4338017-abbott-laboratories-abt-ceo-r...  2020-04-16    ABT   NaN  \n",
       "20  /article/4318424-abbott-laboratories-abt-ceo-m...  2020-01-22    ABT   NaN  \n",
       "22  /article/4395192-accenture-plcs-acn-ceo-julie-...  2020-12-17    ACN   NaN  \n",
       "24  /article/4376150-accenture-plc-acn-ceo-julie-s...  2020-09-24    ACN   NaN  \n",
       "26  /article/4355660-accenture-plcs-acn-ceo-julie-...  2020-06-25    ACN   NaN  \n",
       "28  /article/4333092-accenture-plc-acn-ceo-julie-s...  2020-03-19    ACN   NaN  \n",
       "30  /article/4394124-adobe-inc-adbe-ceo-shantanu-n...  2020-12-10   ADBE   NaN  \n",
       "32  /article/4374554-adobe-inc-adbe-ceo-shantanu-n...  2020-09-15   ADBE   NaN  \n",
       "34  /article/4353449-adobe-inc-adbe-ceo-shantanu-n...  2020-06-11   ADBE   NaN  \n",
       "35  /article/4331715-adobe-inc-s-adbe-ceo-shantanu...  2020-03-12   ADBE   NaN  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = getTextFromArticle('/article/4338017-abbott-laboratories-abt-ceo-robert-ford-on-q1-2020-results-earnings-call-transcript', browser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df_ECT.iloc[150:].iterrows():\n",
    "    try:\n",
    "        text = getTextFromArticle(row['url'],browser)\n",
    "        time.sleep(1)\n",
    "        file = open(row['title'] + '.txt','w')\n",
    "        time.sleep(1)\n",
    "        file.writelines(text) \n",
    "        file.close()\n",
    "    except:\n",
    "        continue\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>Ford Motor Company (F) CEO Jim Farley on Q3 20...</td>\n",
       "      <td>/article/4382475-ford-motor-company-f-ceo-jim-...</td>\n",
       "      <td>2020-10-29</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>Ford Motor Company's (F) CEO Jim Hackett on Q2...</td>\n",
       "      <td>/article/4362728-ford-motor-companys-f-ceo-jim...</td>\n",
       "      <td>2020-07-31</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>Facebook's (FB) CEO Mark Zuckerberg on Q3 2020...</td>\n",
       "      <td>/article/4382959-facebooks-fb-ceo-mark-zuckerb...</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>FB</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>Facebook, Inc. (FB) CEO Mark Zuckerberg on Q2 ...</td>\n",
       "      <td>/article/4362679-facebook-inc-fb-ceo-mark-zuck...</td>\n",
       "      <td>2020-07-30</td>\n",
       "      <td>FB</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390</th>\n",
       "      <td>Facebook, Inc. (FB) CEO Mark Zuckerberg on Q1 ...</td>\n",
       "      <td>/article/4341309-facebook-inc-fb-ceo-mark-zuck...</td>\n",
       "      <td>2020-04-29</td>\n",
       "      <td>FB</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1031</th>\n",
       "      <td>Walmart, Inc. (WMT) CEO Doug McMillon on Q1 20...</td>\n",
       "      <td>/article/4348814-walmart-inc-wmt-ceo-doug-mcmi...</td>\n",
       "      <td>2020-05-19</td>\n",
       "      <td>WMT</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1035</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) Q3 2020 Results ...</td>\n",
       "      <td>/article/4383411-exxon-mobil-corporation-xom-q...</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1037</th>\n",
       "      <td>Exxon Mobil Corp (XOM) Q2 2020 Results - Earni...</td>\n",
       "      <td>/article/4363013-exxon-mobil-corp-xom-q2-2020-...</td>\n",
       "      <td>2020-07-31</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) CEO Darren Woods...</td>\n",
       "      <td>/article/4342148-exxon-mobil-corporation-xom-c...</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>Exxon Mobil Corporation (XOM) CEO Darren Woods...</td>\n",
       "      <td>/article/4320707-exxon-mobil-corporation-xom-c...</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>XOM</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "377   Ford Motor Company (F) CEO Jim Farley on Q3 20...   \n",
       "385   Ford Motor Company's (F) CEO Jim Hackett on Q2...   \n",
       "387   Facebook's (FB) CEO Mark Zuckerberg on Q3 2020...   \n",
       "389   Facebook, Inc. (FB) CEO Mark Zuckerberg on Q2 ...   \n",
       "390   Facebook, Inc. (FB) CEO Mark Zuckerberg on Q1 ...   \n",
       "...                                                 ...   \n",
       "1031  Walmart, Inc. (WMT) CEO Doug McMillon on Q1 20...   \n",
       "1035  Exxon Mobil Corporation (XOM) Q3 2020 Results ...   \n",
       "1037  Exxon Mobil Corp (XOM) Q2 2020 Results - Earni...   \n",
       "1039  Exxon Mobil Corporation (XOM) CEO Darren Woods...   \n",
       "1041  Exxon Mobil Corporation (XOM) CEO Darren Woods...   \n",
       "\n",
       "                                                    url        date ticker  \\\n",
       "377   /article/4382475-ford-motor-company-f-ceo-jim-...  2020-10-29      F   \n",
       "385   /article/4362728-ford-motor-companys-f-ceo-jim...  2020-07-31      F   \n",
       "387   /article/4382959-facebooks-fb-ceo-mark-zuckerb...  2020-10-30     FB   \n",
       "389   /article/4362679-facebook-inc-fb-ceo-mark-zuck...  2020-07-30     FB   \n",
       "390   /article/4341309-facebook-inc-fb-ceo-mark-zuck...  2020-04-29     FB   \n",
       "...                                                 ...         ...    ...   \n",
       "1031  /article/4348814-walmart-inc-wmt-ceo-doug-mcmi...  2020-05-19    WMT   \n",
       "1035  /article/4383411-exxon-mobil-corporation-xom-q...  2020-10-30    XOM   \n",
       "1037  /article/4363013-exxon-mobil-corp-xom-q2-2020-...  2020-07-31    XOM   \n",
       "1039  /article/4342148-exxon-mobil-corporation-xom-c...  2020-05-01    XOM   \n",
       "1041  /article/4320707-exxon-mobil-corporation-xom-c...  2020-01-31    XOM   \n",
       "\n",
       "      text  \n",
       "377    NaN  \n",
       "385    NaN  \n",
       "387    NaN  \n",
       "389    NaN  \n",
       "390    NaN  \n",
       "...    ...  \n",
       "1031   NaN  \n",
       "1035   NaN  \n",
       "1037   NaN  \n",
       "1039   NaN  \n",
       "1041   NaN  \n",
       "\n",
       "[252 rows x 5 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT.iloc[150:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiurl=\"https://en.wikipedia.org/wiki/S%26P_100\"\n",
    "table_class=\"wikitable sortable jquery-tablesorter\"\n",
    "response=requests.get(wikiurl)\n",
    "print(response.status_code)\n",
    "# parse data from the html into a beautifulsoup object\n",
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "sandp100=soup.find_all('table')\n",
    "df=pd.read_html(str(sandp100))\n",
    "\n",
    "#convert list to dataframe\n",
    "df=pd.DataFrame(df[2])\n",
    "print(df.head(25))\n",
    "df.to_csv('sandp100.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd # library for data analysis\n",
    "import requests # library to handle requests\n",
    "from bs4 import BeautifulSoup # library to parse HTML documents\n",
    "\n",
    "wikiurl2=\"https://en.wikipedia.org/wiki/List_of_S%26P_500_companies\"\n",
    "response=requests.get(wikiurl2)\n",
    "print(response.status_code)\n",
    "# parse data from the html into a beautifulsoup object\n",
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "sandp500=soup.find_all('table')\n",
    "df2=pd.read_html(str(sandp500))\n",
    "\n",
    "#convert list to dataframe\n",
    "#df2=pd.DataFrame(df[2])\n",
    "#print(df.head(25))\n",
    "#df.to_csv('sandp100.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sandp100_df = pd.read_csv('sandp100.csv')\n",
    "sandp500_df = df2[0]\n",
    "sandp100_df_merged = sandp100_df.merge(sandp500_df, how='left', left_on='Symbol', right_on='Symbol').drop(columns=['Security', 'SEC filings'])\n",
    "sandp100_df_merged.to_csv('sandp100_more_detail.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd # library for data analysis\n",
    "import requests # library to handle requests\n",
    "from bs4 import BeautifulSoup # library to parse HTML documents\n",
    "\n",
    "wikiurl2=\"https://en.wikipedia.org/wiki/List_of_S%26P_500_companies\"\n",
    "response=requests.get(wikiurl2)\n",
    "print(response.status_code)\n",
    "# parse data from the html into a beautifulsoup object\n",
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "sandp500=soup.find_all('table')\n",
    "df2=pd.read_html(str(sandp500))\n",
    "\n",
    "#convert list to dataframe\n",
    "#df2=pd.DataFrame(df[2])\n",
    "#print(df.head(25))\n",
    "#df.to_csv('sandp100.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2[0].to_csv('sandp500.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = []\n",
    "df = pd.read_csv('sandp500.csv', usecols = ['Symbol'])\n",
    "tickers = df['Symbol'].tolist()\n",
    "browser = init_browser()\n",
    "for ticker in tickers:\n",
    "    results = getTranscriptInfo(ticker, browser)\n",
    "    all_results = all_results + results\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ECT500 = pd.DataFrame(all_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3M Company 2020 Q3 - Results - Earnings Call P...</td>\n",
       "      <td>/article/4381577-3m-company-2020-q3-results-ea...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3M Company 2020 Q2 - Results - Earnings Call P...</td>\n",
       "      <td>/article/4361248-3m-company-2020-q2-results-ea...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3M Company (MMM) CEO Michael Roman on UBS Glob...</td>\n",
       "      <td>/article/4351738-3m-company-mmm-ceo-michael-ro...</td>\n",
       "      <td>2020-06-03</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9773</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9774</th>\n",
       "      <td>Zoetis 2018 Q2 - Results - Earnings Call Slides</td>\n",
       "      <td>/article/4193869-zoetis-2018-q2-results-earnin...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9775</th>\n",
       "      <td>Zoetis (ZTS) Investor Presentation - Slideshow</td>\n",
       "      <td>/article/4174742-zoetis-zts-investor-presentat...</td>\n",
       "      <td>2018-05-16</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9776</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9777</th>\n",
       "      <td>Zoetis 2018 Q1 - Results - Earnings Call Slides</td>\n",
       "      <td>/article/4168515-zoetis-2018-q1-results-earnin...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9778 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "1     3M Company 2020 Q3 - Results - Earnings Call P...   \n",
       "2     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "3     3M Company 2020 Q2 - Results - Earnings Call P...   \n",
       "4     3M Company (MMM) CEO Michael Roman on UBS Glob...   \n",
       "...                                                 ...   \n",
       "9773  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "9774    Zoetis 2018 Q2 - Results - Earnings Call Slides   \n",
       "9775     Zoetis (ZTS) Investor Presentation - Slideshow   \n",
       "9776  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "9777    Zoetis 2018 Q1 - Results - Earnings Call Slides   \n",
       "\n",
       "                                                    url       date ticker  \n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM  \n",
       "1     /article/4381577-3m-company-2020-q3-results-ea... 2020-10-27    MMM  \n",
       "2     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM  \n",
       "3     /article/4361248-3m-company-2020-q2-results-ea... 2020-07-28    MMM  \n",
       "4     /article/4351738-3m-company-mmm-ceo-michael-ro... 2020-06-03    MMM  \n",
       "...                                                 ...        ...    ...  \n",
       "9773  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS  \n",
       "9774  /article/4193869-zoetis-2018-q2-results-earnin... 2018-08-02    ZTS  \n",
       "9775  /article/4174742-zoetis-zts-investor-presentat... 2018-05-16    ZTS  \n",
       "9776  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS  \n",
       "9777  /article/4168515-zoetis-2018-q1-results-earnin... 2018-05-02    ZTS  \n",
       "\n",
       "[9778 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ECT500.to_csv('transcript_list_500.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q1 2020 Res...</td>\n",
       "      <td>/article/4340658-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-04-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3M's (MMM) CEO Mike Roman on Q4 2019 Results -...</td>\n",
       "      <td>/article/4319588-3ms-mmm-ceo-mike-roman-on-q4-...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2019 Res...</td>\n",
       "      <td>/article/4298957-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9768</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...</td>\n",
       "      <td>/article/4259182-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9769</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...</td>\n",
       "      <td>/article/4241305-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9771</th>\n",
       "      <td>Zoetis (ZTS) Q3 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4217256-zoetis-zts-q3-2018-results-ea...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9773</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9776</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5273 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "2     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "6     3M Company (MMM) CEO Mike Roman on Q1 2020 Res...   \n",
       "9     3M's (MMM) CEO Mike Roman on Q4 2019 Results -...   \n",
       "12    3M Company (MMM) CEO Mike Roman on Q3 2019 Res...   \n",
       "...                                                 ...   \n",
       "9768  Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...   \n",
       "9769  Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...   \n",
       "9771  Zoetis (ZTS) Q3 2018 Results - Earnings Call T...   \n",
       "9773  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "9776  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "\n",
       "                                                    url       date ticker  \n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM  \n",
       "2     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM  \n",
       "6     /article/4340658-3m-company-mmm-ceo-mike-roman... 2020-04-28    MMM  \n",
       "9     /article/4319588-3ms-mmm-ceo-mike-roman-on-q4-... 2020-01-28    MMM  \n",
       "12    /article/4298957-3m-company-mmm-ceo-mike-roman... 2019-10-24    MMM  \n",
       "...                                                 ...        ...    ...  \n",
       "9768  /article/4259182-zoetis-inc-zts-ceo-juan-ramon... 2019-05-02    ZTS  \n",
       "9769  /article/4241305-zoetis-inc-zts-ceo-juan-ramon... 2019-02-14    ZTS  \n",
       "9771  /article/4217256-zoetis-zts-q3-2018-results-ea... 2018-11-01    ZTS  \n",
       "9773  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS  \n",
       "9776  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS  \n",
       "\n",
       "[5273 rows x 4 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df_ECT500 = df_ECT500[df_ECT500['title'].str.contains(\"Earnings Call Transcript\")]\n",
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Symbol</th>\n",
       "      <th>Security</th>\n",
       "      <th>SEC filings</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub-Industry</th>\n",
       "      <th>Headquarters Location</th>\n",
       "      <th>Date first added</th>\n",
       "      <th>CIK</th>\n",
       "      <th>Founded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MMM</td>\n",
       "      <td>3M Company</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "      <td>66740</td>\n",
       "      <td>1902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABT</td>\n",
       "      <td>Abbott Laboratories</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>1964-03-31</td>\n",
       "      <td>1800</td>\n",
       "      <td>1888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABBV</td>\n",
       "      <td>AbbVie Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>1551152</td>\n",
       "      <td>2013 (1888)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABMD</td>\n",
       "      <td>ABIOMED Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>Danvers, Massachusetts</td>\n",
       "      <td>2018-05-31</td>\n",
       "      <td>815094</td>\n",
       "      <td>1981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ACN</td>\n",
       "      <td>Accenture plc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>IT Consulting &amp; Other Services</td>\n",
       "      <td>Dublin, Ireland</td>\n",
       "      <td>2011-07-06</td>\n",
       "      <td>1467373</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>YUM</td>\n",
       "      <td>Yum! Brands Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Louisville, Kentucky</td>\n",
       "      <td>1997-10-06</td>\n",
       "      <td>1041061</td>\n",
       "      <td>1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>ZBRA</td>\n",
       "      <td>Zebra Technologies</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Electronic Equipment &amp; Instruments</td>\n",
       "      <td>Lincolnshire, Illinois</td>\n",
       "      <td>2019-12-23</td>\n",
       "      <td>877212</td>\n",
       "      <td>1969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>ZBH</td>\n",
       "      <td>Zimmer Biomet</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>Warsaw, Indiana</td>\n",
       "      <td>2001-08-07</td>\n",
       "      <td>1136869</td>\n",
       "      <td>1927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>ZION</td>\n",
       "      <td>Zions Bancorp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Regional Banks</td>\n",
       "      <td>Salt Lake City, Utah</td>\n",
       "      <td>2001-06-22</td>\n",
       "      <td>109380</td>\n",
       "      <td>1873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>ZTS</td>\n",
       "      <td>Zoetis</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "      <td>1555280</td>\n",
       "      <td>1952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>505 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Symbol             Security SEC filings             GICS Sector  \\\n",
       "0      MMM           3M Company     reports             Industrials   \n",
       "1      ABT  Abbott Laboratories     reports             Health Care   \n",
       "2     ABBV          AbbVie Inc.     reports             Health Care   \n",
       "3     ABMD          ABIOMED Inc     reports             Health Care   \n",
       "4      ACN        Accenture plc     reports  Information Technology   \n",
       "..     ...                  ...         ...                     ...   \n",
       "500    YUM      Yum! Brands Inc     reports  Consumer Discretionary   \n",
       "501   ZBRA   Zebra Technologies     reports  Information Technology   \n",
       "502    ZBH        Zimmer Biomet     reports             Health Care   \n",
       "503   ZION        Zions Bancorp     reports              Financials   \n",
       "504    ZTS               Zoetis     reports             Health Care   \n",
       "\n",
       "                      GICS Sub-Industry     Headquarters Location  \\\n",
       "0              Industrial Conglomerates       St. Paul, Minnesota   \n",
       "1                 Health Care Equipment   North Chicago, Illinois   \n",
       "2                       Pharmaceuticals   North Chicago, Illinois   \n",
       "3                 Health Care Equipment    Danvers, Massachusetts   \n",
       "4        IT Consulting & Other Services           Dublin, Ireland   \n",
       "..                                  ...                       ...   \n",
       "500                         Restaurants      Louisville, Kentucky   \n",
       "501  Electronic Equipment & Instruments    Lincolnshire, Illinois   \n",
       "502               Health Care Equipment           Warsaw, Indiana   \n",
       "503                      Regional Banks      Salt Lake City, Utah   \n",
       "504                     Pharmaceuticals  Florham Park, New Jersey   \n",
       "\n",
       "    Date first added      CIK      Founded  \n",
       "0         1976-08-09    66740         1902  \n",
       "1         1964-03-31     1800         1888  \n",
       "2         2012-12-31  1551152  2013 (1888)  \n",
       "3         2018-05-31   815094         1981  \n",
       "4         2011-07-06  1467373         1989  \n",
       "..               ...      ...          ...  \n",
       "500       1997-10-06  1041061         1997  \n",
       "501       2019-12-23   877212         1969  \n",
       "502       2001-08-07  1136869         1927  \n",
       "503       2001-06-22   109380         1873  \n",
       "504       2013-06-21  1555280         1952  \n",
       "\n",
       "[505 rows x 9 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sandp500_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>ticker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q1 2020 Res...</td>\n",
       "      <td>/article/4340658-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-04-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3M's (MMM) CEO Mike Roman on Q4 2019 Results -...</td>\n",
       "      <td>/article/4319588-3ms-mmm-ceo-mike-roman-on-q4-...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2019 Res...</td>\n",
       "      <td>/article/4298957-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>MMM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9768</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...</td>\n",
       "      <td>/article/4259182-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9769</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...</td>\n",
       "      <td>/article/4241305-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9771</th>\n",
       "      <td>Zoetis (ZTS) Q3 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4217256-zoetis-zts-q3-2018-results-ea...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9773</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9776</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5273 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "2     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "6     3M Company (MMM) CEO Mike Roman on Q1 2020 Res...   \n",
       "9     3M's (MMM) CEO Mike Roman on Q4 2019 Results -...   \n",
       "12    3M Company (MMM) CEO Mike Roman on Q3 2019 Res...   \n",
       "...                                                 ...   \n",
       "9768  Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...   \n",
       "9769  Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...   \n",
       "9771  Zoetis (ZTS) Q3 2018 Results - Earnings Call T...   \n",
       "9773  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "9776  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "\n",
       "                                                    url       date ticker  \n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM  \n",
       "2     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM  \n",
       "6     /article/4340658-3m-company-mmm-ceo-mike-roman... 2020-04-28    MMM  \n",
       "9     /article/4319588-3ms-mmm-ceo-mike-roman-on-q4-... 2020-01-28    MMM  \n",
       "12    /article/4298957-3m-company-mmm-ceo-mike-roman... 2019-10-24    MMM  \n",
       "...                                                 ...        ...    ...  \n",
       "9768  /article/4259182-zoetis-inc-zts-ceo-juan-ramon... 2019-05-02    ZTS  \n",
       "9769  /article/4241305-zoetis-inc-zts-ceo-juan-ramon... 2019-02-14    ZTS  \n",
       "9771  /article/4217256-zoetis-zts-q3-2018-results-ea... 2018-11-01    ZTS  \n",
       "9773  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS  \n",
       "9776  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS  \n",
       "\n",
       "[5273 rows x 4 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sandp500_df_merged = df_ECT500.merge(sandp500_df, how='left', left_on='ticker', right_on='Symbol').drop(columns=['Security', 'SEC filings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub-Industry</th>\n",
       "      <th>Headquarters Location</th>\n",
       "      <th>Date first added</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q1 2020 Res...</td>\n",
       "      <td>/article/4340658-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-04-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3M's (MMM) CEO Mike Roman on Q4 2019 Results -...</td>\n",
       "      <td>/article/4319588-3ms-mmm-ceo-mike-roman-on-q4-...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2019 Res...</td>\n",
       "      <td>/article/4298957-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>1976-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5268</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...</td>\n",
       "      <td>/article/4259182-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5269</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...</td>\n",
       "      <td>/article/4241305-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5270</th>\n",
       "      <td>Zoetis (ZTS) Q3 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4217256-zoetis-zts-q3-2018-results-ea...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5271</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5272</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5273 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "1     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "2     3M Company (MMM) CEO Mike Roman on Q1 2020 Res...   \n",
       "3     3M's (MMM) CEO Mike Roman on Q4 2019 Results -...   \n",
       "4     3M Company (MMM) CEO Mike Roman on Q3 2019 Res...   \n",
       "...                                                 ...   \n",
       "5268  Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...   \n",
       "5269  Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...   \n",
       "5270  Zoetis (ZTS) Q3 2018 Results - Earnings Call T...   \n",
       "5271  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "5272  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "\n",
       "                                                    url       date Symbol  \\\n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM   \n",
       "1     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM   \n",
       "2     /article/4340658-3m-company-mmm-ceo-mike-roman... 2020-04-28    MMM   \n",
       "3     /article/4319588-3ms-mmm-ceo-mike-roman-on-q4-... 2020-01-28    MMM   \n",
       "4     /article/4298957-3m-company-mmm-ceo-mike-roman... 2019-10-24    MMM   \n",
       "...                                                 ...        ...    ...   \n",
       "5268  /article/4259182-zoetis-inc-zts-ceo-juan-ramon... 2019-05-02    ZTS   \n",
       "5269  /article/4241305-zoetis-inc-zts-ceo-juan-ramon... 2019-02-14    ZTS   \n",
       "5270  /article/4217256-zoetis-zts-q3-2018-results-ea... 2018-11-01    ZTS   \n",
       "5271  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS   \n",
       "5272  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS   \n",
       "\n",
       "      GICS Sector         GICS Sub-Industry     Headquarters Location  \\\n",
       "0     Industrials  Industrial Conglomerates       St. Paul, Minnesota   \n",
       "1     Industrials  Industrial Conglomerates       St. Paul, Minnesota   \n",
       "2     Industrials  Industrial Conglomerates       St. Paul, Minnesota   \n",
       "3     Industrials  Industrial Conglomerates       St. Paul, Minnesota   \n",
       "4     Industrials  Industrial Conglomerates       St. Paul, Minnesota   \n",
       "...           ...                       ...                       ...   \n",
       "5268  Health Care           Pharmaceuticals  Florham Park, New Jersey   \n",
       "5269  Health Care           Pharmaceuticals  Florham Park, New Jersey   \n",
       "5270  Health Care           Pharmaceuticals  Florham Park, New Jersey   \n",
       "5271  Health Care           Pharmaceuticals  Florham Park, New Jersey   \n",
       "5272  Health Care           Pharmaceuticals  Florham Park, New Jersey   \n",
       "\n",
       "     Date first added  \n",
       "0          1976-08-09  \n",
       "1          1976-08-09  \n",
       "2          1976-08-09  \n",
       "3          1976-08-09  \n",
       "4          1976-08-09  \n",
       "...               ...  \n",
       "5268       2013-06-21  \n",
       "5269       2013-06-21  \n",
       "5270       2013-06-21  \n",
       "5271       2013-06-21  \n",
       "5272       2013-06-21  \n",
       "\n",
       "[5273 rows x 8 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT500 = sandp500_df_merged.drop(columns=['ticker', 'CIK', 'Founded'])\n",
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub-Industry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q1 2020 Res...</td>\n",
       "      <td>/article/4340658-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-04-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3M's (MMM) CEO Mike Roman on Q4 2019 Results -...</td>\n",
       "      <td>/article/4319588-3ms-mmm-ceo-mike-roman-on-q4-...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2019 Res...</td>\n",
       "      <td>/article/4298957-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5268</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...</td>\n",
       "      <td>/article/4259182-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5269</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...</td>\n",
       "      <td>/article/4241305-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5270</th>\n",
       "      <td>Zoetis (ZTS) Q3 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4217256-zoetis-zts-q3-2018-results-ea...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5271</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5272</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5273 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "1     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "2     3M Company (MMM) CEO Mike Roman on Q1 2020 Res...   \n",
       "3     3M's (MMM) CEO Mike Roman on Q4 2019 Results -...   \n",
       "4     3M Company (MMM) CEO Mike Roman on Q3 2019 Res...   \n",
       "...                                                 ...   \n",
       "5268  Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...   \n",
       "5269  Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...   \n",
       "5270  Zoetis (ZTS) Q3 2018 Results - Earnings Call T...   \n",
       "5271  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "5272  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "\n",
       "                                                    url       date Symbol  \\\n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM   \n",
       "1     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM   \n",
       "2     /article/4340658-3m-company-mmm-ceo-mike-roman... 2020-04-28    MMM   \n",
       "3     /article/4319588-3ms-mmm-ceo-mike-roman-on-q4-... 2020-01-28    MMM   \n",
       "4     /article/4298957-3m-company-mmm-ceo-mike-roman... 2019-10-24    MMM   \n",
       "...                                                 ...        ...    ...   \n",
       "5268  /article/4259182-zoetis-inc-zts-ceo-juan-ramon... 2019-05-02    ZTS   \n",
       "5269  /article/4241305-zoetis-inc-zts-ceo-juan-ramon... 2019-02-14    ZTS   \n",
       "5270  /article/4217256-zoetis-zts-q3-2018-results-ea... 2018-11-01    ZTS   \n",
       "5271  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS   \n",
       "5272  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS   \n",
       "\n",
       "      GICS Sector         GICS Sub-Industry  \n",
       "0     Industrials  Industrial Conglomerates  \n",
       "1     Industrials  Industrial Conglomerates  \n",
       "2     Industrials  Industrial Conglomerates  \n",
       "3     Industrials  Industrial Conglomerates  \n",
       "4     Industrials  Industrial Conglomerates  \n",
       "...           ...                       ...  \n",
       "5268  Health Care           Pharmaceuticals  \n",
       "5269  Health Care           Pharmaceuticals  \n",
       "5270  Health Care           Pharmaceuticals  \n",
       "5271  Health Care           Pharmaceuticals  \n",
       "5272  Health Care           Pharmaceuticals  \n",
       "\n",
       "[5273 rows x 6 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT500 = df_ECT500.drop(columns=['Headquarters Location', 'Date first added'])\n",
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ECT500.to_csv('transcript_list_500.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df_ECT500.iterrows():\n",
    "    try:\n",
    "        text = getTextFromArticle(row['url'],browser)\n",
    "        time.sleep(1)\n",
    "        file = open(row['title'] + '.txt','w')\n",
    "        time.sleep(1)\n",
    "        file.writelines(text) \n",
    "        file.close()\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub-Industry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2020 Res...</td>\n",
       "      <td>/article/4381810-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3M (MMM) CEO, Michael Roman on Q2 2020 Results...</td>\n",
       "      <td>/article/4361384-3m-mmm-ceo-michael-roman-on-q...</td>\n",
       "      <td>2020-07-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q1 2020 Res...</td>\n",
       "      <td>/article/4340658-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2020-04-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3M's (MMM) CEO Mike Roman on Q4 2019 Results -...</td>\n",
       "      <td>/article/4319588-3ms-mmm-ceo-mike-roman-on-q4-...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3M Company (MMM) CEO Mike Roman on Q3 2019 Res...</td>\n",
       "      <td>/article/4298957-3m-company-mmm-ceo-mike-roman...</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>MMM</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5268</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...</td>\n",
       "      <td>/article/4259182-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5269</th>\n",
       "      <td>Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...</td>\n",
       "      <td>/article/4241305-zoetis-inc-zts-ceo-juan-ramon...</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5270</th>\n",
       "      <td>Zoetis (ZTS) Q3 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4217256-zoetis-zts-q3-2018-results-ea...</td>\n",
       "      <td>2018-11-01</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5271</th>\n",
       "      <td>Zoetis (ZTS) Q2 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4193916-zoetis-zts-q2-2018-results-ea...</td>\n",
       "      <td>2018-08-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5272</th>\n",
       "      <td>Zoetis (ZTS) Q1 2018 Results - Earnings Call T...</td>\n",
       "      <td>/article/4168618-zoetis-zts-q1-2018-results-ea...</td>\n",
       "      <td>2018-05-02</td>\n",
       "      <td>ZTS</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5273 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     3M Company (MMM) CEO Mike Roman on Q3 2020 Res...   \n",
       "1     3M (MMM) CEO, Michael Roman on Q2 2020 Results...   \n",
       "2     3M Company (MMM) CEO Mike Roman on Q1 2020 Res...   \n",
       "3     3M's (MMM) CEO Mike Roman on Q4 2019 Results -...   \n",
       "4     3M Company (MMM) CEO Mike Roman on Q3 2019 Res...   \n",
       "...                                                 ...   \n",
       "5268  Zoetis, Inc. (ZTS) CEO Juan Ramon Alaix on Q1 ...   \n",
       "5269  Zoetis, Inc. (ZTS) CEO Juan Ramón Alaix on Q4 ...   \n",
       "5270  Zoetis (ZTS) Q3 2018 Results - Earnings Call T...   \n",
       "5271  Zoetis (ZTS) Q2 2018 Results - Earnings Call T...   \n",
       "5272  Zoetis (ZTS) Q1 2018 Results - Earnings Call T...   \n",
       "\n",
       "                                                    url       date Symbol  \\\n",
       "0     /article/4381810-3m-company-mmm-ceo-mike-roman... 2020-10-27    MMM   \n",
       "1     /article/4361384-3m-mmm-ceo-michael-roman-on-q... 2020-07-28    MMM   \n",
       "2     /article/4340658-3m-company-mmm-ceo-mike-roman... 2020-04-28    MMM   \n",
       "3     /article/4319588-3ms-mmm-ceo-mike-roman-on-q4-... 2020-01-28    MMM   \n",
       "4     /article/4298957-3m-company-mmm-ceo-mike-roman... 2019-10-24    MMM   \n",
       "...                                                 ...        ...    ...   \n",
       "5268  /article/4259182-zoetis-inc-zts-ceo-juan-ramon... 2019-05-02    ZTS   \n",
       "5269  /article/4241305-zoetis-inc-zts-ceo-juan-ramon... 2019-02-14    ZTS   \n",
       "5270  /article/4217256-zoetis-zts-q3-2018-results-ea... 2018-11-01    ZTS   \n",
       "5271  /article/4193916-zoetis-zts-q2-2018-results-ea... 2018-08-02    ZTS   \n",
       "5272  /article/4168618-zoetis-zts-q1-2018-results-ea... 2018-05-02    ZTS   \n",
       "\n",
       "      GICS Sector         GICS Sub-Industry  \n",
       "0     Industrials  Industrial Conglomerates  \n",
       "1     Industrials  Industrial Conglomerates  \n",
       "2     Industrials  Industrial Conglomerates  \n",
       "3     Industrials  Industrial Conglomerates  \n",
       "4     Industrials  Industrial Conglomerates  \n",
       "...           ...                       ...  \n",
       "5268  Health Care           Pharmaceuticals  \n",
       "5269  Health Care           Pharmaceuticals  \n",
       "5270  Health Care           Pharmaceuticals  \n",
       "5271  Health Care           Pharmaceuticals  \n",
       "5272  Health Care           Pharmaceuticals  \n",
       "\n",
       "[5273 rows x 6 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECT500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}