ch8/named-entity-recognition/ner-with-haystack-search-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from haystack import Pipeline\n",
    "from haystack.components.preprocessors import DocumentCleaner\n",
    "from haystack.components.websearch import SerperDevWebSearch\n",
    "from haystack.components.fetchers import LinkContentFetcher\n",
    "from haystack.components.converters import HTMLToDocument\n",
    "from haystack.components.writers import DocumentWriter\n",
    "from haystack import Pipeline\n",
    "from haystack.components.extractors import NamedEntityExtractor\n",
    "from haystack import component, Document\n",
    "from typing import Any, Dict, List, Union\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "load_dotenv(\".env\")\n",
    "open_ai_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "serper_api_key = os.getenv(\"SERPERDEV_API_KEY\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define custom component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@component\n",
    "class NERPopulator():\n",
    "    \"\"\"This function extracts named entities from a list of\n",
    "    documents and returns the result in a structured format.\n",
    "\n",
    "    Args:\n",
    "        documents (list): List of Haystack Document objects\n",
    "\n",
    "    Returns:\n",
    "        extracted_data (list): A list of dictionaries containing the extracted entities, \n",
    "        to make it Haystack-compatible we will return this list as a dictionary with the key 'documents'\n",
    "    \"\"\"\n",
    "    \n",
    "    @component.output_types(documents=List[Document])\n",
    "    def run(self, sources: List[Document]) -> None:\n",
    "        extracted_data = []\n",
    "\n",
    "        for document in sources:\n",
    "            content = document.content\n",
    "            doc_id = document.id\n",
    "            named_entities = document.meta.get('named_entities', [])\n",
    "            url = document.meta.get('url', 'N/A')  # Default to 'N/A' if URL is not available\n",
    "\n",
    "            # Sets to store unique entities by type\n",
    "            entities_by_type = {\n",
    "                \"LOC\": set(),\n",
    "                \"PER\": set(),\n",
    "                \"ORG\": set(),\n",
    "                \"MISC\": set()\n",
    "            }\n",
    "            \n",
    "            # Loop through the entities and filter by score and type\n",
    "            for entity in named_entities:\n",
    "                if float(entity.score) < 0.8:\n",
    "                    continue\n",
    "                \n",
    "                word = content[entity.start:entity.end]\n",
    "                if entity.entity in entities_by_type:\n",
    "                    entities_by_type[entity.entity].add(word)  # Use set to ensure uniqueness\n",
    "            \n",
    "            # Prepare the meta field with comma-separated values\n",
    "            meta = {\n",
    "                \"LOC\": \",\".join(entities_by_type[\"LOC\"]),\n",
    "                \"PER\": \",\".join(entities_by_type[\"PER\"]),\n",
    "                \"ORG\": \",\".join(entities_by_type[\"ORG\"]),\n",
    "                \"MISC\": \",\".join(entities_by_type[\"MISC\"]),\n",
    "                \"url\": url\n",
    "            }\n",
    "            \n",
    "            # Append the result for this document\n",
    "            extracted_data.append({\n",
    "                'document_id': doc_id,\n",
    "                'content': content,\n",
    "                'meta': meta\n",
    "            })\n",
    "        \n",
    "\n",
    "        return {\"documents\": extracted_data}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Haystack pipeline with custom component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at dslim/bert-base-NER were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<haystack.core.pipeline.pipeline.Pipeline object at 0x3057230b0>\n",
       "🚅 Components\n",
       "  - search: SerperDevWebSearch\n",
       "  - fetcher: LinkContentFetcher\n",
       "  - htmldocument: HTMLToDocument\n",
       "  - cleaner: DocumentCleaner\n",
       "  - extractor: NamedEntityExtractor\n",
       "  - ner: NERPopulator\n",
       "🛤️ Connections\n",
       "  - search.links -> fetcher.urls (List[str])\n",
       "  - fetcher.streams -> htmldocument.sources (List[ByteStream])\n",
       "  - htmldocument.documents -> cleaner.documents (List[Document])\n",
       "  - cleaner.documents -> extractor.documents (List[Document])\n",
       "  - extractor.documents -> ner.sources (List[Document])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Initialize pipeline\n",
    "pipeline = Pipeline()\n",
    "web_search = SerperDevWebSearch(top_k=10,\n",
    "                                allowed_domains=[\"https://ca.finance.yahoo.com/\"])\n",
    "link_content = LinkContentFetcher(retry_attempts=3,\n",
    "                                  timeout=10)\n",
    "html_to_doc = HTMLToDocument()\n",
    "document_cleaner = DocumentCleaner(\n",
    "                                remove_empty_lines=True,\n",
    "                                remove_extra_whitespaces=True,\n",
    "                                remove_repeated_substrings=False,\n",
    "                                remove_substrings=['\\n-']\n",
    "                            )\n",
    "extractor = NamedEntityExtractor(backend=\"hugging_face\", model=\"dslim/bert-base-NER\")\n",
    "extractor.warm_up()\n",
    "\n",
    "ner_component = NERPopulator()\n",
    "\n",
    "# Add components\n",
    "pipeline.add_component(name='search', instance=web_search)\n",
    "pipeline.add_component(name ='fetcher' , instance= link_content)\n",
    "pipeline.add_component(name='htmldocument', instance=html_to_doc)\n",
    "pipeline.add_component(name='cleaner', instance=document_cleaner)\n",
    "pipeline.add_component(name='extractor', instance=extractor)\n",
    "pipeline.add_component(name='ner', instance=ner_component)\n",
    "\n",
    "# Connect components to one another\n",
    "pipeline.connect(\"search.links\", \"fetcher.urls\")\n",
    "pipeline.connect(\"fetcher\", \"htmldocument\")\n",
    "pipeline.connect(\"htmldocument\", \"cleaner\")\n",
    "pipeline.connect(\"cleaner\", \"extractor\")\n",
    "pipeline.connect(\"extractor\", \"ner\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use pipeline to search for all articles related to Elon Musk and extract entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"Elon Musk\"\n",
    "output = pipeline.run(data={\"search\":{\"query\":query}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "extracted_documents = output['ner']['documents']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save document as DataFrame object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document_id</th>\n",
       "      <th>content</th>\n",
       "      <th>meta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5fcf252d1a589288db70c6252d42b040bbbe8b1647211a...</td>\n",
       "      <td>Musk ascends as a political force beyond his w...</td>\n",
       "      <td>{'LOC': 'California,Washington,America', 'PER'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f08ab77c1b78b84b150b74d800e6fe8e03e1ef792a3402...</td>\n",
       "      <td>WASHINGTON (Reuters) -Elon Musk and SpaceX fac...</td>\n",
       "      <td>{'LOC': 'Washington,U.S.,Israel', 'PER': 'Joe ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>830b24f764cb2af6949f3a9ff77c9c310faefcb923174e...</td>\n",
       "      <td>In This Article:\\nThe richest person in the wo...</td>\n",
       "      <td>{'LOC': '', 'PER': 'Trump,V,k Ramaswamy,Jeff,i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02d1bb7493336996c815d9b5c5feb154385410e5ebaf95...</td>\n",
       "      <td>(Reuters) - Tesla CEO Elon Musk's net worth cr...</td>\n",
       "      <td>{'LOC': '', 'PER': 'Sophia,Chandra,Jeff Be,on ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a2f4a4d3ed4d36701eab9c8fbf5da39a274ce1a98608ea...</td>\n",
       "      <td>LONDON (AP) — It’s a photo that sent a tremor ...</td>\n",
       "      <td>{'LOC': 'Florida,U.K.,Mar-a-Lago,K,U,Britain,L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>21f41abc36296146fb7652a6a558cbe699d20bc092a318...</td>\n",
       "      <td>By Mike Scarcella\\n(Reuters) -OpenAI asked a f...</td>\n",
       "      <td>{'LOC': 'California,Oakland,U.S', 'PER': 'on M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>75a09a359ac2d1c5d1ab1fe884a5e332b27fe33cbc57cb...</td>\n",
       "      <td>Elon Musk has demanded to know if President Jo...</td>\n",
       "      <td>{'LOC': 'White House', 'PER': 'Spiro,Gensler,T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>6e0cad97242cda6c3c99eea3d22ee378efe8e1efdad597...</td>\n",
       "      <td>A 7-year-old rivalry between tech leaders Elon...</td>\n",
       "      <td>{'LOC': 'California,Oakland,U.S', 'PER': 'Sam ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>d5de664e379522c362aead76d58cfe8991bf57737c5702...</td>\n",
       "      <td>In This Article:\\nBy Tom Hals\\nWILMINGTON, Del...</td>\n",
       "      <td>{'LOC': 'U.S.,Delaware', 'PER': 'Mu,Kathale,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1eeb07b5eb740d1717c5b1e86606c1eb1bdca9867701b5...</td>\n",
       "      <td>MUSCAT, Oman, Dec. 19, 2024 /CNW/ -- Oman Inve...</td>\n",
       "      <td>{'LOC': 'Oman,United States', 'PER': 'hidi,Abd...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         document_id  \\\n",
       "0  5fcf252d1a589288db70c6252d42b040bbbe8b1647211a...   \n",
       "1  f08ab77c1b78b84b150b74d800e6fe8e03e1ef792a3402...   \n",
       "2  830b24f764cb2af6949f3a9ff77c9c310faefcb923174e...   \n",
       "3  02d1bb7493336996c815d9b5c5feb154385410e5ebaf95...   \n",
       "4  a2f4a4d3ed4d36701eab9c8fbf5da39a274ce1a98608ea...   \n",
       "5  21f41abc36296146fb7652a6a558cbe699d20bc092a318...   \n",
       "6  75a09a359ac2d1c5d1ab1fe884a5e332b27fe33cbc57cb...   \n",
       "7  6e0cad97242cda6c3c99eea3d22ee378efe8e1efdad597...   \n",
       "8  d5de664e379522c362aead76d58cfe8991bf57737c5702...   \n",
       "9  1eeb07b5eb740d1717c5b1e86606c1eb1bdca9867701b5...   \n",
       "\n",
       "                                             content  \\\n",
       "0  Musk ascends as a political force beyond his w...   \n",
       "1  WASHINGTON (Reuters) -Elon Musk and SpaceX fac...   \n",
       "2  In This Article:\\nThe richest person in the wo...   \n",
       "3  (Reuters) - Tesla CEO Elon Musk's net worth cr...   \n",
       "4  LONDON (AP) — It’s a photo that sent a tremor ...   \n",
       "5  By Mike Scarcella\\n(Reuters) -OpenAI asked a f...   \n",
       "6  Elon Musk has demanded to know if President Jo...   \n",
       "7  A 7-year-old rivalry between tech leaders Elon...   \n",
       "8  In This Article:\\nBy Tom Hals\\nWILMINGTON, Del...   \n",
       "9  MUSCAT, Oman, Dec. 19, 2024 /CNW/ -- Oman Inve...   \n",
       "\n",
       "                                                meta  \n",
       "0  {'LOC': 'California,Washington,America', 'PER'...  \n",
       "1  {'LOC': 'Washington,U.S.,Israel', 'PER': 'Joe ...  \n",
       "2  {'LOC': '', 'PER': 'Trump,V,k Ramaswamy,Jeff,i...  \n",
       "3  {'LOC': '', 'PER': 'Sophia,Chandra,Jeff Be,on ...  \n",
       "4  {'LOC': 'Florida,U.K.,Mar-a-Lago,K,U,Britain,L...  \n",
       "5  {'LOC': 'California,Oakland,U.S', 'PER': 'on M...  \n",
       "6  {'LOC': 'White House', 'PER': 'Spiro,Gensler,T...  \n",
       "7  {'LOC': 'California,Oakland,U.S', 'PER': 'Sam ...  \n",
       "8  {'LOC': 'U.S.,Delaware', 'PER': 'Mu,Kathale,en...  \n",
       "9  {'LOC': 'Oman,United States', 'PER': 'hidi,Abd...  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(extracted_documents)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'document_id': '5fcf252d1a589288db70c6252d42b040bbbe8b1647211a465891d3e773257f80',\n",
       "  'content': 'Musk ascends as a political force beyond his wealth by tanking budget deal\\n1 / 2\\nMusk ascends as a political force beyond his wealth by tanking budget deal\\nThomas Beaumont\\n5 min read\\nIn the first major flex of his influence since Donald Trump was elected, Elon Musk brought to a sudden halt a bipartisan budget proposal by posting constantly on his X megaphone and threatening Republicans with primary challenges.\\nThe social media warnings from the world\\'s wealthiest man preceded Trump\\'s condemnation of a measure negotiated by GOP House Speaker Mike Johnson, which effectively killed the stopgap measure that was designed to prevent a partial shutdown of the federal government.\\nWashington was scrambled a day after Musk\\'s public pressure campaign. Trump on Thursday first declined to say whether he had confidence in Johnson. But later in the day, Trump praised him and House leaders for producing “a very good Deal,” after they announced a new plan to fund the government and lift the debt ceiling.\\nBefore the new deal was reached, Congressional Democrats mocked their GOP counterparts, with several suggesting Trump had been relegated to vice president.\\n“Welcome to the Elon Musk presidency,” Democratic Rep. Robert Garcia of California wrote on X.\\nWhat was clear, though, is Musk’s ascendance as a political force, a level of influence enabled by his great wealth. In addition to owning X, Musk is the CEO of Tesla and Space X.\\n“There is no doubt he does wield a lot of influence over Republicans right now due to his proximity to Trump,” said Chris Pack, former communications director for the National Republican Congressional Committee and the Senate Leadership Fund.\\nBut Pack also said that Musk\\'s threats pose potential risks for House Republicans, who begin next year with a five-seat majority that will shrink temporarily because of Trump’s nomination of some GOP lawmakers to administration posts.\\n“This isn’t going to help pass the agenda if you are going to cost a bunch of Republicans in very razor-thin moderate seats if you’re going to make them lose in primaries,” Pack said. “All that does is give the keys to these districts over to the Democrats.”\\nMusk spent an estimated $250 million during the presidential campaign to support Trump, contributing heavily to America PAC, a super political action committee that deployed canvassers, aired TV ads and reached voters digitally in battleground states. He had signaled after the election he was willing to back GOP primary challenges to Republican members of Congress seeking re-election in 2026 who waver on Trump\\'s appointments and agenda.\\nHe renewed the threat pointedly Wednesday.\\n“Any member of the House or Senate who votes for this outrageous spending bill deserves to be voted out in 2 years!” he wrote on X. He also called it “one of the worst bills ever written.”\\nMusk wasn\\'t alone in fanning GOP anger against the bill, which included several compromise measures to get Democratic support in the Senate in the final weeks before Republicans take control of that chamber. Biotech entrepreneur and former GOP presidential candidate Vivek Ramaswamy, who is Musk’s partner leading the new Department of Government Efficiency, also posted against the bill, as did Trump\\'s eldest son, Donald Trump Jr.\\nMusk played down his role at times, suggesting after some praise online, “All I can do is bring things to the attention of the people, so they may voice their support if they so choose.” And the president-elect told NBC News that he had spoken to Musk prior to the Tesla CEO\\'s first posts.\\n“I told him that if he agrees with me, that he could put out a statement,” Trump said.\\nKaroline Leavitt, the incoming White House press secretary, pushed back against Democratic critics who suggested Musk was calling the shots.\\n“As soon as President Trump released his official stance on the CR, Republicans on Capitol Hill echoed his point of view,\" Leavitt said in a statement, referring to the continuing resolution. \"President Trump is the leader of the Republican Party. Full stop.”\\nThroughout the day Wednesday, Musk replied to posts on X from Republican House members announcing opposition to the bill with words of thanks, and punctuating their public commitments.\\nAnd he took a victory lap after Trump came out against the bill: “The voice of the people was heard. This was a good day for America.\"\\nHe was responding to Kentucky Rep. Andy Barr\\'s post: “The phone was ringing off the hook today. And you know why? Because they were reading tweets...from Musk and Vivek Ramaswamy.”\\nBy late afternoon Thursday and before the new plan was put to a vote, Musk declared success and posted a picture of the new, slimmed down alternative alongside the voluminous, original measure he helped kill. “This shows how much your voice matters,” he posted. “And having a President like @realDonaldTrump means that your voice is finally heard.”\\nConservative activists at the annual AmericaFest gathering in Phoenix cheered Musk Thursday and hailed the suggestion he could replace Johnson as speaker. There’s no requirement that the speaker be an elected member of the House of Representatives.\\n“Should Mike Johnson remain speaker of the House?” conservative media host Jack Posobiec asked his audience during a live taping of his talk show, prompting a chorus of “Noooooo!!!” from his audience.\\nJohnson had been scheduled to attend AmericaFest, but canceled after the budget deal fell apart.\\n“Should Elon Musk be speaker of the House?” Posobiec asked his audience, prompting cheers.\\n___\\nBeaumont reported from Des Moines, Iowa. Associated Press Bill Barrow contributed to this report from Phoenix.',\n",
       "  'meta': {'LOC': 'California,Washington,America',\n",
       "   'PER': 'Mike Johnson,Thomas Beaumont,Pack,on Musk,Trump,Chris Pack,Johnson,Robert Garcia,Mu,Donald Trump',\n",
       "   'ORG': 'X,House,Tesla and Space X,PA,National Republican Congressional Committee,Senate Leadership Fund',\n",
       "   'MISC': 'Rep,Democrats,Deal,Democratic,GOP,Republicans',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/musk-ascends-political-force-beyond-204319247.html'}},\n",
       " {'document_id': 'f08ab77c1b78b84b150b74d800e6fe8e03e1ef792a3402c233c59a435e1e31a4',\n",
       "  'content': 'WASHINGTON (Reuters) -Elon Musk and SpaceX face at least three federal reviews over whether they have complied with federal reporting rules aimed at protecting national security, the New York Times reported on Tuesday.\\nThis is not the first time the security practices of the SpaceX founder have been questioned by the Pentagon.\\nA 2018 incident in which Musk, who is also CEO of Tesla, smoked marijuana on a live Web show with U.S. comedian Joe Rogan also sparked a security clearance review by the Pentagon.\\nSuch reviews ordinarily occur once instances of a potential infraction come to light, Pentagon officials have said.\\nThe New York Times said the new reviews were initiated by the Air Force, the Defense Department\\'s Inspector General and the undersecretary of defense for intelligence and security. It cited eight people with knowledge of the rocket company and internal documents.\\n\"Elon Musk and his rocket company, SpaceX, have repeatedly failed to comply with federal reporting protocols aimed at protecting state secrets, including by not providing some details of his meetings with foreign leaders,\" the newspaper report said.\\nThe Air Force and undersecretary of defense for intelligence and security referred queries to the Defense Department\\'s Office of Inspector General. That office, in turn, said that as a matter of policy, it could \"neither confirm nor deny the existence of an ongoing investigation.\" It added: \"This is to protect the confidentiality of those involved and to protect the integrity of the investigative process.\"\\nSpaceX President and Chief Operating Officer Gwynne Shotwell declined to comment on the security clearance issues or the probes when asked by a Reuters reporter as she left an event in Washington.\\nMusk did not immediately respond to a request for comment.\\nThe New York Times said the Air Force recently cited potential security risks in denying high-level security access for Musk, and that allies including Israel have flagged concerns that the billionaire CEO might share sensitive data.\\nLast month, two Democratic U.S. senators, Jeanne Shaheen and Jack Reed, wrote in a letter that reports that Musk has held multiple calls with Russian officials, including President Vladimir Putin, should be investigated by the Pentagon and law-enforcement agencies on national security grounds.\\nAir Force Secretary Frank Kendall responded last week to Shaheen that he shared those concerns but could not comment on the status of a person\\'s security clearance.\\n(Reporting by Doina Chiacu, Mike Stone and Joey Roulette in Washington; Editing by Rami Ayyub and Matthew Lewis)',\n",
       "  'meta': {'LOC': 'Washington,U.S.,Israel',\n",
       "   'PER': 'Joe Rogan,Frank,Jack,Mike,Joey,G,Putin,Mu,wynne Shotwell,Vladimir',\n",
       "   'ORG': 'Pentagon,Report,Elon Musk,WASHINGTON,Air Force,The New York Times,U,Jeanne,R,Reuters,Defense Department,SpaceX,Tesla,New York Times,Last,El',\n",
       "   'MISC': 'Democratic,Russian',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/elon-musk-spacex-face-federal-184331986.html'}},\n",
       " {'document_id': '830b24f764cb2af6949f3a9ff77c9c310faefcb923174ef7ee5815bf3a533b51',\n",
       "  'content': 'In This Article:\\nThe richest person in the world just hit an even higher milestone.\\nElon Musk’s net worth has reached $400 billion, according to Bloomberg, making him the first person ever to cross that mark.\\nBehind his nearly $20 billion jump in wealth was a deal that shot up Musk’s rocket company SpaceX’s valuation to roughly $350 billion, Bloomberg reported Tuesday. SpaceX and its investors agreed to purchase as much as $1.25 billion of insider shares.\\nCNN has reached out to SpaceX for comment, but the company typically does not respond to media requests.\\nSince the aftermath of the 2024 US presidential election, Musk has enjoyed a huge surge in his personal wealth.\\nAn alliance with President-elect Donald Trump has pushed his ventures to the forefront. Musk is the chief executive of Tesla and SpaceX, as well as the owner of X and CEO of other ventures, including Neuralink, xAI and the Boring Company. Now, along with Vivek Ramaswamy, he will oversee a new “Department of Government Efficiency” (DOGE), named after a memecoin.\\nTesla shares closed at a record high on Wednesday, hitting $424.77 at market close. Musk, who is Tesla’s largest individual shareholder, has also enjoyed the stock propelling higher in part due to the broader market — NASDAQ topped 20,000 for the first time ever Wednesday. Since Election Day, the EV maker’s stock has rallied roughly 65% on investors’ belief that Musk’s influence in the Trump administration will usher in an era of deregulation that will benefit the company.\\nunknown content item\\nHis artificial intelligence startup, xAI, also more than doubled in value in November amid a new funding round, surging to $50 billion from a few months ago, according to the Wall Street Journal.\\nThe world’s wealthiest man is now roughly $136 billion richer since November 5, according to the Bloomberg Billionaires Index.\\nWhile the top spots on the billionaires index typically switch around, Musk is racing past his wealthy peers. As of December 10, before his wealth skyrocketed to $400 billion, he was already $140 billion richer than the second-richest man on Earth, Amazon founder Jeff Bezos.\\nMusk’s record-breaking Tesla-shareholder-approved pay package, which was worth $101 billion, was rejected by a Delaware court last week.\\nBloomberg’s calculation still seems to include the pay package, which is now worth as much as $120 billion based on the current value of Tesla shares.\\nBut Musk will still enjoy his riches, with numerous ways to gain significantly more wealth in the coming years.\\nFor more CNN news and newsletters create an account at CNN.com',\n",
       "  'meta': {'LOC': '',\n",
       "   'PER': 'Trump,V,k Ramaswamy,Jeff,ive,Bloomberg,Mu,Donald Trump',\n",
       "   'ORG': 'Bloomberg,X,xAI,Boring Company,D,CNN,Journal,SpaceX,Tesla,euralink,N,El',\n",
       "   'MISC': 'Election Day',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/elon-musk-becomes-first-person-184648913.html'}},\n",
       " {'document_id': '02d1bb7493336996c815d9b5c5feb154385410e5ebaf959311330b6155e1f14f',\n",
       "  'content': '(Reuters) - Tesla CEO Elon Musk\\'s net worth crossed $400 billion on Thursday, the Forbes real-time billionaires list showed, boosted by a nearly 71% surge in the automaker\\'s shares this year and the soaring valuation of his rocket company SpaceX.\\nMusk is Tesla\\'s largest shareholder with about a 13% stake in the company. The EV maker\\'s shares rose to a record high of $424.9 on Wednesday, extending a rally that kicked off following the Nov. 5 election.\\nHis net worth was also boosted by an insider share sale of SpaceX, which Musk heads. The rocket company and its investors agreed to purchase as much as $1.25 billion of its common shares, Bloomberg News reported on Tuesday.\\n\"What\\'s really crazy about this is that almost no investors wanted to sell shares even at a $350B valuation!\" Musk wrote in a reply to a post on X referring to the Bloomberg report.\\nHe added that SpaceX has reduced the number of shares it bought back from employees in order to bring in some new investors.\\nTesla investors have pinned their hopes on Musk\\'s relationship with President-elect Donald Trump, hoping the billionaire\\'s role in shaping the incoming Republican administration\\'s policy agenda would benefit the automaker.\\nBesides Tesla, Musk also heads brain technology startup Neuralink. Both SpaceX and Neuralink rely heavily on government regulation and policy and could benefit from a business-friendly administration.\\nMusk, who spent over a quarter of a billion dollars to help Trump win the election, also owns social media site X and tunnel builder The Boring Company.\\nMusk is followed on the billionaires list by Amazon founder Jeff Bezos - with a net worth of $243.7 billion, while Oracle co-founder Larry Ellison is in the third spot with a net worth of $223.4 billion.\\n(Reporting by Deborah Sophia and Rishi Kant in Bengaluru; Editing by Krishna Chandra Eluri and Anil D\\'Silva)',\n",
       "  'meta': {'LOC': '',\n",
       "   'PER': 'Sophia,Chandra,Jeff Be,on Musk,Ellison,Trump,Deborah,Silva,Larry,Mu,Donald Trump',\n",
       "   'ORG': 'X,Neuralink,The Boring Company,Reuters,Bloomberg News,SpaceX,Tesla,Oracle,euralink,Amazon,N,El',\n",
       "   'MISC': 'Republican',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/elon-musks-net-worth-tops-143636708.html'}},\n",
       " {'document_id': 'a2f4a4d3ed4d36701eab9c8fbf5da39a274ce1a98608ea65f755d1474215bc5a',\n",
       "  'content': \"LONDON (AP) — It’s a photo that sent a tremor through British politics: Elon Musk flanked by British politician Nigel Farage and a wealthy backer, in front of a gilt-framed painting of a young Donald Trump.\\nTaken this week at Trump’s Mar-a-Lago estate in Florida, the image suggested that Musk, a key player in the incoming U.S. administration, could soon turn his disruptive attention to the U.K.\\nFarage, Trump’s highest-profile British champion, confirmed talks are underway about Musk making a hefty donation to Farage's party Reform U.K. The Times of London reported it could be as much as 100 million pounds ($127 million), which would be far and away the largest political donation in U.K. history. The reports have sparked calls for Britain’s rules on political donations to be tightened — quickly.\\n“We did discuss money,” Farage told broadcaster GB News after the meeting with Musk. “That’s a negotiation we will go back and have again. He is not against giving us money. He hasn’t fully decided whether he will.”\\nBritain has strict limits on how much political parties can spend on elections, but they can accept unlimited donations, as long as the donors are U.K. voters or companies registered in Britain. Musk’s social network X has a British arm, Twitter U.K. Ltd., with a registered address in London.\\nCritics say that is a loophole that allows foreign influence in U.K. politics. The voting watchdog, the Electoral Commission, is calling for changes including limiting the amount a company can donate, so that it can't put in more money than it earns in Britain.\\n“It’s crucial that U.K. voters have trust in the financing of our political system,” the commission's chief executive Vijay Rangarajan told The Guardian. “The system needs strengthening, and we have been calling for changes to the law since 2013, to protect the electoral system from foreign interference.”\\nBritain’s center-left Labour Party pledged during the summer election campaign to tighten the rules on political donations, though legislation is not scheduled in the coming year. Prime Minister Keir Starmer's spokesman Dave Pares said Wednesday that work is already underway to “reinforce existing safeguards” against “impermissible proxy donations.”\\nBoth the Labour government and the right-of-center opposition Conservatives are trying to figure out how to deal with Musk, who has taken a keen interest in the U.K. – and seemingly formed a strong dislike for Starmer.\\nMusk often posts on X about the U.K., retweeting criticism of Starmer and the hashtag TwoTierKeir – shorthand for an unsubstantiated claim that Britain has “two-tier policing,” with far-right protesters treated more harshly than pro-Palestinian or Black Lives Matter demonstrators. Musk has compared British attempts to weed out online misinformation to the Soviet Union, and during summer anti-immigrant violence across the U.K. tweeted that “civil war is inevitable.”\",\n",
       "  'meta': {'LOC': 'Florida,U.K.,Mar-a-Lago,K,U,Britain,London,U.S.',\n",
       "   'PER': 'Farage,on Musk,Trump,Nigel Farage,Mu,Donald Trump',\n",
       "   'ORG': 'Electoral Commission,X,Twitter U.K. Ltd,Pa,GB News,Ke,Times of London,Labour,Reform U.K,Star,Ra,Guardian,Labour Party,AP',\n",
       "   'MISC': 'British',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/elon-musk-considers-funding-nigel-150306115.html'}},\n",
       " {'document_id': '21f41abc36296146fb7652a6a558cbe699d20bc092a31886165f4579c20eb92b',\n",
       "  'content': 'By Mike Scarcella\\n(Reuters) -OpenAI asked a federal judge in California on Friday to reject a request by billionaire Elon Musk to halt the ChatGPT maker’s conversion to a for-profit company.\\nOpenAI also published a trove of emails and text messages with Musk on its website to argue that he initially backed for-profit status for OpenAI before walking away from the company after failing to get a majority equity stake and full control.\\nMusk, who was an OpenAI cofounder, has since launched a competing artificial intelligence company, xAI.\\nMusk sued OpenAI, its CEO Sam Altman and others in August, claiming they violated contract provisions by putting profits ahead of the public good in the push to advance AI. In November, he asked U.S. District Judge Yvonne Gonzalez Rogers in Oakland for a preliminary injunction blocking OpenAI from converting to a for-profit structure.\\nA lawyer for Musk did not immediately respond to a request for comment on Friday.\\nOpenAI\\'s blog post said Musk \"should be competing in the marketplace rather than the courtroom.\"\\nMusk has since added Microsoft and others as defendants to his lawsuit, alleging OpenAI was scheming to sideline rivals and monopolize the market for generative artificial intelligence.\\nOpenAI\\'s court filing denied any conspiracy to restrain AI market competition, and it said Musk\\'s request for a preliminary injunction was based on \"unsupported allegations.\"\\nIn a separate court filing, Microsoft on Friday said it and OpenAI \"are independent companies that each pursue their own strategies and compete vigorously with each other and many others.\" Microsoft said its OpenAI partnership has \"fueled innovation between them and others.\"\\nOpenAI started as a nonprofit in 2014 and has become the face of generative AI through billions of dollars in funding from Microsoft. In October, it closed a $6.6 billion funding round from investors, which could value the company at $157 billion.\\nMusk’s xAI earlier this month said it had raised about $6 billion in equity financing.\\nOpenAI is working on a plan to restructure its core business into a for-profit benefit corporation. The OpenAI nonprofit would own a minority stake in the for-profit company.\\nRogers is scheduled to hear arguments on Musk\\'s injunction bid on Jan. 14.\\n(Reporting by Mike Scarcella; Editing by David Bario, Daniel Wallis and Bill Berkrot)',\n",
       "  'meta': {'LOC': 'California,Oakland,U.S',\n",
       "   'PER': 'on Musk,Sam Altman,Mike Scarcella,Mu,Yvonne Gonzalez Rogers',\n",
       "   'ORG': 'xAI,Sc,OpenA,Mike,Reuters,AI,Cha,xA,Rogers,Microsoft,OpenAI,El',\n",
       "   'MISC': 'AI',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/openai-fires-back-elon-musk-212307411.html'}},\n",
       " {'document_id': '75a09a359ac2d1c5d1ab1fe884a5e332b27fe33cbc57cb6fe5d57a0c00205f45',\n",
       "  'content': 'Elon Musk has demanded to know if President Joe Biden is targeting him after reigniting his bitter feud with the Securities and Exchange Commission.\\nThe billionaire Trump confidante has accused the SEC of harassing him for over six years and suggested the campaign against him was politically motivated.\\nHe posted a letter on his X social media platform revealing that the SEC has reopened an investigation into his brain-chip startup Neuralink.\\nThe letter from Musk’s attorney Alex Spiro also said the SEC gave him a 48-hour ultimatum to settle a probe into his $44 billion investment in Twitter or risk facing charges.\\n“Oh Gary, how could you do this to me?” Musk wrote with a teary emoji on X in a reference to outgoing SEC Chair Gary Gensler.\\nSpiro writes that the demands come after “more than six years of harassment of Mr. Musk by the Commission and its Staff.”\\nHe said the SEC indicated the enforcement action was “the result of a directive from their superiors”. He continued: “This series of events makes clear that the Commission is not motivated to seek the truth but instead is engaged in an improperly motivated campaign against Mr. Must and the individuals and companies associated with him.\\n“We demand to know who directed these actions,” he asked Gensler, “whether it was you or the White House.”\\nSpiro concludes that he and Musk will not be scared off, saying: “These tactics and misguided scheme will not intimidate us.”\\nMusk has taken digs at the SEC over the years since 2018 when he was sued after tweeting about taking his Tesla electric car company private. The Neuralink probe reportedly involves allegations that Musk misled investors about a brain implant it developed.\\nThe Tesla founder stumped up more than a quarter of a billion dollars to boost Donald Trump’s election campaign and will be heading up a new cost-cutting task force in the new administration.',\n",
       "  'meta': {'LOC': 'White House',\n",
       "   'PER': 'Spiro,Gensler,Trump,Must,Alex Spiro,Gary,Joe B,Gary Gensler,Spi,”,Mu,sk,Donald',\n",
       "   'ORG': 'X,Neuralink,Elon,Tesla,Twitter,SEC,Commission,Securities and Exchange Commission',\n",
       "   'MISC': '',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/whining-elon-musk-squeals-sec-103752120.html'}},\n",
       " {'document_id': '6e0cad97242cda6c3c99eea3d22ee378efe8e1efdad59775bd8d8d20cd672f71',\n",
       "  'content': 'A 7-year-old rivalry between tech leaders Elon Musk and Sam Altman over who should run OpenAI and prevent an artificial intelligence \"dictatorship\" is now heading to a federal judge as Musk seeks to halt the ChatGPT maker\\'s ongoing shift into a for-profit company.\\nMusk, an early OpenAI investor and board member, sued the artificial intelligence company earlier this year alleging it had betrayed its founding aims as a nonprofit research lab benefiting the public good rather than pursuing profits.\\nMusk has since escalated the dispute, adding new claims and asking for a court order that would stop OpenAI’s plans to convert itself into a for-profit business more fully.\\nThe world\\'s richest man, whose companies include Tesla, SpaceX and social media platform X, last year started his own rival AI company, xAI. Musk says it faces unfair competition from OpenAI and its close business partner Microsoft, which has supplied the huge computing resources needed to build AI systems such as ChatGPT.\\n“OpenAI and Microsoft together exploiting Musk’s donations so they can build a for-profit monopoly, one now specifically targeting xAI, is just too much,” says Musk\\'s filing that alleges the companies are violating the terms of Musk’s foundational contributions to the charity.\\nOpenAI filed a response Friday opposing Musk’s requested order, saying it would “debilitate OpenAI’s business” and mission to the advantage of Musk and his own AI company and is based on “far-fetched” legal claims. A hearing is set for January before U.S. District Judge Yvonne Gonzalez Rogers in Oakland, California.\\nAt the heart of the dispute is a 2017 internal power struggle at the fledgling startup that led to Altman becoming OpenAI\\'s CEO.\\nMusk also sought to be CEO and in an email outlined a plan where he would “unequivocally have initial control of the company” but said that would be temporary. He grew frustrated after two other OpenAI co-founders said he would hold too much power as a major shareholder and chief executive if the startup succeeded in its goal to achieve better-than-human AI known as artificial general intelligence, or AGI. Musk has long voiced concerns about how advanced forms of AI could threaten humanity.\\n“The current structure provides you with a path where you end up with unilateral absolute control over the AGI,\" said a 2017 email to Musk from co-founders Ilya Sutskever and Greg Brockman. “You stated that you don\\'t want to control the final AGI, but during this negotiation, you\\'ve shown to us that absolute control is extremely important to you.”',\n",
       "  'meta': {'LOC': 'California,Oakland,U.S',\n",
       "   'PER': 'Sam Altman,Mu,Yvonne Gonzalez Rogers,on Musk',\n",
       "   'ORG': 'X,xAI,AG,AI,Cha,SpaceX,Open,Tesla,Microsoft,OpenAI,El',\n",
       "   'MISC': 'AI',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/openais-legal-battle-elon-musk-202720975.html'}},\n",
       " {'document_id': 'd5de664e379522c362aead76d58cfe8991bf57737c5702f45b3735ec1fe1fa57',\n",
       "  'content': 'In This Article:\\nBy Tom Hals\\nWILMINGTON, Delaware (Reuters) - Tesla cannot give Elon Musk a $56 billion compensation package despite a vote in support of the CEO\\'s pay deal by the company\\'s shareholders, a Delaware judge ruled on Monday.\\nBelow is a look at what could come next for Tesla and its billionaire founder, who is still seeking a huge payday from the company:\\nWHAT DOES MUSK WANT?\\nMusk told a special committee of the Tesla board soon after a judge voided his compensation in January that he wanted a similar-sized replacement package, according to a securities filing.\\nIn addition, earlier this year he said on his social media platform X that he wanted a larger stake in Tesla or he might develop some products outside the company. Musk\\'s other companies include rocket venture SpaceX and Neuralink, which develops brain implants.\\nTESLA COULD APPEAL THE RULING\\nMusk and Tesla\\'s board could appeal and try to reverse the ruling at the Delaware Supreme Court, a process that typically takes around a year.\\nThe case, which involved the largest-ever pay deal at a U.S. public company, raises issues that have rarely been addressed by Delaware judges, adding uncertainty to an appeal.\\nFor example, the trial court judge, Chancellor Kathaleen McCormick, found that Musk controlled the compensation negotiations, even though he owned only about 22% of Tesla\\'s stock.\\nIn addition, Tesla has acknowledged that the June vote by shareholders to ratify Musk\\'s pay was a \"novel\" legal tactic and has said it was unclear how it would be treated under Delaware law.\\nTESLA COULD DEVISE A NEW PLAN\\nTesla\\'s board could craft a new pay package, although that could be very expensive.\\nThe original plan, agreed to by Musk and the company in 2018, awarded him stock options if the company hit very aggressive performance and financial targets. The stock options allowed Musk to buy Tesla stock priced at the 2018 level. The company exceeded the targets, and Tesla\\'s stock has risen 10-fold since then, making the options incredibly valuable.\\nTesla booked a cost of $2.6 billion when the 2018 plan went into effect. The company has said that a replacement plan for the same cost today would likely have to be less than 10% of the size of the 2018 plan.\\nCOULD TESLA JUST RESTORE THE OLD PLAN?\\nTesla could offer Musk the same 304 million stock options with the same $23.34 exercise price used in the 2018 plan. If shareholders wanted to challenge that, they would have to sue in Texas, where the company reincorporated this year, rather than the Court of Chancery in Delaware.',\n",
       "  'meta': {'LOC': 'U.S.,Delaware',\n",
       "   'PER': 'Mu,Kathale,en McCormick,Tom Hals',\n",
       "   'ORG': 'TON,X,Musk,Neuralink,ESLA,PLAN\\nTesla,Delaware Supreme Court,T,Reuters,Te,SpaceX,Tesla,El',\n",
       "   'MISC': '',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/explainer-next-musk-judge-rules-004342200.html'}},\n",
       " {'document_id': '1eeb07b5eb740d1717c5b1e86606c1eb1bdca9867701b53f255d3db685f96db1',\n",
       "  'content': \"MUSCAT, Oman, Dec. 19, 2024 /CNW/ -- Oman Investment Authority (OIA), the investment arm of the Sultanate of Oman has invested in xAI, a leading artificial intelligence company that ranks among the top five in its industry and one of Elon Musk's companies. By focusing on advanced technologies, the authority aims to diversify its international portfolio while capitalizing on transformative industries with significant growth potential.\\nAccording to OIA, xAI has achieved two major milestones since its launch by Elon Musk in March 2023: the establishment of a large-scale data center in the United States and the rollout of its advanced platform, Grok-2. These achievements are expected to further strengthen the company's market position and drive stronger returns on OIA's investment.\\nHE Abdulsalam bin Mohammed Al-Murshidi, President of Oman Investment Authority, stated that the investment is the result of a strong and constructive relationship that OIA has built over the years with Elon Musk. He emphasized that this investment supports Oman's broader goals under Vision 2040, which enables the Sultanate to adopt world-class technologies and drive future progress.\\nxAI aims to be a pioneer in AI systems, offering the ability to process and analyze real-time data, an essential advantage over many competing AI systems. The platform provides advanced tools and detailed analytics, enabling users to optimize large AI models and manage various types of visual data, such as documents, charts, graphs, and photographs.\\nOIA is also a shareholder in SpaceX, Musk's groundbreaking aerospace company known for its Starlink satellite communication system. This collaboration has been highly successful, with OIA achieving an internal rate of return exceeding 37% on its SpaceX investment. The strong returns are attributed to SpaceX's rapid global expansion and the deployment of its satellite communication services in numerous countries worldwide.\\nAbout Oman Investment Authority (OIA)\\nOman Investment Authority (OIA) is the investment arm of the Sultanate entrusted with managing, investing, and developing the nation's national and international assets. With a diverse investment portfolio locally and globally, OIA invests in over 40 countries, targeting various sectors, including, energy, logistics, ICT, services, finance and investment, food security, tourism, mining, industry, and aviation. OIA is committed to driving sustainable economic growth and contributing to the well-being of Oman's people while also supporting the Sultanate's vision of becoming a leading investment destination in the region.\",\n",
       "  'meta': {'LOC': 'Oman,United States',\n",
       "   'PER': 'hidi,Abdulsalam bin Mohammed Al-Mu',\n",
       "   'ORG': 'Musk,I,xAI,Elon Musk,Authority,OIA,Oman Investment Authority,O,SpaceX,Starlink',\n",
       "   'MISC': 'Vision 2040,AI,Grok-2',\n",
       "   'url': 'https://ca.finance.yahoo.com/news/oman-investment-authority-invests-elon-132500023.html'}}]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLP pipelines",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}