Stop_words_Spacy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "223f7ec9-0488-4a7c-9d2e-54d7f8c80ea7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9ef3d2d2-0329-442c-9a31-28f42ed7a91a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "326"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(STOP_WORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7f778086-2f2f-45a9-972c-6bc856dbe9ec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good\n",
      "way\n",
      "start\n",
      "NLP\n",
      "journey\n",
      "exercising\n",
      "tutorial\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "# Steps to pre-process stop word\n",
    "sen = \"This is a good way to start you NLP journey by exercising this tutorial.\"\n",
    "doc = nlp(sen)\n",
    "\n",
    "for token in doc:\n",
    "    if not token.is_stop:\n",
    "        print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f1f4a8e-35b1-4406-b70b-9ab8b78f112e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def Preprocess(sent):\n",
    "    doc = nlp(sent)\n",
    "    \n",
    "    li = [token.text for token in doc if not token.is_stop and not token.is_punct]\n",
    "    return \" \".join(li) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bdc6a1d1-5e96-4d3f-9e95-31bcb2fd6787",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'primary example preprocess sentence paragraph'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Preprocess(\"This is a primary example of how to preprocess a sentence or paragraph.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "340b06c8-ddae-4ed2-b152-20bdaab07b47",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>Convicted Bomb Plotter Sentenced to 30 Years</td>\n",
       "      <td>PORTLAND, Oregon. – Mohamed Osman Mohamud, 23,...</td>\n",
       "      <td>2014-10-01T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[National Security Division (NSD)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12-919</td>\n",
       "      <td>$1 Million in Restitution Payments Announced t...</td>\n",
       "      <td>WASHINGTON – North Carolina’s Waccamaw River...</td>\n",
       "      <td>2012-07-25T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11-1002</td>\n",
       "      <td>$1 Million Settlement Reached for Natural Reso...</td>\n",
       "      <td>BOSTON– A $1-million settlement has been...</td>\n",
       "      <td>2011-08-03T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10-015</td>\n",
       "      <td>10 Las Vegas Men Indicted \\r\\nfor Falsifying V...</td>\n",
       "      <td>WASHINGTON—A federal grand jury in Las Vegas...</td>\n",
       "      <td>2010-01-08T00:00:00-05:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                              title  \\\n",
       "0     None       Convicted Bomb Plotter Sentenced to 30 Years   \n",
       "1  12-919   $1 Million in Restitution Payments Announced t...   \n",
       "2  11-1002  $1 Million Settlement Reached for Natural Reso...   \n",
       "3   10-015  10 Las Vegas Men Indicted \\r\\nfor Falsifying V...   \n",
       "4   18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "\n",
       "                                            contents  \\\n",
       "0  PORTLAND, Oregon. – Mohamed Osman Mohamud, 23,...   \n",
       "1    WASHINGTON – North Carolina’s Waccamaw River...   \n",
       "2        BOSTON– A $1-million settlement has been...   \n",
       "3    WASHINGTON—A federal grand jury in Las Vegas...   \n",
       "4  The U.S. Department of Justice, the U.S. Envir...   \n",
       "\n",
       "                        date         topics  \\\n",
       "0  2014-10-01T00:00:00-04:00             []   \n",
       "1  2012-07-25T00:00:00-04:00             []   \n",
       "2  2011-08-03T00:00:00-04:00             []   \n",
       "3  2010-01-08T00:00:00-05:00             []   \n",
       "4  2018-07-09T00:00:00-04:00  [Environment]   \n",
       "\n",
       "                                     components  \n",
       "0            [National Security Division (NSD)]  \n",
       "1  [Environment and Natural Resources Division]  \n",
       "2  [Environment and Natural Resources Division]  \n",
       "3  [Environment and Natural Resources Division]  \n",
       "4  [Environment and Natural Resources Division]  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_json('combined.json',lines=True)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3553381d-b8a7-46f2-9b4a-7be0a7572d69",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# df[\"New_content\"] = df.contents.apply(Preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0cb623a7-2362-499c-a012-20423609186a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13087, 6)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "26b8b85b-dc2b-412a-9188-d7481326b9aa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14-1412</td>\n",
       "      <td>14 Indicted in Connection with New England Com...</td>\n",
       "      <td>A 131-count criminal indictment was unsealed t...</td>\n",
       "      <td>2014-12-17T00:00:00-05:00</td>\n",
       "      <td>[Consumer Protection]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>17-1419</td>\n",
       "      <td>2017 Southeast Regional Animal Cruelty Prosecu...</td>\n",
       "      <td>The United States Attorney’s Office for the Mi...</td>\n",
       "      <td>2017-12-14T00:00:00-05:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division, U...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                              title  \\\n",
       "4    18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "7   14-1412  14 Indicted in Connection with New England Com...   \n",
       "19  17-1419  2017 Southeast Regional Animal Cruelty Prosecu...   \n",
       "\n",
       "                                             contents  \\\n",
       "4   The U.S. Department of Justice, the U.S. Envir...   \n",
       "7   A 131-count criminal indictment was unsealed t...   \n",
       "19  The United States Attorney’s Office for the Mi...   \n",
       "\n",
       "                         date                 topics  \\\n",
       "4   2018-07-09T00:00:00-04:00          [Environment]   \n",
       "7   2014-12-17T00:00:00-05:00  [Consumer Protection]   \n",
       "19  2017-12-14T00:00:00-05:00          [Environment]   \n",
       "\n",
       "                                           components  \n",
       "4        [Environment and Natural Resources Division]  \n",
       "7                                    [Civil Division]  \n",
       "19  [Environment and Natural Resources Division, U...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df.topics.str.len()!=0]\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a41deeae-9f02-4191-97dc-e48c4fbc0fa7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b0e73930-ce04-4718-9900-d18630242fc9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"New_content\"] = df.contents.apply(Preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b3ace344-3f38-441e-8210-0a0577471662",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "      <th>New_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "      <td>U.S. Department Justice U.S. Environmental Pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14-1412</td>\n",
       "      <td>14 Indicted in Connection with New England Com...</td>\n",
       "      <td>A 131-count criminal indictment was unsealed t...</td>\n",
       "      <td>2014-12-17T00:00:00-05:00</td>\n",
       "      <td>[Consumer Protection]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "      <td>131 count criminal indictment unsealed today B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>17-1419</td>\n",
       "      <td>2017 Southeast Regional Animal Cruelty Prosecu...</td>\n",
       "      <td>The United States Attorney’s Office for the Mi...</td>\n",
       "      <td>2017-12-14T00:00:00-05:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division, U...</td>\n",
       "      <td>United States Attorney Office Middle District ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                              title  \\\n",
       "4    18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "7   14-1412  14 Indicted in Connection with New England Com...   \n",
       "19  17-1419  2017 Southeast Regional Animal Cruelty Prosecu...   \n",
       "\n",
       "                                             contents  \\\n",
       "4   The U.S. Department of Justice, the U.S. Envir...   \n",
       "7   A 131-count criminal indictment was unsealed t...   \n",
       "19  The United States Attorney’s Office for the Mi...   \n",
       "\n",
       "                         date                 topics  \\\n",
       "4   2018-07-09T00:00:00-04:00          [Environment]   \n",
       "7   2014-12-17T00:00:00-05:00  [Consumer Protection]   \n",
       "19  2017-12-14T00:00:00-05:00          [Environment]   \n",
       "\n",
       "                                           components  \\\n",
       "4        [Environment and Natural Resources Division]   \n",
       "7                                    [Civil Division]   \n",
       "19  [Environment and Natural Resources Division, U...   \n",
       "\n",
       "                                          New_content  \n",
       "4   U.S. Department Justice U.S. Environmental Pro...  \n",
       "7   131 count criminal indictment unsealed today B...  \n",
       "19  United States Attorney Office Middle District ...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e9968d42-6e38-4f18-9dbe-fb12fd61f41b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The U.S. Department of Justice, the U.S. Environmental Protection Agency (EPA), and the Rhode Island Department of Environmental Management (RIDEM) an'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.contents[4][:150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "af928043-4233-4caa-8b8b-1f7b80b629ea",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'U.S. Department Justice U.S. Environmental Protection Agency EPA Rhode Island Department Environmental Management RIDEM announced today subsidiaries S'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.New_content[4][:150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "917b445f-0836-456b-85f5-27bce95c6c25",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''\n",
    "    - From a Given Text, Count the number of stop words in it.\n",
    "    - Print the percentage of stop word tokens compared to all tokens in a given text.\n",
    "'''\n",
    "text = '''\n",
    "Thor: Love and Thunder is a 2022 American superhero film based on Marvel Comics featuring the character Thor, produced by Marvel Studios and \n",
    "distributed by Walt Disney Studios Motion Pictures. It is the sequel to Thor: Ragnarok (2017) and the 29th film in the Marvel Cinematic Universe (MCU).\n",
    "The film is directed by Taika Waititi, who co-wrote the script with Jennifer Kaytin Robinson, and stars Chris Hemsworth as Thor alongside Christian Bale, Tessa Thompson,\n",
    "Jaimie Alexander, Waititi, Russell Crowe, and Natalie Portman. In the film, Thor attempts to find inner peace, but must return to action and recruit Valkyrie (Thompson),\n",
    "Korg (Waititi), and Jane Foster (Portman)—who is now the Mighty Thor—to stop Gorr the God Butcher (Bale) from eliminating all gods.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9d367c07-5d7f-4ec5-a70c-0431145cc777",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "767"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "abfb01e4-01d9-4419-b1ff-b39baf327b4b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of stop_words: 40 \n",
      "percentage of stop words compared to all the tokens: 25.0\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(text)\n",
    "\n",
    "count = 0\n",
    "Token = 0\n",
    "for token in doc:\n",
    "    if token.is_stop:\n",
    "        count +=1\n",
    "    Token +=1\n",
    "per = count/Token *100\n",
    "print(\"Number of stop_words:\",count,\"\\npercentage of stop words compared to all the tokens:\",per)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0b115cd2-8c30-477a-bc80-3ba6c953e57f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9e99aa24-437b-4a7c-a853-f985a2743618",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'  India men national cricket team known Team India Men Blue represents India men international cricket \\n governed Board Control Cricket India BCCI Member International Cricket Council ICC Test \\n Day International ODI Twenty20 International T20I status Cricket introduced India British sailors 18th century \\n cricket club established 1792 India national cricket team played Test match 25 June 1932 Lord sixth team \\n granted test cricket status \\n'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1 = ''' The India men's national cricket team, also known as Team India or the Men in Blue, represents India in men's international cricket.\n",
    "It is governed by the Board of Control for Cricket in India (BCCI), and is a Full Member of the International Cricket Council (ICC) with Test,\n",
    "One Day International (ODI) and Twenty20 International (T20I) status. Cricket was introduced to India by British sailors in the 18th century, and the \n",
    "first cricket club was established in 1792. India's national cricket team played its first Test match on 25 June 1932 at Lord's, becoming the sixth team to be\n",
    "granted test cricket status.\n",
    "'''\n",
    "\n",
    "doc1 = nlp(text1)\n",
    "doc1 = Preprocess(doc1)\n",
    "doc1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}