Cforum positive sentiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Message_lemmatized</th>\n",
       "      <th>sentiment_score</th>\n",
       "      <th>sentiment_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fast</td>\n",
       "      <td>0.200</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>applied june 2017 file transferred nairobi off...</td>\n",
       "      <td>0.067</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>believe rewuested passport something good spouse</td>\n",
       "      <td>0.700</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>yeah gckey already showing application approve...</td>\n",
       "      <td>0.200</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hey guysbri think belong groupbrbrapplication ...</td>\n",
       "      <td>0.062</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>change far brum still waiting medical req</td>\n",
       "      <td>0.050</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>yeah figured much congrats best</td>\n",
       "      <td>0.600</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>excellent news side canada guy maybe message c...</td>\n",
       "      <td>0.500</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>indeed excellent news dont mind telling part c...</td>\n",
       "      <td>1.000</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>hey guy excited right today asked medical rece...</td>\n",
       "      <td>0.220</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Message_lemmatized  sentiment_score  \\\n",
       "0                                               fast            0.200   \n",
       "1  applied june 2017 file transferred nairobi off...            0.067   \n",
       "2   believe rewuested passport something good spouse            0.700   \n",
       "3  yeah gckey already showing application approve...            0.200   \n",
       "4  hey guysbri think belong groupbrbrapplication ...            0.062   \n",
       "5          change far brum still waiting medical req            0.050   \n",
       "6                    yeah figured much congrats best            0.600   \n",
       "7  excellent news side canada guy maybe message c...            0.500   \n",
       "8  indeed excellent news dont mind telling part c...            1.000   \n",
       "9  hey guy excited right today asked medical rece...            0.220   \n",
       "\n",
       "  sentiment_category  \n",
       "0           positive  \n",
       "1           positive  \n",
       "2           positive  \n",
       "3           positive  \n",
       "4           positive  \n",
       "5           positive  \n",
       "6           positive  \n",
       "7           positive  \n",
       "8           positive  \n",
       "9           positive  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "filename = 'Cforum_positive.csv'\n",
    "df = pd.read_csv(filename)\n",
    "df[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>tf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dm</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>submitted</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>waiting</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>requested</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>office</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>document</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>went</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ecas</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>transferred</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>aboutbrdoes</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>update</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>nairobi</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>done</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>october</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mycic</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>medical</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>file</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>might</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>supporting</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>say</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>june</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>long</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>hear</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>process</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2017</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>pcc</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>anyone</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>email</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>know</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>19th</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>dataxflisttypeoli</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>date</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>brolli</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>receipt</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>voliolin</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>come</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>classsmilie</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>matter</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>gotten</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>3rd</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>mean</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>quebec</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>19</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>partner</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>commonlaw</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>27braor1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457</th>\n",
       "      <td>timelinebrbrapplication</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>458</th>\n",
       "      <td>17brbiometrics</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>459</th>\n",
       "      <td>sponsoring</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>kenyan</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>18brcsq</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>anything</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>4brsa</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>brbrhere</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>august</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>24brapplication</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>invitation</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>service</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>16brbiometrics</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>11brecas</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>471 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       words    tf\n",
       "0                         dm   4.0\n",
       "1                  submitted   5.0\n",
       "2                    waiting   4.0\n",
       "3                  requested   3.0\n",
       "4                     office   8.0\n",
       "5                   document   4.0\n",
       "6                       went   1.0\n",
       "7                       ecas   2.0\n",
       "8                transferred   2.0\n",
       "9                aboutbrdoes   1.0\n",
       "10                    update   4.0\n",
       "11                   nairobi  16.0\n",
       "12                      done   2.0\n",
       "13                   october   1.0\n",
       "14                     mycic   1.0\n",
       "15                   medical  11.0\n",
       "16                      file   8.0\n",
       "17                     might   1.0\n",
       "18                supporting   1.0\n",
       "19                       say   3.0\n",
       "20                      june   3.0\n",
       "21                      long   4.0\n",
       "22                      hear   1.0\n",
       "23                   process   7.0\n",
       "24                      2017   4.0\n",
       "25                       pcc   4.0\n",
       "26                    anyone   1.0\n",
       "27                     email   7.0\n",
       "28                      know   7.0\n",
       "29                      19th   1.0\n",
       "..                       ...   ...\n",
       "441        dataxflisttypeoli   1.0\n",
       "442                     date   1.0\n",
       "443                   brolli   1.0\n",
       "444                  receipt   1.0\n",
       "445                 voliolin   1.0\n",
       "446                     come   1.0\n",
       "447              classsmilie   1.0\n",
       "448                   matter   1.0\n",
       "449                   gotten   2.0\n",
       "450                      3rd   1.0\n",
       "451                     mean   1.0\n",
       "452                   quebec   1.0\n",
       "453                       19   1.0\n",
       "454                  partner   1.0\n",
       "455                commonlaw   1.0\n",
       "456                 27braor1   1.0\n",
       "457  timelinebrbrapplication   1.0\n",
       "458           17brbiometrics   1.0\n",
       "459               sponsoring   1.0\n",
       "460                   kenyan   1.0\n",
       "461                  18brcsq   1.0\n",
       "462                 anything   1.0\n",
       "463                    4brsa   1.0\n",
       "464                 brbrhere   1.0\n",
       "465                   august   1.0\n",
       "466          24brapplication   1.0\n",
       "467               invitation   1.0\n",
       "468                  service   1.0\n",
       "469           16brbiometrics   1.0\n",
       "470                 11brecas   1.0\n",
       "\n",
       "[471 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#term frequency - not very useful\n",
    "tf1 = (df['Message_lemmatized'][1:30]).apply(lambda x: pd.value_counts(x.split(\" \"))).sum(axis = 0).reset_index()\n",
    "tf1.columns = ['words','tf']\n",
    "tf1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>tf</th>\n",
       "      <th>idf</th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.112214</td>\n",
       "      <td>12.448855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>submitted</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.399896</td>\n",
       "      <td>16.999480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>waiting</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.903895</td>\n",
       "      <td>11.615581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>requested</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.396833</td>\n",
       "      <td>13.190500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>office</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.373307</td>\n",
       "      <td>18.986456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>document</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.473622</td>\n",
       "      <td>9.894487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>went</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.860358</td>\n",
       "      <td>3.860358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ecas</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.282750</td>\n",
       "      <td>8.565499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>transferred</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.236107</td>\n",
       "      <td>10.472214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>aboutbrdoes</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>update</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.194978</td>\n",
       "      <td>12.779911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>nairobi</td>\n",
       "      <td>16.0</td>\n",
       "      <td>6.733216</td>\n",
       "      <td>107.731454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>done</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.161924</td>\n",
       "      <td>6.323848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>october</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.440050</td>\n",
       "      <td>4.440050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mycic</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.209968</td>\n",
       "      <td>6.209968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>medical</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.863318</td>\n",
       "      <td>31.496496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>file</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.688521</td>\n",
       "      <td>21.508170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>might</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.205019</td>\n",
       "      <td>3.205019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>supporting</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.693620</td>\n",
       "      <td>4.693620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>say</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.490092</td>\n",
       "      <td>7.470276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>june</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.653602</td>\n",
       "      <td>10.960806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>long</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.300169</td>\n",
       "      <td>9.200678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>hear</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.101403</td>\n",
       "      <td>3.101403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>process</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.023123</td>\n",
       "      <td>14.161859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2017</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.312848</td>\n",
       "      <td>17.251391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>pcc</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.099085</td>\n",
       "      <td>20.396341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>anyone</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.881739</td>\n",
       "      <td>2.881739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>email</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.025760</td>\n",
       "      <td>21.180320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>know</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.778269</td>\n",
       "      <td>12.447886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>19th</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.138509</td>\n",
       "      <td>6.138509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>dataxflisttypeoli</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.714045</td>\n",
       "      <td>7.714045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>date</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.289830</td>\n",
       "      <td>2.289830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>brolli</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.672591</td>\n",
       "      <td>6.672591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>receipt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.657688</td>\n",
       "      <td>4.657688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>voliolin</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>come</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.400429</td>\n",
       "      <td>2.400429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>classsmilie</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.536014</td>\n",
       "      <td>2.536014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>matter</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.860358</td>\n",
       "      <td>3.860358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>gotten</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.516821</td>\n",
       "      <td>11.033641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>3rd</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.890684</td>\n",
       "      <td>4.890684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>mean</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.985920</td>\n",
       "      <td>2.985920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>quebec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.301798</td>\n",
       "      <td>4.301798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>19</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.621090</td>\n",
       "      <td>3.621090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>partner</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.465610</td>\n",
       "      <td>4.465610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>commonlaw</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.323754</td>\n",
       "      <td>5.323754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>27braor1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457</th>\n",
       "      <td>timelinebrbrapplication</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.812657</td>\n",
       "      <td>8.812657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>458</th>\n",
       "      <td>17brbiometrics</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>459</th>\n",
       "      <td>sponsoring</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.117547</td>\n",
       "      <td>5.117547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>kenyan</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.812657</td>\n",
       "      <td>8.812657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>18brcsq</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>anything</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.464360</td>\n",
       "      <td>3.464360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>4brsa</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.505805</td>\n",
       "      <td>9.505805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>brbrhere</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.386767</td>\n",
       "      <td>5.386767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>august</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.085270</td>\n",
       "      <td>4.085270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>24brapplication</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>invitation</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.756534</td>\n",
       "      <td>4.756534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>service</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.628069</td>\n",
       "      <td>3.628069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>16brbiometrics</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.198952</td>\n",
       "      <td>10.198952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>11brecas</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.505805</td>\n",
       "      <td>9.505805</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>471 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       words    tf        idf       tfidf\n",
       "0                         dm   4.0   3.112214   12.448855\n",
       "1                  submitted   5.0   3.399896   16.999480\n",
       "2                    waiting   4.0   2.903895   11.615581\n",
       "3                  requested   3.0   4.396833   13.190500\n",
       "4                     office   8.0   2.373307   18.986456\n",
       "5                   document   4.0   2.473622    9.894487\n",
       "6                       went   1.0   3.860358    3.860358\n",
       "7                       ecas   2.0   4.282750    8.565499\n",
       "8                transferred   2.0   5.236107   10.472214\n",
       "9                aboutbrdoes   1.0  10.198952   10.198952\n",
       "10                    update   4.0   3.194978   12.779911\n",
       "11                   nairobi  16.0   6.733216  107.731454\n",
       "12                      done   2.0   3.161924    6.323848\n",
       "13                   october   1.0   4.440050    4.440050\n",
       "14                     mycic   1.0   6.209968    6.209968\n",
       "15                   medical  11.0   2.863318   31.496496\n",
       "16                      file   8.0   2.688521   21.508170\n",
       "17                     might   1.0   3.205019    3.205019\n",
       "18                supporting   1.0   4.693620    4.693620\n",
       "19                       say   3.0   2.490092    7.470276\n",
       "20                      june   3.0   3.653602   10.960806\n",
       "21                      long   4.0   2.300169    9.200678\n",
       "22                      hear   1.0   3.101403    3.101403\n",
       "23                   process   7.0   2.023123   14.161859\n",
       "24                      2017   4.0   4.312848   17.251391\n",
       "25                       pcc   4.0   5.099085   20.396341\n",
       "26                    anyone   1.0   2.881739    2.881739\n",
       "27                     email   7.0   3.025760   21.180320\n",
       "28                      know   7.0   1.778269   12.447886\n",
       "29                      19th   1.0   6.138509    6.138509\n",
       "..                       ...   ...        ...         ...\n",
       "441        dataxflisttypeoli   1.0   7.714045    7.714045\n",
       "442                     date   1.0   2.289830    2.289830\n",
       "443                   brolli   1.0   6.672591    6.672591\n",
       "444                  receipt   1.0   4.657688    4.657688\n",
       "445                 voliolin   1.0  10.198952   10.198952\n",
       "446                     come   1.0   2.400429    2.400429\n",
       "447              classsmilie   1.0   2.536014    2.536014\n",
       "448                   matter   1.0   3.860358    3.860358\n",
       "449                   gotten   2.0   5.516821   11.033641\n",
       "450                      3rd   1.0   4.890684    4.890684\n",
       "451                     mean   1.0   2.985920    2.985920\n",
       "452                   quebec   1.0   4.301798    4.301798\n",
       "453                       19   1.0   3.621090    3.621090\n",
       "454                  partner   1.0   4.465610    4.465610\n",
       "455                commonlaw   1.0   5.323754    5.323754\n",
       "456                 27braor1   1.0  10.198952   10.198952\n",
       "457  timelinebrbrapplication   1.0   8.812657    8.812657\n",
       "458           17brbiometrics   1.0  10.198952   10.198952\n",
       "459               sponsoring   1.0   5.117547    5.117547\n",
       "460                   kenyan   1.0   8.812657    8.812657\n",
       "461                  18brcsq   1.0  10.198952   10.198952\n",
       "462                 anything   1.0   3.464360    3.464360\n",
       "463                    4brsa   1.0   9.505805    9.505805\n",
       "464                 brbrhere   1.0   5.386767    5.386767\n",
       "465                   august   1.0   4.085270    4.085270\n",
       "466          24brapplication   1.0  10.198952   10.198952\n",
       "467               invitation   1.0   4.756534    4.756534\n",
       "468                  service   1.0   3.628069    3.628069\n",
       "469           16brbiometrics   1.0  10.198952   10.198952\n",
       "470                 11brecas   1.0   9.505805    9.505805\n",
       "\n",
       "[471 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculating term weighting\n",
    "import numpy as np\n",
    "for i,word in enumerate(tf1['words']):\n",
    "    tf1.loc[i, 'idf'] = np.log(df.shape[0]/(len(df[df['Message_lemmatized'].str.contains(word)])))\n",
    "tf1['tfidf'] = tf1['tf'] * tf1['idf']\n",
    "tf1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>tf</th>\n",
       "      <th>idf</th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dm</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.112214</td>\n",
       "      <td>12.448855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>submitted</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.399896</td>\n",
       "      <td>16.999480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>waiting</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.903895</td>\n",
       "      <td>11.615581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>requested</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.396833</td>\n",
       "      <td>13.190500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>office</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.373307</td>\n",
       "      <td>18.986456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       words   tf       idf      tfidf\n",
       "0         dm  4.0  3.112214  12.448855\n",
       "1  submitted  5.0  3.399896  16.999480\n",
       "2    waiting  4.0  2.903895  11.615581\n",
       "3  requested  3.0  4.396833  13.190500\n",
       "4     office  8.0  2.373307  18.986456"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#converting tf1 into a dataframe\n",
    "Ps = pd.DataFrame(tf1)\n",
    "Ps.head()\n",
    "#import csv\n",
    "#with open(\"Cforum_sentiments\", \"w\") as _file:\n",
    " #   writer = csv.writer(_file)\n",
    "  #  writer.writerow(dfs.columns.values)\n",
    "   # for index, row in dfs.iterrows():\n",
    "    #    writer.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['äô', 'dont', 'date', 'dec', 'december', 'decide', 'decision', 'dependent', 'detail', 'discus', 'dispatched', 'dm', 'document', 'done', 'due', 'change', 'e000', 'ecas', 'eligibility', 'email', 'emaillili', 'end', 'entry', 'ethiopia', 'ethiopiabrvisa', 'every', 'everything', 'exact', 'dataxflisttypeolthe', 'dataxflisttypeoli', 'dataxflisttypeolcommunicated', 'dataxflisttypeol05', 'changesbrbackground', 'changesbrbiometrics', 'check', 'checkbrfinal', 'checked', 'cic', 'class', 'classsmilie', 'classusername', 'click', 'come', 'commonlaw', 'confirmation', 'congrats', 'continue', 'conveniencebrbrbrapplication', 'copied', 'country', 'cvacbrbrthanks', 'cvacbrmedical', 'daniel', 'datashortname', 'datauserid715646', 'datausernamemtunguyas', 'dataxfinitmembertooltip', 'exam', 'excellent', 'excited', 'given', 'going', 'good', 'got', 'gotten', 'group', 'groupbrbrapplication', 'guy', 'guysbri', 'hail', 'handy', 'happy', 'havent', 'hear', 'hello', 'help', 'helpbr', 'helpfully', 'herebrbrmy', 'herr', 'hey', 'hi', 'hopefully', 'hrefhttpswwwcanadavisacomcanadaimmigrationdiscussionboardmembers715646mtunguyasa', 'icvaci', 'id', 'go', 'give', 'exited', 'getting', 'expand', 'experience', 'explanation', 'express', 'extending', 'family', 'far', 'fastim', 'feb', 'federal', 'fifth', 'figured', 'file', 'fill', 'final', 'find', 'fingerprint', 'followed', 'forever', 'form', 'forum', 'friend', 'friendsbrbri', 'gckey', 'get', 'changed', 'certificatebr4', 'img', '5669', '23', '24brapplication', '25', '26', '27braor1', '2nd', '3018', '312018lili', '3rd', '442018', '4brsa', '52018', '6months', 'center', 'able', 'aboutbrdoes', 'account', 'activate', 'addis', 'additional', 'admission', 'adr', 'advance', 'advantagedisadvantage', 'allah', 'almostly', '2232018brbrnow', '22', '21', '2018brpassport', '06', '07', '072017brsch', '11', '11brecas', '13', '152017brfile', '16', '16brbiometrics', '17brbiometrics', '18brcsq', '19', '19th', '2017', '2017bbbbbbbrbras', '2017braor', '2017brb2', '2017brdm', '2017brmedical', '2017brppr', '2017brreceived', '2017brsch', '2017brsponsor', '2018', '2018brlanded', 'already', 'also', 'alt', 'br', 'brb3', 'brbrapplication', 'brbrhere', 'brbri', 'brbrthank', 'brbrthis', 'brbrwas', 'brcountry', 'bri', 'brin', 'brinterview', 'brnext', 'bro', 'brolli', 'brreview', 'brum', 'bryesterday', 'bschedule', 'bsep', 'btw', 'busy', 'called', 'canada', 'cant', 'case', 'brand', 'big', 'alxamdulilah', 'bforwarded', 'another', 'answered', 'anyone', 'anything', 'anyways', 'applicant', 'application', 'applicationprofile', 'applied', 'applying', 'appreciate', 'approval', 'approved', 'asked', 'attach', 'august', 'automated', 'back', 'background', 'backgrounddeclaration', 'based', 'bc', 'believe', 'belong', 'best', 'ie', 'imm', 'äòstarted', 'started', 'someone', 'something', 'sorry', 'south', 'specific', 'sponsored', 'sponsoring', 'sponsorship', 'sponsorshipbrcountry', 'spouse', 'srcdataimagegifbase64r0lgodlhaqabaiaaaaaaapyh5baeaaaaalaaaaaabaaeaaaibraa7', 'start', 'status', 'requirementsbrreview', 'still', 'stream', 'submitted', 'sudan', 'supporting', 'sure', 'tail', 'take', 'taken', 'telling', 'thank', 'thanks', 'soft', 'smiliesprite1', 'smiliesprite', 'skilled', 'result', 'return', 'reviewing', 'rewuested', 'right', 'samebrbrhow', 'saw', 'say', 'saying', 'schedule', 'scholarship', 'section', 'seems', 'seen', 'send', 'sent', 'sep', 'september', 'service', 'share', 'shortly', 'showed', 'showing', 'side', 'since', 'thats', 'think', 'thread', 'value', 'vfs', 'via', 'visa', 'vo', 'voliolin', 'waiting', 'want', 'wanted', 'web', 'wellbrbrsorry', 'went', 'whatapp', 'whether', 'whole', 'wife', 'willing', 'wondering', 'worker', 'would', 'wow', 'wrote', 'yeah', 'yet', 'youbrbrgood', 'youbrreview', 'vancouver', 'vai', 'till', 'usually', 'time', 'timeline', 'timelinebrbrapplication', 'titlesmile', 'today', 'took', 'tracking', 'transfer', 'transfered', 'transferred', 'trasfered', 'try', 'trying', 'two', 'type', 'uci', 'unable', 'understand', 'university', 'update', 'upfront', 'uploaded', 'ur', 'use', 'using', 'responding', 'requested', 'immigration', 'medical', 'long', 'luck', 'madebr', 'make', 'manage', 'maneuver', 'march', 'masd', 'matter', 'may', 'maybe', 'mean', 'meet', 'request', 'mentioned', 'message', 'might', 'mind', 'mine', 'month', 'much', 'mycic', 'nairobi', 'necessary', 'need', 'new', 'logged', 'living', 'link', 'like', 'indeed', 'information', 'insha', 'instruction', 'interview', 'invitation', 'island', 'issue', 'jan', 'january', 'januarybrthat', 'join', 'july', 'june', 'kenyabrbrapplication', 'kenyabrbrb1', 'kenyan', 'kind', 'know', 'landed', 'landing', 'late', 'learnt', 'let', 'letter', 'news', 'nov', 'nrobi', 'number', 'primary', 'proceed', 'process', 'processed', 'processingb', 'progress', 'properly', 'provide', 'provided', 'quebec', 'question', 'quick', 'quickly', 'rather', 'ready', 'really', 'recall', 'receipt', 'received', 'red', 'related', 'reply', 'replyingbrour', 'replyso', 'req', 'prearrival', 'pr', 'posted', 'post', 'numberbrgckey', 'october', 'office', 'ok', 'one', 'onwards', 'open', 'opinion', 'option', 'organization', 'part', 'partner', 'pass', 'passed', 'passport', 'pasted', 'patientbrbrwhat', 'pay', 'pcc', 'people', 'pick', 'point', 'police', 'possible', '052018brbrif', 'processing']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "keywords = Ps['words']\n",
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(keywords)\n",
    "indices = np.argsort(vectorizer.idf_)[::-1]\n",
    "features = vectorizer.get_feature_names()\n",
    "top_n = 700\n",
    "top_featuresp = [features[i] for i in indices[:top_n]]\n",
    "print (top_featuresp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bforwarded', '2017brdm', 'numberbrgckey', 'vfs', 'replyso', 'classusername', 'gckey', 'almostly', '16brbiometrics', '2017brsponsor', 'uploaded', '2018brpassport', 'e000', 'mycic', '27braor1', 'samebrbrhow', '2232018brbrnow', 'brbri', '2017brmedical', 'voliolin', 'friendsbrbri', 'brbrthis', 'changesbrbackground', 'bryesterday', 'changesbrbiometrics', 'äòstarted', '4brsa', '152017brfile', 'trasfered', '6months', 'brcountry', 'helpbr', 'brbrwas', 'aboutbrdoes', 'prearrival', '052018brbrif', 'timeline', 'wellbrbrsorry', '2017brppr', 'processingb', 'brnext', 'requirementsbrreview', '2018brlanded', 'kenyabrbrb1', 'äô', '3rd', 'conveniencebrbrbrapplication', 'alxamdulilah', 'timelinebrbrapplication', '11brecas', 'datauserid715646', 'imm', '18brcsq', 'sponsorshipbrcountry', 'brbrhere', 'madebr', 'brolli', 'herebrbrmy', '2017brreceived', '17brbiometrics', 'patientbrbrwhat', 'januarybrthat', 'whatapp', 'dataxflisttypeolthe', 'groupbrbrapplication', 'bsep', 'titlesmile', 'smiliesprite1', 'youbrbrgood', 'cvacbrbrthanks', 'replyingbrour', 'guysbri', 'certificatebr4', 'classsmilie', 'brb3', 'brbrapplication', 'insha', '2nd', 'cvacbrmedical', 'datashortname', 'dataxfinitmembertooltip', '2017brsch', '19th', 'fastim', 'masd', 'nrobi', '2017brb2', 'smiliesprite', 'icvaci', 'brbrthank', '312018lili', 'applicationprofile', 'commonlaw', 'dataxflisttypeolcommunicated', 'brinterview', 'ethiopiabrvisa', '072017brsch', 'ecas', 'brreview', '2017braor', 'kenyabrbrapplication', '24brapplication', 'req', 'checkbrfinal', 'bschedule', 'rewuested', 'datausernamemtunguyas', 'dataxflisttypeol05', '2017bbbbbbbrbras', 'backgrounddeclaration', 'maneuver', 'youbrreview', 'dataxflisttypeoli', 'advantagedisadvantage', 'emaillili'}\n"
     ]
    }
   ],
   "source": [
    "from spellchecker import SpellChecker\n",
    "spell = SpellChecker() \n",
    "  \n",
    "# find those words that may be misspelled \n",
    "misspelled = spell.unknown(top_featuresp) \n",
    "print(misspelled) \n",
    "\n",
    "#for word in misspelled: \n",
    "    #Get the one `most likely` answer \n",
    "    #print(spell.correction(word)) \n",
    "  \n",
    "    # Get a list of `likely` options \n",
    "    #print(spell.candidates(word)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['dont', 'date', 'dec', 'december', 'decide', 'decision', 'dependent', 'detail', 'discus', 'dispatched', 'dm', 'document', 'done', 'due', 'change', 'eligibility', 'email', 'end', 'entry', 'ethiopia', 'every', 'everything', 'exact', 'check', 'checked', 'cic', 'class', 'click', 'come', 'confirmation', 'congrats', 'continue', 'copied', 'country', 'daniel', 'exam', 'excellent', 'excited', 'given', 'going', 'good', 'got', 'gotten', 'group', 'guy', 'hail', 'handy', 'happy', 'havent', 'hear', 'hello', 'help', 'helpfully', 'herr', 'hey', 'hi', 'hopefully', 'hrefhttpswwwcanadavisacomcanadaimmigrationdiscussionboardmembers715646mtunguyasa', 'id', 'go', 'give', 'exited', 'getting', 'expand', 'experience', 'explanation', 'express', 'extending', 'family', 'far', 'feb', 'federal', 'fifth', 'figured', 'file', 'fill', 'final', 'find', 'fingerprint', 'followed', 'forever', 'form', 'forum', 'friend', 'get', 'changed', 'img', '5669', '23', '25', '26', '3018', '442018', '52018', 'center', 'able', 'account', 'activate', 'addis', 'additional', 'admission', 'adr', 'advance', 'allah', '22', '21', '06', '07', '11', '13', '16', '19', '2017', '2018', 'already', 'also', 'alt', 'br', 'bri', 'brin', 'bro', 'brum', 'btw', 'busy', 'called', 'canada', 'cant', 'case', 'brand', 'big', 'another', 'answered', 'anyone', 'anything', 'anyways', 'applicant', 'application', 'applied', 'applying', 'appreciate', 'approval', 'approved', 'asked', 'attach', 'august', 'automated', 'back', 'background', 'based', 'bc', 'believe', 'belong', 'best', 'ie', 'started', 'someone', 'something', 'sorry', 'south', 'specific', 'sponsored', 'sponsoring', 'sponsorship', 'spouse', 'srcdataimagegifbase64r0lgodlhaqabaiaaaaaaapyh5baeaaaaalaaaaaabaaeaaaibraa7', 'start', 'status', 'still', 'stream', 'submitted', 'sudan', 'supporting', 'sure', 'tail', 'take', 'taken', 'telling', 'thank', 'thanks', 'soft', 'skilled', 'result', 'return', 'reviewing', 'right', 'saw', 'say', 'saying', 'schedule', 'scholarship', 'section', 'seems', 'seen', 'send', 'sent', 'sep', 'september', 'service', 'share', 'shortly', 'showed', 'showing', 'side', 'since', 'thats', 'think', 'thread', 'value', 'via', 'visa', 'vo', 'waiting', 'want', 'wanted', 'web', 'went', 'whether', 'whole', 'wife', 'willing', 'wondering', 'worker', 'would', 'wow', 'wrote', 'yeah', 'yet', 'vancouver', 'vai', 'till', 'usually', 'time', 'today', 'took', 'tracking', 'transfer', 'transfered', 'transferred', 'try', 'trying', 'two', 'type', 'uci', 'unable', 'understand', 'university', 'update', 'upfront', 'ur', 'use', 'using', 'responding', 'requested', 'immigration', 'medical', 'long', 'luck', 'make', 'manage', 'march', 'matter', 'may', 'maybe', 'mean', 'meet', 'request', 'mentioned', 'message', 'might', 'mind', 'mine', 'month', 'much', 'nairobi', 'necessary', 'need', 'new', 'logged', 'living', 'link', 'like', 'indeed', 'information', 'instruction', 'interview', 'invitation', 'island', 'issue', 'jan', 'january', 'join', 'july', 'june', 'kenyan', 'kind', 'know', 'landed', 'landing', 'late', 'learnt', 'let', 'letter', 'news', 'nov', 'number', 'primary', 'proceed', 'process', 'processed', 'progress', 'properly', 'provide', 'provided', 'quebec', 'question', 'quick', 'quickly', 'rather', 'ready', 'really', 'recall', 'receipt', 'received', 'red', 'related', 'reply', 'pr', 'posted', 'post', 'october', 'office', 'ok', 'one', 'onwards', 'open', 'opinion', 'option', 'organization', 'part', 'partner', 'pass', 'passed', 'passport', 'pasted', 'pay', 'pcc', 'people', 'pick', 'point', 'police', 'possible', 'processing']\n"
     ]
    }
   ],
   "source": [
    "#removing misspelled words\n",
    "\n",
    "words = top_features\n",
    "MS = misspelled\n",
    "for word in list(top_featuresp):  # iterating on a copy since removing will mess things up\n",
    "    if word in MS:\n",
    "        top_featuresp.remove(word)\n",
    "print(top_featuresp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}