code/pipeline/WNTRAC-trainer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Copyright 2020 IBM Corporation\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "# http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy import random\n",
    "import nltk\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, f1_score\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "import pickle\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Point to the training file and model output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = \"data/ground_truth.csv\"\n",
    "base_out_model_path='models'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The ground truth should contain two columns, sentenceText (containg the sentence) and Label (groundtruth) for this sentence classification task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len =  1415\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>sentenceText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>travelrestrictions</td>\n",
       "      <td>Russia halts all international air traffic wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>serviceorplaceclosed</td>\n",
       "      <td>Moscow closes all restaurants, bars, parks, an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>economy</td>\n",
       "      <td>Sweden's central bank, Sveriges Riksbank, exte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unk</td>\n",
       "      <td>UK Prime Minister Boris Johnson announces that...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serviceorplaceclosed</td>\n",
       "      <td>The 2020 Open Championship, was originally sch...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Label                                       sentenceText\n",
       "0    travelrestrictions  Russia halts all international air traffic wit...\n",
       "1  serviceorplaceclosed  Moscow closes all restaurants, bars, parks, an...\n",
       "2               economy  Sweden's central bank, Sveriges Riksbank, exte...\n",
       "3                   unk  UK Prime Minister Boris Johnson announces that...\n",
       "4  serviceorplaceclosed  The 2020 Open Championship, was originally sch..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(training_data, encoding = \"utf-8\", usecols=['Label', 'sentenceText'])\n",
    "print (\"len = \", len(df))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label distribution is shown below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unk                     516\n",
       "serviceorplaceclosed    270\n",
       "travelrestrictions      215\n",
       "misc                    162\n",
       "confinement             132\n",
       "economy                  53\n",
       "stateofemergency         45\n",
       "gatheringrestriction     13\n",
       "tracing                   9\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')\n",
    "STOPWORDS = set(stopwords.words('english'))\n",
    "\n",
    "def clean_text(text):\n",
    "    text = text.lower() # lowercase text\n",
    "    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text\n",
    "    text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text\n",
    "    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # delete stopwors from text\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doing 10 fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_fold=KFold(n_splits=10)\n",
    "\n",
    "def cross_validate(model):\n",
    "    all_y_test, all_y_pred=[],[]\n",
    "    for train_index, test_index in k_fold.split(df):\n",
    "        X_train, X_test = df.iloc[train_index]['sentenceText'].values, df.iloc[test_index]['sentenceText'].values\n",
    "        y_train, y_test = df.iloc[train_index]['Label'].values, df.iloc[test_index]['Label'].values\n",
    "        model.fit(X_train, y_train)\n",
    "        prediction = model.predict(X_test)\n",
    "        all_y_test.extend(y_test)\n",
    "        all_y_pred.extend(prediction)\n",
    "    return all_y_test, all_y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit + Save model on 100% of train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_save_model(model, out_path):\n",
    "    print (model)\n",
    "    X_train, y_train = df.sentenceText, df.Label\n",
    "    model.fit(X_train, y_train)\n",
    "    pickle.dump(model, open(out_path, 'wb'))\n",
    "    print ('Saved model in ', out_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfidf', TfidfTransformer()),\n",
    "                ('clf', SGDClassifier(loss='hinge', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None)),\n",
    "               ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.665017667844523\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         confinement       0.73      0.64      0.68       132\n",
      "             economy       0.80      0.68      0.73        53\n",
      "gatheringrestriction       1.00      0.08      0.14        13\n",
      "                misc       0.33      0.08      0.13       162\n",
      "serviceorplaceclosed       0.77      0.76      0.76       270\n",
      "    stateofemergency       0.59      0.44      0.51        45\n",
      "             tracing       1.00      0.33      0.50         9\n",
      "  travelrestrictions       0.72      0.80      0.76       215\n",
      "                 unk       0.61      0.79      0.68       516\n",
      "\n",
      "           micro avg       0.67      0.67      0.67      1415\n",
      "           macro avg       0.73      0.51      0.54      1415\n",
      "        weighted avg       0.65      0.67      0.64      1415\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y, yhat = cross_validate(sgd)\n",
    "tags = list(sorted(df['Label'].unique()))\n",
    "print('accuracy %s' % accuracy_score(yhat, y))\n",
    "print(classification_report(y, yhat,target_names=tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
      "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
      "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
      "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
      "        strip...dom_state=42, shuffle=True, tol=None,\n",
      "       validation_fraction=0.1, verbose=0, warm_start=False))])\n",
      "Saved model in  models/svm.sav\n"
     ]
    }
   ],
   "source": [
    "saved_model = train_save_model(sgd, os.path.join(base_out_model_path, 'svm.sav'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving SGD Estimator for confidence estimation purposes (to be used during apply)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
      "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
      "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
      "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
      "        strip...dom_state=42, shuffle=True, tol=None,\n",
      "       validation_fraction=0.1, verbose=0, warm_start=False))])\n",
      "Saved model in  models/svm_estimator.sav\n"
     ]
    }
   ],
   "source": [
    "sgd_mod = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfidf', TfidfTransformer()),\n",
    "                ('clf', SGDClassifier(loss='modified_huber', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None)),\n",
    "               ])\n",
    "saved_model = train_save_model(sgd_mod, os.path.join(base_out_model_path, 'svm_estimator.sav'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "logreg = Pipeline([('vect', CountVectorizer()),\n",
    "                ('tfidf', TfidfTransformer()),\n",
    "                ('clf', LogisticRegression(n_jobs=1, C=1e5)),\n",
    "               ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.6530035335689046\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         confinement       0.73      0.68      0.70       132\n",
      "             economy       0.84      0.68      0.75        53\n",
      "gatheringrestriction       1.00      0.08      0.14        13\n",
      "                misc       0.29      0.25      0.27       162\n",
      "serviceorplaceclosed       0.78      0.76      0.77       270\n",
      "    stateofemergency       0.68      0.42      0.52        45\n",
      "             tracing       1.00      0.33      0.50         9\n",
      "  travelrestrictions       0.71      0.76      0.74       215\n",
      "                 unk       0.63      0.71      0.66       516\n",
      "\n",
      "           micro avg       0.65      0.65      0.65      1415\n",
      "           macro avg       0.74      0.52      0.56      1415\n",
      "        weighted avg       0.65      0.65      0.65      1415\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y, yhat = cross_validate(logreg)\n",
    "print('accuracy %s' % accuracy_score(y, yhat))\n",
    "print(classification_report(y, yhat,target_names=tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
      "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
      "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
      "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
      "        strip...penalty='l2', random_state=None,\n",
      "          solver='warn', tol=0.0001, verbose=0, warm_start=False))])\n",
      "Saved model in  models/lr.sav\n"
     ]
    }
   ],
   "source": [
    "saved_model = train_save_model(logreg, os.path.join(base_out_model_path, 'lr.sav'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from simpletransformers.classification import ClassificationModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Various model hyperparameters. Set use_cuda=True when cuda is unavailable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = False\n",
    "model_arch= 'bert'\n",
    "model_type = 'bert-base-uncased'\n",
    "output_dir = os.path.join(base_out_model_path+'bert_model')\n",
    "overwrite_output_dir = True\n",
    "num_train_epochs = 3\n",
    "train_size_ratio=0.8\n",
    "early_stopping_patience=3\n",
    "reprocess_input_data=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preparing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['labels'] = df.apply(lambda row: tags.index(str(row['Label'])), axis=1)\n",
    "df = df.rename(columns={'sentenceText':'text'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Dev Splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = (int) (train_size_ratio*len(df))\n",
    "train_df = df[0:train_size]\n",
    "eval_df = df[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ClassificationModel(model_arch, model_type, num_labels=len(tags), use_cuda = use_cuda, args={'output_dir':output_dir,'reprocess_input_data': reprocess_input_data, 'overwrite_output_dir': overwrite_output_dir, 'early_stopping_patience': early_stopping_patience, 'num_train_epochs':num_train_epochs, 'learning_rate': 5e-5, 'evaluate_during_training': True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc1b2bc55c1146d28a99feb07d40363e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1132.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceda9df9bc184be3a47b5701e8571810",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "230452a6fae6406ba4bcb02d3bc19acb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Current iteration', max=142.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 2.006536\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57c053b274df4662b3c83dc2263b059f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Current iteration', max=142.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.234115\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2bb1be24c114f04a047a16932642577",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Current iteration', max=142.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.032582\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.train_model(train_df, eval_df=eval_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b08562ca46eb49e7b63c363747471e4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=283.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92a14ce9ee1c43989807acde52b63e8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=36.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "result, model_outputs, wrong_predictions = model.eval_model(eval_df, acc=classification_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.57      0.63        21\n",
      "           1       1.00      0.67      0.80         6\n",
      "           2       1.00      1.00      1.00         2\n",
      "           3       0.80      0.26      0.39        46\n",
      "           4       0.91      0.93      0.92        69\n",
      "           5       1.00      1.00      1.00         1\n",
      "           7       0.89      0.87      0.88        54\n",
      "           8       0.63      0.90      0.74        84\n",
      "\n",
      "   micro avg       0.77      0.77      0.77       283\n",
      "   macro avg       0.87      0.78      0.80       283\n",
      "weighted avg       0.79      0.77      0.75       283\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(result['acc'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}