bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b821cb92",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import io\n",
    "import time\n",
    "import gc\n",
    "import random\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils import data\n",
    "from torch.nn import functional as F\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "b212e219",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is GPU used? (0=yes, -1=no): -1\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "current_cuda_device = -1\n",
    "if torch.cuda.is_available():\n",
    "    current_cuda_device = torch.cuda.current_device()\n",
    "print(f'Is GPU used? (0=yes, -1=no): {current_cuda_device}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ec1a48b",
   "metadata": {},
   "outputs": [],
   "source": [
    "CRAWL_EMBEDDING_PATH = '/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/crawl-300d-2M.vec'\n",
    "GLOVE_EMBEDDING_PATH = '/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/glove.840B.300d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aad6cf5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(data):\n",
    "    '''\n",
    "    Credit goes to https://www.kaggle.com/gpreda/jigsaw-fast-compact-solution\n",
    "    '''\n",
    "    punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~`\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n",
    "    def clean_special_chars(text, punct):\n",
    "        for p in punct:\n",
    "            text = text.replace(p, ' ')\n",
    "        return text\n",
    "\n",
    "    data = data.astype(str).apply(lambda x: clean_special_chars(x, punct))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1233892",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "train = pd.read_csv('/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/train.csv')\n",
    "test = pd.read_csv('/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/test.csv')\n",
    "test_private = pd.read_csv('/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/test_private_expanded.csv')\n",
    "test_public = pd.read_csv('/home/klazaridou/projects/jigsaw-unintended-bias-in-toxicity-classification/test_public_expanded.csv')\n",
    "# id,target,comment_text,severe_toxicity,obscene,identity_attack,insult,threat,asian,atheist,bisexual,black,buddhist,christian,female,heterosexual,hindu,homosexual_gay_or_lesbian,intellectual_or_learning_disability,jewish,latino,male,muslim,other_disability,other_gender,other_race_or_ethnicity,other_religion,other_sexual_orientation,physical_disability,psychiatric_or_mental_illness,transgender,white,created_date,publication_id,parent_id,article_id,rating,funny,wow,sad,likes,disagree,sexual_explicit,identity_annotator_count,toxicity_annotator_count\n",
    "print(f'Train and test shapes: {train.shape}, {test.shape}')\n",
    "print(f'Test private and test public shapes: {test_private.shape}, {test_public.shape}')  # all features and binarized toxicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18c40f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess text\n",
    "x_train = preprocess(train['comment_text'])\n",
    "x_test = preprocess(test['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81a06106",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get targets\n",
    "y_train = np.where(train['target'] >= 0.5, 1, 0)\n",
    "y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]\n",
    "print(f'y_train: {y_train}')\n",
    "print(f'y_aux_train: {y_aux_train}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a71b0569",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Negative examples: {np.histogram(y_train)[0][0]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b47a079",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Positive examples: {np.histogram(y_train)[0][9]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d45cf42",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get fewer data for efficiency\n",
    "# print(f'Training data example row: {train.iloc[[2]]}')\n",
    "train_small = train.sample(n=100000, weights='target')\n",
    "targets = train_small['target']\n",
    "print(f'Training small hist: {targets.hist(bins=2)}')\n",
    "print(f'Small training data shape: {targets.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44476e9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess text small dataset\n",
    "x_train = preprocess(train_small['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a172b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get targets for small dataset\n",
    "y_train = np.where(train_small['target'] >= 0.5, 1, 0)\n",
    "y_aux_train = train_small[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]\n",
    "print(f'y_train: {y_train}')\n",
    "print(f'y_aux_train: {y_aux_train}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2693b81c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Negative examples: {np.histogram(y_train)[0][0]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95255d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Positive examples: {np.histogram(y_train)[0][9]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14479dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize and vectorize text\n",
    "from keras.preprocessing import text, sequence # works with tensorflow>=2.7\n",
    "tokenizer = text.Tokenizer()\n",
    "tokenizer.fit_on_texts(list(x_train) + list(x_test))  # fit both vocabularies\n",
    "x_train = tokenizer.texts_to_sequences(x_train)  # translate into integers\n",
    "x_test = tokenizer.texts_to_sequences(x_test)\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)  # pad for balanced text length\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9231b3da",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary = None\n",
    "vocabulary = vocabulary or len(tokenizer.word_index) + 1\n",
    "print(f'words in vocabulary: {vocabulary}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0101f8c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# build embedding matrix\n",
    "from tqdm.notebook import tqdm_notebook as tqdm\n",
    "def get_coefs(word, *arr):\n",
    "    return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "def load_embeddings(path):\n",
    "    with open(path) as f:\n",
    "        return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))\n",
    "def build_matrix(word_index, path):\n",
    "    embedding_index = load_embeddings(path)\n",
    "    embedding_matrix = np.zeros((len(word_index) + 1, 300))\n",
    "    unknown_words = []\n",
    "    for word, i in word_index.items():\n",
    "        try:\n",
    "            embedding_matrix[i] = embedding_index[word]\n",
    "        except KeyError:\n",
    "            unknown_words.append(word)\n",
    "    return embedding_matrix, unknown_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00b36bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71465a00",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('unknown words (crawl): ', len(unknown_words_crawl))\n",
    "print(f'crawl_matrix: {crawl_matrix.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19c3839c",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    print(f'i: {word}: embedding len: {len(crawl_matrix[i])} ')\n",
    "    counter += 1\n",
    "    if counter == 1:\n",
    "        break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff4a975a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd87b4b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('unknown words (glove): ', len(unknown_words_glove))\n",
    "print(f'glove_matrix: {glove_matrix.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56f9b6cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    print(f'i: {word}: embedding len: {len(glove_matrix[i])} ')\n",
    "    counter += 1\n",
    "    if counter == 1:\n",
    "        break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1dd5d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)  \n",
    "# TODO: pad not common words because length is 300\n",
    "print(f'concatanated matrix: {embedding_matrix.shape}')\n",
    "del crawl_matrix\n",
    "del glove_matrix\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d94bacc",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    print(f'i: {word}: embedding len: {len(embedding_matrix[i])} ')\n",
    "    counter += 1\n",
    "    if counter == 1:\n",
    "        break;\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6d80d0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_MODELS = 1\n",
    "LSTM_UNITS = 2\n",
    "DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS\n",
    "MAX_LEN = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c07118ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "class SpatialDropout(nn.Dropout2d):\n",
    "    def forward(self, x):\n",
    "        x = x.unsqueeze(2)    # (N, T, 1, K)\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, K, 1, T)\n",
    "        x = super(SpatialDropout, self).forward(x)  # (N, K, 1, T), some features are masked\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, T, 1, K)\n",
    "        x = x.squeeze(2)  # (N, T, K)\n",
    "        return x\n",
    "    \n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, embedding_matrix, num_aux_targets):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        embed_size = embedding_matrix.shape[1]\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocabulary, embed_size)\n",
    "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.embedding_dropout = SpatialDropout(0.3)\n",
    "        \n",
    "        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "    \n",
    "        self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        \n",
    "        self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)\n",
    "        self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        h_embedding = self.embedding(x)\n",
    "        h_embedding = self.embedding_dropout(h_embedding)\n",
    "        \n",
    "        h_lstm1, _ = self.lstm1(h_embedding)\n",
    "        h_lstm2, _ = self.lstm2(h_lstm1)\n",
    "        \n",
    "        # global average pooling\n",
    "        avg_pool = torch.mean(h_lstm2, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(h_lstm2, 1)\n",
    "        \n",
    "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
    "        h_conc_linear1  = F.relu(self.linear1(h_conc))\n",
    "        h_conc_linear2  = F.relu(self.linear2(h_conc))\n",
    "        \n",
    "        hidden = h_conc + h_conc_linear1 + h_conc_linear2\n",
    "        \n",
    "        result = self.linear_out(hidden)\n",
    "        aux_result = self.linear_aux_out(hidden)\n",
    "        out = torch.cat([result, aux_result], 1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fa7cd59",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform to tensors and then datasets\n",
    "x_train_torch = torch.tensor(x_train, dtype=torch.long)\n",
    "x_test_torch = torch.tensor(x_test, dtype=torch.long)\n",
    "y_train_torch = torch.tensor(np.hstack([y_train[:, np.newaxis], y_aux_train]), dtype=torch.float32)\n",
    "print(f'y_train_torch: {y_train_torch}')\n",
    "train_dataset = data.TensorDataset(x_train_torch, y_train_torch)\n",
    "test_dataset = data.TensorDataset(x_test_torch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b90da8da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# training\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def train_model(model, train, test, loss_fn, output_dim, lr=0.001,\n",
    "                batch_size=32, n_epochs=2,\n",
    "                enable_checkpoint_ensemble=True):\n",
    "    # lr=0.001, batch_size=512, n_epochs=4\n",
    "    param_lrs = [{'params': param, 'lr': lr} for param in model.parameters()]\n",
    "    optimizer = torch.optim.Adam(param_lrs, lr=lr)\n",
    "\n",
    "    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)\n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
    "    all_test_preds = []\n",
    "    checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]\n",
    "    \n",
    "    for epoch in range(n_epochs):\n",
    "        start_time = time.time()\n",
    "        \n",
    "        scheduler.step()\n",
    "        \n",
    "        model.train()\n",
    "        avg_loss = 0.\n",
    "        \n",
    "        for data in tqdm(train_loader, disable=False):\n",
    "            x_batch = data[:-1]\n",
    "            y_batch = data[-1]\n",
    "\n",
    "            y_pred = model(*x_batch)            \n",
    "            loss = loss_fn(y_pred, y_batch)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "            avg_loss += loss.item() / len(train_loader)\n",
    "            \n",
    "        model.eval()\n",
    "        test_preds = np.zeros((len(test), output_dim))\n",
    "    \n",
    "        for i, x_batch in enumerate(test_loader):\n",
    "            y_pred = sigmoid(model(*x_batch).detach().cpu().numpy())\n",
    "\n",
    "            test_preds[i * batch_size:(i+1) * batch_size, :] = y_pred\n",
    "\n",
    "        all_test_preds.append(test_preds)\n",
    "        elapsed_time = time.time() - start_time\n",
    "        print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'.format(\n",
    "              epoch + 1, n_epochs, avg_loss, elapsed_time))\n",
    "\n",
    "    if enable_checkpoint_ensemble:\n",
    "        test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)    \n",
    "    else:\n",
    "        test_preds = all_test_preds[-1]\n",
    "        \n",
    "    return test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2299c666",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_test_preds = []\n",
    "\n",
    "def seed_everything(seed=1234):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "for model_idx in range(NUM_MODELS):\n",
    "    print('Model ', model_idx)\n",
    "    seed_everything(1234 + model_idx)\n",
    "    \n",
    "    model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])\n",
    "    # model.cuda()\n",
    "    \n",
    "    test_preds = train_model(model, train_dataset, test_dataset, output_dim=y_train_torch.shape[-1], \n",
    "                             loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))\n",
    "    all_test_preds.append(test_preds)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a11256e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a8bb1ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7338ecb9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}