nmt_transformer_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nmt_transformer_model.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPLJNgwG0hRdPh7yqsvxQuc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/singularity014/Transformer_NeuroMachineTranslation/blob/master/nmt_transformer_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9pLHR-FnVI8y",
        "colab_type": "text"
      },
      "source": [
        "# 1) import dependencies\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w2OSB6KOQIIt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import math\n",
        "import re\n",
        "\n",
        "from google.colab import drive"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TtNCl-_IQTEe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0c97edb4-82ba-4854-ca46-922f9354dde2"
      },
      "source": [
        "import tensorflow as tf\n",
        "tf.__version__"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.3.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_7ifKAmQlFl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras import layers\n",
        "import tensorflow_datasets as tfds"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnW4hTNLRccT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "bd462447-71ce-4b92-c644-a65758fbf71c"
      },
      "source": [
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvDHT2IUU0M7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7mjAUxHVS01",
        "colab_type": "text"
      },
      "source": [
        "# II) Data pre-processing\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCcrXg4qVaWu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# English (Translated)\n",
        "with open(\"/content/drive/My Drive/nmt_data/europarl-v7.fr-en.en\", 'r', encoding='utf-8') as f:\n",
        "    europarl_en = f.read()\n",
        "\n",
        "# French (Original)\n",
        "with open(\"/content/drive/My Drive/nmt_data/europarl-v7.fr-en.fr\", 'r', encoding='utf-8') as f:\n",
        "    europarl_fr = f.read()\n",
        "\n",
        "# Prefixes English     \n",
        "with open(\"/content/drive/My Drive/nmt_data/nonbreaking_prefix.en\", 'r', encoding='utf-8') as f:\n",
        "    non_breaking_prefix_en = f.read()\n",
        "\n",
        "# Prefixes French \n",
        "with open(\"/content/drive/My Drive/nmt_data/nonbreaking_prefix.fr\",\n",
        "          mode='r',\n",
        "          encoding='utf-8') as f:\n",
        "    non_breaking_prefix_fr = f.read()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dN_tJJxFbv4o",
        "colab_type": "text"
      },
      "source": [
        "## Cleaning data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FtpbDsJqVeia",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "non_breaking_prefix_en = non_breaking_prefix_en.split(\"\\n\")\n",
        "non_breaking_prefix_en = [' ' + pref + '.' for pref in non_breaking_prefix_en]\n",
        "non_breaking_prefix_fr = non_breaking_prefix_fr.split(\"\\n\")\n",
        "non_breaking_prefix_fr = [' ' + pref + '.' for pref in non_breaking_prefix_fr]"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZn558KIYh2Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_en = europarl_en\n",
        "# Add $$$ after non ending sentence points\n",
        "for prefix in non_breaking_prefix_en:\n",
        "    corpus_en = corpus_en.replace(prefix, prefix + '$$$')\n",
        "corpus_en = re.sub(r\"\\.(?=[0-9]|[a-z]|[A-Z])\", \".$$$\", corpus_en)\n",
        "# Remove $$$ markers\n",
        "corpus_en = re.sub(r\".\\$\\$\\$\", '', corpus_en)\n",
        "# Clear multiple spaces\n",
        "corpus_en = re.sub(r\"  +\", \" \", corpus_en)\n",
        "corpus_en = corpus_en.split('\\n')\n",
        "\n",
        "corpus_fr = europarl_fr\n",
        "for prefix in non_breaking_prefix_fr:\n",
        "    corpus_fr = corpus_fr.replace(prefix, prefix + '$$$')\n",
        "corpus_fr = re.sub(r\"\\.(?=[0-9]|[a-z]|[A-Z])\", \".$$$\", corpus_fr)\n",
        "corpus_fr = re.sub(r\".\\$\\$\\$\", '', corpus_fr)\n",
        "corpus_fr = re.sub(r\"  +\", \" \", corpus_fr)\n",
        "corpus_fr = corpus_fr.split('\\n')"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4cIBRpcGcGR",
        "colab_type": "text"
      },
      "source": [
        "## Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swi5ykctctE4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    corpus_en, target_vocab_size=2**13)\n",
        "tokenizer_fr = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    corpus_fr, target_vocab_size=2**13)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AkyXo1CjD-J1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VOCAB_SIZE_EN = tokenizer_en.vocab_size + 2 # = 8190\n",
        "VOCAB_SIZE_FR = tokenizer_fr.vocab_size + 2 # = 8171"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6j9SAmbGhQG",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9kq16-BERca",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = [[VOCAB_SIZE_EN-2] + tokenizer_en.encode(sentence) + [VOCAB_SIZE_EN-1]\n",
        "          for sentence in corpus_en]\n",
        "outputs = [[VOCAB_SIZE_FR-2] + tokenizer_fr.encode(sentence) + [VOCAB_SIZE_FR-1]\n",
        "           for sentence in corpus_fr]"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mc-byEEzGrb9",
        "colab_type": "text"
      },
      "source": [
        "## Remove Too Long Sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kj2-PKEEWg1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LENGTH = 20\n",
        "idx_to_remove = [count for count, sent in enumerate(inputs)\n",
        "                 if len(sent) > MAX_LENGTH]\n",
        "for idx in reversed(idx_to_remove):\n",
        "    del inputs[idx]\n",
        "    del outputs[idx]\n",
        "idx_to_remove = [count for count, sent in enumerate(outputs)\n",
        "                 if len(sent) > MAX_LENGTH]\n",
        "for idx in reversed(idx_to_remove):\n",
        "    del inputs[idx]\n",
        "    del outputs[idx]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "myLRNMWwG4QE",
        "colab_type": "text"
      },
      "source": [
        "## Inputs/outputs creation\n",
        "As we train with batches, we need each input to have the same length. We pad with the appropriate token, and we will make sure this padding token doesn't interfere with our training later."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NYRoud54Gwje",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}