RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d504118e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A sample from the generated dataset:\n",
      "['a', 'a', 'a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'EOS']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def create_dataset(num_sequences=2**8):\n",
    "    \n",
    "    '''\n",
    "    generates a number os sequences to be used as the data set\n",
    "    \n",
    "    Args:\n",
    "    `num_sequences`: the number of sequences to be generated.\n",
    "    \n",
    "    Returns list of sequences \n",
    "    '''\n",
    "    samples = []\n",
    "    \n",
    "    for _ in range(num_sequences): \n",
    "        num_tokens = np.random.randint(1, 12)\n",
    "        sample = ['a'] * num_tokens + ['b'] * num_tokens + ['EOS']\n",
    "        samples.append(sample)\n",
    "        \n",
    "    return samples\n",
    "\n",
    "sequences = create_dataset()\n",
    "\n",
    "print('A sample from the generated dataset:')\n",
    "print(sequences[0]) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a9281c7",
   "metadata": {},
   "source": [
    "for this project, i will be using one-hot encoding to represent the vocabulary for training. For datasets with larger vocabulary however, one-hot encoding often becomes inefficient because of the size of each sparse vector. To overcome this challenge it is common practice to truncate the vocabulary to contain the most used words and represent the rest with a special symbol, UNK , to define unknown/unimportant words.\n",
    "\n",
    "The create a one-hot encoding for the dataset, we need to assign each word in the vocabulary an index. this can be done by creating two dictionaries: One that gives the index for a given word, and one for the reverse. If a given word is not co ntained in the known vocabulary, it will be given the value of UNK. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e77d3e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "256 sentences and 4 unique tokens in dataset (including UNK).\n",
      "\n",
      "The index of 'b' is 1\n",
      "The word corresponding to index 1 is 'b'\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def sequences_to_dicts(sequences):\n",
    "    \n",
    "    flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "    \n",
    "    # Flatten the dataset\n",
    "    all_words = flatten(sequences)\n",
    "    # Count number of word occurences\n",
    "    word_count = defaultdict(int)\n",
    "    for word in flatten(sequences):\n",
    "        word_count[word] += 1\n",
    "\n",
    "    # Sort by frequency\n",
    "    word_count = sorted(list(word_count.items()), key=lambda l: -l[1])\n",
    "    # Create a list of all unique words\n",
    "    unique_words = [item[0] for item in word_count]\n",
    "    # Add UNK token to list of words\n",
    "    unique_words.append('UNK')\n",
    "    # Count number of sequences and number of unique words\n",
    "    num_sentences, vocab_size = len(sequences), len(unique_words)\n",
    "    # Create dictionaries to go from word to index and back\n",
    "    # If a word is not in the vocabulary, it will be assigned to token 'UNK'\n",
    "    word_to_idx = defaultdict(lambda: vocab_size-1)\n",
    "    idx_to_word = defaultdict(lambda: 'UNK')\n",
    "    # Fill dictionaries\n",
    "    for idx, word in enumerate(unique_words):\n",
    "        word_to_idx[word] = idx\n",
    "        idx_to_word[idx] = word\n",
    "\n",
    "    return word_to_idx, idx_to_word, num_sentences, vocab_size\n",
    "\n",
    "\n",
    "word_to_idx, idx_to_word, num_sequences, vocab_size = sequences_to_dicts(sequences)\n",
    "\n",
    "print(f'{num_sequences} sentences and {len(word_to_idx)} unique tokens in dataset (including UNK).\\n')\n",
    "print('The index of \\'b\\' is', word_to_idx['b'])\n",
    "print(f'The word corresponding to index 1 is \\'{idx_to_word[1]}\\'')\n",
    "\n",
    "assert idx_to_word[word_to_idx['b']] == 'b', \\\n",
    "    'Consistency error: something went wrong in the conversion.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3054df35",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/tf/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from torch.utils import data\n",
    "\n",
    "\n",
    "class Dataset(data.Dataset):\n",
    "    def __init__(self, inputs, targets):\n",
    "        self.inputs = inputs\n",
    "        self.targets = targets\n",
    "\n",
    "    def __len__(self):\n",
    "        # Return the size of the dataset\n",
    "        return len(self.targets)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        # Retrieve inputs and targets at the given index\n",
    "        X = self.inputs[index]\n",
    "        y = self.targets[index]\n",
    "\n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "34ab86fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_datasets(sequences, dataset_class, p_train=0.8, p_val=0.1, p_test=0.1):\n",
    "    # Define partition sizes\n",
    "    num_train = int(len(sequences)*p_train)\n",
    "    num_val = int(len(sequences)*p_val)\n",
    "    num_test = int(len(sequences)*p_test)\n",
    "\n",
    "    # Split sequences into partitions\n",
    "    sequences_train = sequences[:num_train]\n",
    "    sequences_val = sequences[num_train:num_train+num_val]\n",
    "    sequences_test = sequences[-num_test:]\n",
    "\n",
    "    def get_inputs_targets_from_sequences(sequences):\n",
    "        # Define empty lists\n",
    "        inputs, targets = [], []\n",
    "        \n",
    "        # Append inputs and targets s.t. both lists contain L-1 words of a sentence of length L\n",
    "        # but targets are shifted right by one so that we can predict the next word\n",
    "        for sequence in sequences:\n",
    "            inputs.append(sequence[:-1])\n",
    "            targets.append(sequence[1:])\n",
    "            \n",
    "        return inputs, targets\n",
    "\n",
    "    # Get inputs and targets for each partition\n",
    "    inputs_train, targets_train = get_inputs_targets_from_sequences(sequences_train)\n",
    "    inputs_val, targets_val = get_inputs_targets_from_sequences(sequences_val)\n",
    "    inputs_test, targets_test = get_inputs_targets_from_sequences(sequences_test)\n",
    "\n",
    "    # Create datasets\n",
    "    training_set = dataset_class(inputs_train, targets_train)\n",
    "    validation_set = dataset_class(inputs_val, targets_val)\n",
    "    test_set = dataset_class(inputs_test, targets_test)\n",
    "\n",
    "    return training_set, validation_set, test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0630dc06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204 samples in the training set.\n",
      "25 samples in the validation set.\n",
      "25 samples in the test set.\n"
     ]
    }
   ],
   "source": [
    "training_set, validation_set, test_set = create_datasets(sequences, Dataset)\n",
    "\n",
    "print(f'{len(training_set)} samples in the training set.')\n",
    "print(f'{len(validation_set)} samples in the validation set.')\n",
    "print(f'{len(test_set)} samples in the test set.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ab764f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(idx, vocab_size):\n",
    "    \"\"\"\n",
    "    One-hot encodes a single word given its index and the size of the vocabulary.\n",
    "    \n",
    "    Args:\n",
    "     `idx`: the index of the given word\n",
    "     `vocab_size`: the size of the vocabulary\n",
    "    \n",
    "    Returns a 1-D numpy array of length `vocab_size`.\n",
    "    \"\"\"\n",
    "    # Initialize the encoded array\n",
    "    one_hot = np.zeros(vocab_size)\n",
    "    # Set the appropriate element to one\n",
    "    one_hot[idx] = 1.0\n",
    "\n",
    "    return one_hot\n",
    "\n",
    "\n",
    "def one_hot_encode_sequence(sequence, vocab_size):\n",
    "    \"\"\"\n",
    "    One-hot encodes a sequence of words given a fixed vocabulary size.\n",
    "    \n",
    "    Args:\n",
    "     `sequence`: a list of words to encode\n",
    "     `vocab_size`: the size of the vocabulary\n",
    "     \n",
    "    Returns a 3-D numpy array of shape (num words, vocab size, 1).\n",
    "    \"\"\"\n",
    "    # Encode each word in the sentence\n",
    "    encoding = np.array([one_hot_encode(word_to_idx[word], vocab_size) for word in sequence])\n",
    "\n",
    "    # Reshape encoding s.t. it has shape (num words, vocab size, 1)\n",
    "    encoding = encoding.reshape(encoding.shape[0], encoding.shape[1], 1)\n",
    "    \n",
    "    return encoding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f4774e2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one-hot encoding of 'a' has shape (4,).\n",
      "one-hot encoding of 'a b' has shape (2, 4, 1).\n"
     ]
    }
   ],
   "source": [
    "test_word = one_hot_encode(word_to_idx['a'], vocab_size)\n",
    "print(f'one-hot encoding of \\'a\\' has shape {test_word.shape}.')\n",
    "\n",
    "test_sentence = one_hot_encode_sequence(['a', 'b'], vocab_size)\n",
    "print(f'one-hot encoding of \\'a b\\' has shape {test_sentence.shape}.')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f67f4df",
   "metadata": {},
   "source": [
    "U is a weight matrix applied to the given input sample,\n",
    "V is a weight matrix used for the recurrent computation in order to pass memory along the sequence,\n",
    "W is a weight matrix used to compute the output of the every timestep (given that every timestep requires an output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2fc1e434",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U: (50, 4)\n",
      "V: (50, 50)\n",
      "W: (4, 50)\n",
      "b_hidden: (50, 1)\n",
      "b_out: (4, 1)\n"
     ]
    }
   ],
   "source": [
    "hidden_size = 50 # Number of dimensions in the hidden state\n",
    "vocab_size  = len(word_to_idx) # Size of the vocabulary used\n",
    "\n",
    "def init_orthogonal(param):\n",
    "    # Initializes weight parameters orthogonally\n",
    "    if param.ndim < 2:\n",
    "        raise ValueError(\"Only parameters with 2 or more dimensions are supported.\")\n",
    "    \n",
    "    rows, cols = param.shape\n",
    "    new_param = np.random.randn(rows, cols)\n",
    "    if rows < cols:\n",
    "        new_param = new_param.T    \n",
    "    \n",
    "    # Compute QR factorization\n",
    "    q, r = np.linalg.qr(new_param)\n",
    "    \n",
    "    # Make Q uniform \n",
    "    d = np.diag(r, 0)\n",
    "    ph = np.sign(d)\n",
    "    q *= ph\n",
    "\n",
    "    if rows < cols:\n",
    "        q = q.T\n",
    "    \n",
    "    new_param = q\n",
    "    return new_param\n",
    "\n",
    "\n",
    "def init_rnn(hidden_size, vocab_size):\n",
    "    \"\"\"\n",
    "    Initializes recurrent neural network.\n",
    "    Args:\n",
    "     `hidden_size`: the dimensions of the hidden state\n",
    "     `vocab_size`: the dimensions of our vocabulary\n",
    "    \"\"\"\n",
    "     # Weight matrix (input to hidden state)\n",
    "    U = np.zeros((hidden_size, vocab_size))\n",
    "    # Weight matrix (recurrent computation)\n",
    "    V = np.zeros((hidden_size, hidden_size))\n",
    "    # Weight matrix (hidden state to output)\n",
    "    W = np.zeros((vocab_size, hidden_size))\n",
    "    # Bias (hidden state)\n",
    "    b_hidden = np.zeros((hidden_size,1))\n",
    "    # Bias (output)\n",
    "    b_out = np.zeros((vocab_size,1))\n",
    "    \n",
    "    # Initialize weights\n",
    "    U = init_orthogonal(U)\n",
    "    V = init_orthogonal(V)\n",
    "    W = init_orthogonal(W)\n",
    "    \n",
    "    # Return parameters as a tuple\n",
    "    return U, V, W, b_hidden, b_out\n",
    "\n",
    "\n",
    "params = init_rnn(hidden_size=hidden_size, vocab_size=vocab_size)\n",
    "print('U:', params[0].shape)\n",
    "print('V:', params[1].shape)\n",
    "print('W:', params[2].shape)\n",
    "print('b_hidden:', params[3].shape)\n",
    "print('b_out:', params[4].shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "404e6d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x, derivative=False):\n",
    "    \"\"\"\n",
    "    Computes the element-wise sigmoid activation function for an array x.\n",
    "\n",
    "    Args:\n",
    "     `x`: the array where the function is applied\n",
    "     `derivative`: if set to True will return the derivative instead of the forward pass\n",
    "    \"\"\"\n",
    "    x_safe = x + 1e-12\n",
    "    f = 1 / (1 + np.exp(-x_safe))\n",
    "    \n",
    "    if derivative: # Return the derivative of the function evaluated at x\n",
    "        return f * (1 - f)\n",
    "    else: # Return the forward pass of the function at x\n",
    "        return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8f6597b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(x, derivative=False):\n",
    "    \"\"\"\n",
    "    Computes the element-wise tanh activation function for an array x.\n",
    "\n",
    "    Args:\n",
    "     `x`: the array where the function is applied\n",
    "     `derivative`: if set to True will return the derivative instead of the forward pass\n",
    "    \"\"\"\n",
    "    x_safe = x + 1e-12\n",
    "    f = (np.exp(x_safe)-np.exp(-x_safe))/(np.exp(x_safe)+np.exp(-x_safe))\n",
    "    \n",
    "    if derivative: # Return the derivative of the function evaluated at x\n",
    "        return 1-f**2\n",
    "    else: # Return the forward pass of the function at x\n",
    "        return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25d8e55e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x, derivative=False):\n",
    "    \"\"\"\n",
    "    Computes the softmax for an array x.\n",
    "    \n",
    "    Args:\n",
    "     `x`: the array where the function is applied\n",
    "     `derivative`: if set to True will return the derivative instead of the forward pass\n",
    "    \"\"\"\n",
    "    x_safe = x + 1e-12\n",
    "    f = np.exp(x_safe) / np.sum(np.exp(x_safe))\n",
    "    \n",
    "    if derivative: # Return the derivative of the function evaluated at x\n",
    "        pass\n",
    "    else: # Return the forward pass of the function at x\n",
    "        return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7fcec181",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input sequence:\n",
      "['a', 'a', 'a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'b']\n",
      "\n",
      "Target sequence:\n",
      "['a', 'a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'EOS']\n",
      "\n",
      "Predicted sequence:\n",
      "['EOS', 'EOS', 'EOS', 'UNK', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS', 'EOS']\n"
     ]
    }
   ],
   "source": [
    "def forward_pass(inputs, hidden_state, params):\n",
    "    \"\"\"\n",
    "    Computes the forward pass of a vanilla RNN.\n",
    "    \n",
    "    Args:\n",
    "     `inputs`: sequence of inputs to be processed\n",
    "     `hidden_state`: an already initialized hidden state\n",
    "     `params`: the parameters of the RNN\n",
    "    \"\"\"\n",
    "    # First we unpack our parameters\n",
    "    U, V, W, b_hidden, b_out = params\n",
    "    # Create a list to store outputs and hidden states\n",
    "    outputs, hidden_states = [], []\n",
    "    \n",
    "    # For each element in input sequence\n",
    "    for t in range(len(inputs)):\n",
    "\n",
    "        # Compute new hidden state\n",
    "        hidden_state = tanh( ( np.dot(U,inputs[t]) + np.dot(V, hidden_state) ) + b_hidden )\n",
    "        # Compute output\n",
    "        out = np.dot(W, hidden_state) + b_out\n",
    "        out =  np.exp(out) / np.sum(np.exp(out))\n",
    "        # Save results and continue\n",
    "        outputs.append(out)\n",
    "        hidden_states.append(hidden_state.copy())\n",
    "    \n",
    "    return outputs, hidden_states\n",
    "\n",
    "# Get first sequence in training set\n",
    "test_input_sequence, test_target_sequence = training_set[0]\n",
    "# One-hot encode input and target sequence\n",
    "test_input = one_hot_encode_sequence(test_input_sequence, vocab_size)\n",
    "test_target = one_hot_encode_sequence(test_target_sequence, vocab_size)\n",
    "# Initialize hidden state as zeros\n",
    "hidden_state = np.zeros((hidden_size, 1))\n",
    "# Now let's try out our new function\n",
    "outputs, hidden_states = forward_pass(test_input, hidden_state, params)\n",
    "\n",
    "print('Input sequence:')\n",
    "print(test_input_sequence)\n",
    "print('\\nTarget sequence:')\n",
    "print(test_target_sequence)\n",
    "print('\\nPredicted sequence:')\n",
    "print([idx_to_word[np.argmax(output)] for output in outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "26ba3532",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.44540115]\n",
      "[0.23565153]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1.44540115])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(-np.log(sum(outputs[1]*test_target[1])+1e-9))\n",
    "print(outputs[1][0])\n",
    "-np.log(outputs[1][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b889ee3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We get a loss of:\n",
      "[21.45656327]\n"
     ]
    }
   ],
   "source": [
    "def clip_gradient_norm(grads, max_norm=0.25):\n",
    "    \"\"\"\n",
    "    Clips gradients to have a maximum norm of `max_norm`.\n",
    "    This is to prevent the exploding gradients problem.\n",
    "    \"\"\" \n",
    "    # Set the maximum of the norm to be of type float\n",
    "    max_norm = float(max_norm)\n",
    "    total_norm = 0\n",
    "    \n",
    "    # Calculate the L2 norm squared for each gradient and add them to the total norm\n",
    "    for grad in grads:\n",
    "        grad_norm = np.sum(np.power(grad, 2))\n",
    "        total_norm += grad_norm\n",
    "    \n",
    "    total_norm = np.sqrt(total_norm)\n",
    "    \n",
    "    # Calculate clipping coeficient\n",
    "    clip_coef = max_norm / (total_norm + 1e-6)\n",
    "    \n",
    "    # If the total norm is larger than the maximum allowable norm, then clip the gradient\n",
    "    if clip_coef < 1:\n",
    "        for grad in grads:\n",
    "            grad *= clip_coef\n",
    "    \n",
    "    return grads\n",
    "\n",
    "\n",
    "def backward_pass(inputs, outputs, hidden_states, targets, params):\n",
    "    \"\"\"\n",
    "    Computes the backward pass of a vanilla RNN.\n",
    "    \n",
    "    Args:\n",
    "     `inputs`: sequence of inputs to be processed\n",
    "     `outputs`: sequence of outputs from the forward pass\n",
    "     `hidden_states`: sequence of hidden_states from the forward pass\n",
    "     `targets`: sequence of targets\n",
    "     `params`: the parameters of the RNN\n",
    "    \"\"\"\n",
    "    # First we unpack our parameters\n",
    "    U, V, W, b_hidden, b_out = params\n",
    "    \n",
    "    # Initialize gradients as zero\n",
    "    d_U, d_V, d_W = np.zeros_like(U), np.zeros_like(V), np.zeros_like(W)\n",
    "    d_b_hidden, d_b_out = np.zeros_like(b_hidden), np.zeros_like(b_out)\n",
    "    \n",
    "    # Keep track of hidden state derivative and loss\n",
    "    d_h_next = np.zeros_like(hidden_states[0])\n",
    "    loss = 0\n",
    "    \n",
    "    # For each element in output sequence\n",
    "    # NB: We iterate backwards s.t. t = N, N-1, ... 1, 0\n",
    "    for t in reversed(range(len(outputs))):\n",
    "\n",
    "        # Compute cross-entropy loss (as a scalar)\n",
    "        # When taking logarithms, it's a good idea to add a small constant (e.g. 1e-9)\n",
    "        loss += -np.log(sum(outputs[t]*targets[t]))# softmax (cross-entropy loss)\n",
    "        # Backpropagate into output (derivative of cross-entropy)\n",
    "        d_o = outputs[t].copy()\n",
    "        d_o[np.argmax(targets[t])] -= 1\n",
    "        # Backpropagate into W\n",
    "        # YOUR CODE HERE!\n",
    "        d_W += np.dot(d_o, hidden_states[t].T)\n",
    "        d_b_out += d_o\n",
    "        \n",
    "        # Backpropagate into h\n",
    "        d_h = np.dot(W.T, d_o) + d_h_next\n",
    "        \n",
    "        # Backpropagate through non-linearity\n",
    "        # (we assume tanh is used here)\n",
    "        d_f = (1 - hidden_states[t]**2) * d_h\n",
    "        d_b_hidden += d_f\n",
    "        \n",
    "        # Backpropagate into U\n",
    "        d_U += np.dot(d_f, inputs[t].T)\n",
    "        \n",
    "        # Backpropagate into V\n",
    "        d_V += np.dot(d_f, hidden_states[t-1].T)\n",
    "        d_h_next = np.dot(V.T, d_f)\n",
    "    \n",
    "    # Pack gradients\n",
    "    grads = d_U, d_V, d_W, d_b_hidden, d_b_out    \n",
    "    \n",
    "    # Clip gradients\n",
    "    grads = clip_gradient_norm(grads)\n",
    "    \n",
    "    return loss, grads\n",
    "\n",
    "\n",
    "loss, grads = backward_pass(test_input, outputs, hidden_states, test_target, params)\n",
    "\n",
    "print('We get a loss of:')\n",
    "print(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aadf5df7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(params, grads, lr=1e-3):\n",
    "    # Take a step\n",
    "    for param, grad in zip(params, grads):\n",
    "        param -= lr * grad\n",
    "    \n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "24d8cdef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, training loss: [19.02317249], validation loss: [22.68834708]\n",
      "Epoch 100, training loss: [5.37454132], validation loss: [5.4337673]\n",
      "Epoch 200, training loss: [5.03159928], validation loss: [5.4337085]\n",
      "Epoch 300, training loss: [4.49371001], validation loss: [4.8152654]\n",
      "Epoch 400, training loss: [4.30337101], validation loss: [4.57112693]\n",
      "Epoch 500, training loss: [4.17921527], validation loss: [4.38228682]\n",
      "Epoch 600, training loss: [3.9831161], validation loss: [4.14281601]\n",
      "Epoch 700, training loss: [3.74140575], validation loss: [3.90602297]\n",
      "Epoch 800, training loss: [3.76116601], validation loss: [3.96332873]\n",
      "Epoch 900, training loss: [3.97098968], validation loss: [4.23304877]\n",
      "Input sentence:\n",
      "['a', 'b']\n",
      "\n",
      "Target sequence:\n",
      "['b', 'EOS']\n",
      "\n",
      "Predicted sequence:\n",
      "['a', 'EOS']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTCklEQVR4nO3dd3gU1eI+8Hc2ZVNINgXSSIFoAOm9G0GpSlMEVOpVQaUIF70iKoqNiPeHoKDcq18lXkFAhACCBZAmvQYQEEGDoSSGll3Sy57fH5PdZNOX7M5ssu/nec6zu1PPDsi+nnPmjCSEECAiIiJyIhq1K0BERESkNAYgIiIicjoMQEREROR0GICIiIjI6TAAERERkdNhACIiIiKnwwBERERETsdV7Qo4IqPRiKtXr8LHxweSJKldHSIiIqoGIQRu376NsLAwaDSVt/EwAJXj6tWriIiIULsaREREdAcuXbqE8PDwSrdhACqHj48PAPkC+vr6qlwbIiIiqg6DwYCIiAjz73hlGIDKYer28vX1ZQAiIiKqZaozfIWDoImIiMjpMAARERGR02EAIiIiIqfDMUBERGR3hYWFyM/PV7saVAe4u7tXeYt7dTAAERGR3QghkJqaivT0dLWrQnWERqNB48aN4e7uXqPjMAAREZHdmMJPUFAQvLy8OLks1YhpouKUlBRERkbW6O8TAxAREdlFYWGhOfwEBgaqXR2qIxo0aICrV6+ioKAAbm5ud3wcDoImIiK7MI358fLyUrkmVJeYur4KCwtrdBwGICIisit2e5Et2ervEwMQEREROR0GICIiInI6DEBEREQK6NWrF2bMmFHt7S9evAhJkpCYmGi3OgHAzp07IUmS001VwLvAFJSbC/z9NyBJQESE2rUhIqLyVDXGZPz48YiPj7f6uOvWrbPqrqWIiAikpKSgfv36Vp+LqsYApKAjR4CePYG77gIuXFC7NkREVJ6UlBTz+9WrV+P111/HuXPnzMs8PT0tts/Pz69WsAkICLCqHi4uLggJCbFqH6o+VbvA4uLi0KlTJ/j4+CAoKAjDhg2z+EuWn5+PWbNmoVWrVvD29kZYWBjGjRuHq1evVnrc+Ph4SJJUpuTk5Nj7K1XK9N9HQYGq1SAiUo8QQGamOkWIalUxJCTEXHQ6HSRJMn/OycmBn58fvvnmG/Tq1QseHh5Yvnw5bty4gccffxzh4eHw8vJCq1atsHLlSovjlu4Ca9SoEebNm4cnn3wSPj4+iIyMxKeffmpeX7oLzNRV9fPPP6Njx47w8vJC9+7dLX43AeCdd95BUFAQfHx88PTTT+Pll19G27ZtrfpjWrt2LVq0aAGtVotGjRphwYIFFus/+eQTxMTEwMPDA8HBwXj00UfN67799lu0atUKnp6eCAwMRJ8+fZCZmWnV+ZWgagDatWsXpkyZggMHDmDr1q0oKChAv379zBcqKysLx44dw5w5c3Ds2DGsW7cOv//+O4YMGVLlsX19fZGSkmJRPDw87P2VKuVa1N7Gx+EQkdPKygLq1VOnZGXZ7GvMmjULzz//PM6ePYv+/fsjJycHHTp0wKZNm/Drr79i0qRJGDt2LA4ePFjpcRYsWICOHTvi+PHjmDx5Mp577jn89ttvle7z6quvYsGCBThy5AhcXV3x5JNPmtetWLEC7777LubPn4+jR48iMjISS5cuteq7HT16FCNHjsRjjz2GU6dOYe7cuZgzZ4652+/IkSN4/vnn8dZbb+HcuXP48ccfERsbC0BuPXv88cfx5JNP4uzZs9i5cyceeeQRiGqGT0UJB5KWliYAiF27dlW4zaFDhwQA8ddff1W4zbJly4ROp6v2eXNycoRerzeXS5cuCQBCr9dbU/0qnTwpBCBEUJBND0tE5JCys7PFmTNnRHZ2dvHCjAz5H0I1SkaG1d+h9O9JUlKSACAWLVpU5b4PPvigeOGFF8yf77vvPjF9+nTz56ioKDFmzBjzZ6PRKIKCgsTSpUstznX8+HEhhBA7duwQAMS2bdvM+2zevFkAMF/jLl26iClTpljUo0ePHqJNmzYV1tN03Fu3bgkhhHjiiSdE3759Lbb517/+JZo3by6EEGLt2rXC19dXGAyGMsc6evSoACAuXrxY4flqqty/V0X0en21f78d6i4wvV4PoPJ+Ur1eD0mS4OfnV+mxMjIyEBUVhfDwcAwaNAjHjx+vcNu4uDjodDpzibDTCGW2ABGR0/PyAjIy1Ck2nJG6Y8eOFp8LCwvx7rvvonXr1ggMDES9evWwZcsWJCcnV3qc1q1bm9+butrS0tKqvU9oaCgAmPc5d+4cOnfubLF96c9VOXv2LHr06GGxrEePHjh//jwKCwvRt29fREVFITo6GmPHjsWKFSuQVdS61qZNGzzwwANo1aoVRowYgc8++wy3bt2y6vxKcZgAJITAzJkz0bNnT7Rs2bLcbXJycvDyyy/jiSeegK+vb4XHatasGeLj47Fx40asXLkSHh4e5j+88syePRt6vd5cLl26ZJPvVBrHABGR05MkwNtbnWLDGam9vb0tPi9YsAALFy7ESy+9hO3btyMxMRH9+/dHXl5epccpPXhakiQYjcZq72O6Y63kPqXvYhNWdj8JISo9ho+PD44dO4aVK1ciNDQUr7/+Otq0aYP09HS4uLhg69at+OGHH9C8eXMsXrwYTZs2RVJSklV1UILDBKCpU6fi5MmTZQaNmeTn5+Oxxx6D0WjEJ598UumxunbtijFjxqBNmza499578c0336BJkyZYvHhxudtrtVr4+vpaFHtgCxARUd30yy+/YOjQoebfnujo6Ar/p9uemjZtikOHDlksO3LkiFXHaN68Ofbs2WOxbN++fWjSpAlcXFwAAK6urujTpw/ef/99nDx5EhcvXsT27dsByAGsR48eePPNN3H8+HG4u7sjISGhBt/KPhziNvhp06Zh48aN2L17N8LDw8usz8/Px8iRI5GUlITt27dbHVA0Gg06deqkyl/GktgCRERUN919991Yu3Yt9u3bB39/f3zwwQdITU3FPffco2g9pk2bhokTJ6Jjx47o3r07Vq9ejZMnTyI6Orrax3jhhRfQqVMnvP322xg1ahT279+PJUuWmBsfNm3ahD///BOxsbHw9/fH999/D6PRiKZNm+LgwYP4+eef0a9fPwQFBeHgwYO4du2a4tehOlQNQEIITJs2DQkJCdi5cycaN25cZhtT+Dl//jx27NiBwMDAOzpPYmIiWrVqZYtq3zFTC1BBgTwij88HJCKqG+bMmYOkpCT0798fXl5emDRpEoYNG2Ye26qU0aNH488//8SLL76InJwcjBw5EhMmTCjTKlSZ9u3b45tvvsHrr7+Ot99+G6GhoXjrrbcwYcIEAICfnx/WrVuHuXPnIicnBzExMVi5ciVatGiBs2fPYvfu3Vi0aBEMBgOioqKwYMECDBw40E7f+M5JwtrOQRuaPHkyvv76a2zYsAFNmzY1L9fpdPD09ERBQQGGDx+OY8eOYdOmTQgODjZvExAQAHd3dwDAuHHj0LBhQ8TFxQEA3nzzTXTt2hUxMTEwGAz46KOP8NVXX2Hv3r3VGgxmMBig0+mg1+tt2h128yZgym/5+cWBiIioLsrJyUFSUhIaN26s+jQkzqxv374ICQnBV199pXZVbKKyv1fW/H6r+hNsmpugV69eFsuXLVuGCRMm4PLly9i4cSMAlJnEaceOHeb9kpOTodEUD2dKT0/HpEmTkJqaCp1Oh3bt2mH37t1Wj4S3tZKBhwGIiIhsLSsrC//5z3/Qv39/uLi4YOXKldi2bRu2bt2qdtUcjqotQI7KXi1A2dnFd2EaDICPj80OTUTkcNgCpLzs7GwMHjwYx44dQ25uLpo2bYrXXnsNjzzyiNpVs5k60QLkbEre7cg7wYiIyNY8PT2xbds2tatRKzjMbfDOoOjuQQAMQERERGpiAFKQJFneCUZERETqYABSGCdDJCIiUh8DkMI4GSIREZH6GIAUxhYgIiIi9TEAKYwtQEREzqFXr16YMWOG+XOjRo2waNGiSveRJAnr16+v8bltdZzKzJ07t8wcfbUJA5DC2AJEROTYBg8ejD59+pS7bv/+/ZAkCceOHbP6uIcPH8akSZNqWj0LFYWQlJQUh3z8hCNhAFIYW4CIiBzbU089he3bt+Ovv/4qs+6LL75A27Zt0b59e6uP26BBA3iZZsO1s5CQEGi1WkXOVVsxACmMLUBERI5t0KBBCAoKQnx8vMXyrKwsrF69Gk899RRu3LiBxx9/HOHh4fDy8kKrVq2wcuXKSo9bugvs/PnziI2NhYeHB5o3b17u4ypmzZqFJk2awMvLC9HR0ZgzZw7yi35A4uPj8eabb+LEiROQJAmSJJnrXLoL7NSpU7j//vvh6emJwMBATJo0CRkZGeb1EyZMwLBhw/D//t//Q2hoKAIDAzFlyhTzuarDaDTirbfeQnh4OLRaLdq2bYsff/zRvD4vLw9Tp05FaGgoPDw80KhRI/MzPAG5NSsyMhJarRZhYWF4/vnnq33uO8GZoBXGFiAicmZCAFlZ6pzby0uej60qrq6uGDduHOLj4/H6669DKtppzZo1yMvLw+jRo5GVlYUOHTpg1qxZ8PX1xebNmzF27FhER0ejS5cuVZ7DaDTikUceQf369XHgwAEYDAaL8UImPj4+iI+PR1hYGE6dOoWJEyfCx8cHL730EkaNGoVff/0VP/74o3n2Z51OV+YYWVlZGDBgALp27YrDhw8jLS0NTz/9NKZOnWoR8nbs2IHQ0FDs2LEDFy5cwKhRo9C2bVtMnDix6osG4MMPP8SCBQvw3//+F+3atcMXX3yBIUOG4PTp04iJicFHH32EjRs34ptvvkFkZCQuXbqES5cuAQC+/fZbLFy4EKtWrUKLFi2QmpqKEydOVOu8d0xQGXq9XgAQer3e5sdu1UoIQIitW21+aCIih5KdnS3OnDkjsrOzzcsyMuR/A9UoGRnVr/vZs2cFALF9+3bzstjYWPH4449XuM+DDz4oXnjhBfPn++67T0yfPt38OSoqSixcuFAIIcRPP/0kXFxcxKVLl8zrf/jhBwFAJCQkVHiO999/X3To0MH8+Y033hBt2rQps13J43z66afC399fZJS4AJs3bxYajUakpqYKIYQYP368iIqKEgUFBeZtRowYIUaNGlVhXUqfOywsTLz77rsW23Tq1ElMnjxZCCHEtGnTxP333y+MRmOZYy1YsEA0adJE5OXlVXg+k/L+XplY8/vNLjCFcSZoIiLH16xZM3Tv3h1ffPEFAOCPP/7AL7/8gieffBIAUFhYiHfffRetW7dGYGAg6tWrhy1btiA5Oblaxz979iwiIyMRHh5uXtatW7cy23377bfo2bMnQkJCUK9ePcyZM6fa5yh5rjZt2sDb29u8rEePHjAajTh37px5WYsWLeBS4plNoaGhSEtLq9Y5DAYDrl69ih49elgs79GjB86ePQtA7mZLTExE06ZN8fzzz2PLli3m7UaMGIHs7GxER0dj4sSJSEhIQIGdfygZgBRm6gLjGCAickZeXkBGhjrF2vHHTz31FNauXQuDwYBly5YhKioKDzzwAABgwYIFWLhwIV566SVs374diYmJ6N+/P/Ly8qp1bCFEmWVSqf65AwcO4LHHHsPAgQOxadMmHD9+HK+++mq1z1HyXKWPXd453Uo+sbtondFotOpcpc9T8tzt27dHUlIS3n77bWRnZ2PkyJF49NFHAQARERE4d+4cPv74Y3h6emLy5MmIjY21agyStTgGSGFsASIiZyZJQImGCIc2cuRITJ8+HV9//TW+/PJLTJw40fxj/ssvv2Do0KEYM2YMAHlMz/nz53HPPfdU69jNmzdHcnIyrl69irCwMADyLfYl7d27F1FRUXj11VfNy0rfmebu7o7CwsIqz/Xll18iMzPT3Aq0d+9eaDQaNGnSpFr1rYqvry/CwsKwZ88exMbGmpfv27cPnTt3tthu1KhRGDVqFB599FEMGDAAN2/eREBAADw9PTFkyBAMGTIEU6ZMQbNmzXDq1Kk7uuOuOhiAFMYWICKi2qFevXoYNWoUXnnlFej1ekyYMMG87u6778batWuxb98++Pv744MPPkBqamq1A1CfPn3QtGlTjBs3DgsWLIDBYLAIOqZzJCcnY9WqVejUqRM2b96MhIQEi20aNWqEpKQkJCYmIjw8HD4+PmVufx89ejTeeOMNjB8/HnPnzsW1a9cwbdo0jB07FsHBwXd2ccrxr3/9C2+88QbuuusutG3bFsuWLUNiYiJWrFgBAFi4cCFCQ0PRtm1baDQarFmzBiEhIfDz80N8fDwKCwvRpUsXeHl54auvvoKnpyeioqJsVr/S2AWmMLYAERHVHk899RRu3bqFPn36IDIy0rx8zpw5aN++Pfr3749evXohJCQEw4YNq/ZxNRoNEhISkJubi86dO+Ppp5/Gu+++a7HN0KFD8c9//hNTp05F27ZtsW/fPsyZM8dim+HDh2PAgAHo3bs3GjRoUO6t+F5eXvjpp59w8+ZNdOrUCY8++igeeOABLFmyxLqLUYXnn38eL7zwAl544QW0atUKP/74IzZu3IiYmBgAcqCcP38+OnbsiE6dOuHixYv4/vvvodFo4Ofnh88++ww9evRA69at8fPPP+O7775DYGCgTetYkiTK64h0cgaDATqdDnq9Hr6+vjY99sCBwI8/AvHxwPjxNj00EZFDycnJQVJSEho3bgwPDw+1q0N1RGV/r6z5/WYLkMI4ESIREZH6GIAUxokQiYiI1McApDC2ABEREamPAUhhbAEiIiJSHwOQwngbPBE5G95rQ7Zkq79PDEAK423wROQsTDMLZ6n19FOqk0wzYZd8bMed4ESICmMLEBE5CxcXF/j5+ZmfJ+Xl5VXhIxmIqsNoNOLatWvw8vKCq2vNIgwDkMLYAkREziQkJAQAqv1QTaKqaDQaREZG1jhMMwApjC1ARORMJElCaGgogoKC7PpgS3Ie7u7u0GhqPoKHAUhhbAEiImfk4uJS4zEbRLbEQdAKYwsQERGR+hiAFMYWICIiIvUxACmMLUBERETqYwBSGGeCJiIiUh8DkML4LDAiIiL1qRqA4uLi0KlTJ/j4+CAoKAjDhg3DuXPnLLYRQmDu3LkICwuDp6cnevXqhdOnT1d57LVr16J58+bQarVo3rw5EhIS7PU1rMIWICIiIvWpGoB27dqFKVOm4MCBA9i6dSsKCgrQr18/ZGZmmrd5//338cEHH2DJkiU4fPgwQkJC0LdvX9y+fbvC4+7fvx+jRo3C2LFjceLECYwdOxYjR47EwYMHlfhalWILEBERkfok4UBPqbt27RqCgoKwa9cuxMbGQgiBsLAwzJgxA7NmzQIA5ObmIjg4GPPnz8czzzxT7nFGjRoFg8GAH374wbxswIAB8Pf3x8qVK6ush8FggE6ng16vh6+vr22+XJHPPgMmTQKGDAE2bLDpoYmIiJyaNb/fDjUGSK/XAwACAgIAAElJSUhNTUW/fv3M22i1Wtx3333Yt29fhcfZv3+/xT4A0L9//wr3yc3NhcFgsCj2wtvgiYiI1OcwAUgIgZkzZ6Jnz55o2bIlACA1NRUAEBwcbLFtcHCweV15UlNTrdonLi4OOp3OXCIiImryVSrF2+CJiIjU5zABaOrUqTh58mS5XVSlH3gmhKjyIWjW7DN79mzo9XpzuXTpkpW1rz62ABEREanPIZ4FNm3aNGzcuBG7d+9GeHi4ebnpKcKpqakIDQ01L09LSyvTwlNSSEhImdaeyvbRarXQarU1+QrVxhYgIiIi9anaAiSEwNSpU7Fu3Tps374djRs3tljfuHFjhISEYOvWreZleXl52LVrF7p3717hcbt162axDwBs2bKl0n2U4u4uv+blqVsPIiIiZ6ZqC9CUKVPw9ddfY8OGDfDx8TG32uh0Onh6ekKSJMyYMQPz5s1DTEwMYmJiMG/ePHh5eeGJJ54wH2fcuHFo2LAh4uLiAADTp09HbGws5s+fj6FDh2LDhg3Ytm0b9uzZo8r3LMnU0JSbq249iIiInJmqAWjp0qUAgF69elksX7ZsGSZMmAAAeOmll5CdnY3Jkyfj1q1b6NKlC7Zs2QIfHx/z9snJydBoihuzunfvjlWrVuG1117DnDlzcNddd2H16tXo0qWL3b9TVdgCREREpD6HmgfIUdhzHqD9+4Hu3YHoaOCPP2x6aCIiIqdWa+cBcgbsAiMiIlIfA5DCTF1gDEBERETqYQBSmKkFiGOAiIiI1MMApDB2gREREamPAUhhJQMQh58TERGpgwFIYaYxQAAfh0FERKQWBiCFlXziBrvBiIiI1MEApDAGICIiIvUxAClJr4fLnl3QaOTBPwxARERE6mAAUtLp00CvXtCKHAC8FZ6IiEgtDEBKKhoBrYWcfNgCREREpA4GICWZApAkJx8GICIiInUwACmpKAC5F7UAsQuMiIhIHQxASjK1ABWNAWILEBERkToYgJRkHgPELjAiIiI1MQApydQFJhiAiIiI1MQApKRSLUAcA0RERKQOBiAlsQuMiIjIITAAKYkBiIiIyCEwACnJxQWQJN4GT0REpDIGICVJEuDuzhYgIiIilTEAKY0BiIiISHUMQEpzdzd3gTEAERERqYMBSGklWoA4BoiIiEgdDEBKc3NjFxgREZHKGICUxjFAREREqmMAUlqJMUDsAiMiIlIHA5DS2AJERESkOgYgpTEAERERqY4BSGm8DZ6IiEh1DEBKc3eHB3IAMAARERGphQFIae7u8EQ2ACA7W+W6EBEROSkGIKWVaAFiACIiIlKHqgFo9+7dGDx4MMLCwiBJEtavX2+xXpKkcsu///3vCo8ZHx9f7j45OTl2/jbVVKIFyFGqRERE5GxUDUCZmZlo06YNlixZUu76lJQUi/LFF19AkiQMHz680uP6+vqW2dfDw8MeX8F67AIjIiJSnauaJx84cCAGDhxY4fqQkBCLzxs2bEDv3r0RHR1d6XElSSqzr8NgFxgREZHqas0YoL///hubN2/GU089VeW2GRkZiIqKQnh4OAYNGoTjx49Xun1ubi4MBoNFsRt2gREREamu1gSgL7/8Ej4+PnjkkUcq3a5Zs2aIj4/Hxo0bsXLlSnh4eKBHjx44f/58hfvExcVBp9OZS0REhK2rX4xdYERERKqrNQHoiy++wOjRo6scy9O1a1eMGTMGbdq0wb333otvvvkGTZo0weLFiyvcZ/bs2dDr9eZy6dIlW1e/GAMQERGR6lQdA1Rdv/zyC86dO4fVq1dbva9Go0GnTp0qbQHSarXQarU1qWL1lRoDJAQgScqcmoiIiGS1ogXo888/R4cOHdCmTRur9xVCIDExEaGhoXao2R0o0QJkNAIFBSrXh4iIyAmp2gKUkZGBCxcumD8nJSUhMTERAQEBiIyMBAAYDAasWbMGCxYsKPcY48aNQ8OGDREXFwcAePPNN9G1a1fExMTAYDDgo48+QmJiIj7++GP7f6HqKBGAALkVyM1NxfoQERE5IVUD0JEjR9C7d2/z55kzZwIAxo8fj/j4eADAqlWrIITA448/Xu4xkpOTodEUN2Slp6dj0qRJSE1NhU6nQ7t27bB792507tzZfl/EGiWeBg/IAcjXV8X6EBEROSFJCCHUroSjMRgM0Ol00Ov18LV1OvnwQ2DGDHi65CKn0B0XLwJRUbY9BRERkTOy5ve7VowBqlPc3QEAnhq5FYh3ghERESmPAUhppgDkkgeAAYiIiEgNDEBKKwpAHhJbgIiIiNTCAKS0Ul1gfBwGERGR8hiAlGYKQHwgKhERkWoYgJRW9CgPPhGeiIhIPQxASvP0lF+QBYBdYERERGpgAFKaKQAZ5QDEFiAiIiLlMQApzRyAMgEwABEREamBAUhppjFADEBERESqYQBSmqkFqFAOQBwDREREpDwGIKWZu8AyALAFiIiISA0MQErjbfBERESqYwBSWlELkFfRbfBZWWpWhoiIyDkxACnNxQVwc4M35DFAmZkq14eIiMgJMQCpwdOTAYiIiEhFDEBq8PBgACIiIlIRA5Aa2AJERESkKgYgNZQIQBwETUREpDwGIDV4eJjvAmMLEBERkfIYgNTALjAiIiJVMQCpgQGIiIhIVQxAaih1F5gQKteHiIjIyTAAqaFEC5DRCOTmqlwfIiIiJ8MApAZPT/MgaIB3ghERESmNAUgNnp5wQwHcXAoBcBwQERGR0hiA1FD0RHhvtzwADEBERERKYwBSQ9ET4b1d5cE/DEBERETKYgBSgykAuTAAERERqYEBSA2mLjCXHAAMQEREREpjAFKDqQVIkw2Ad4EREREpjQFIDV5e8ovE54ERERGpgQFIDd7e8gsfh0FERKQKVQPQ7t27MXjwYISFhUGSJKxfv95i/YQJEyBJkkXp2rVrlcddu3YtmjdvDq1Wi+bNmyMhIcFO3+AOmQKQMQMAAxAREZHSVA1AmZmZaNOmDZYsWVLhNgMGDEBKSoq5fP/995Uec//+/Rg1ahTGjh2LEydOYOzYsRg5ciQOHjxo6+rfOXMAug2AAYiIiEhprmqefODAgRg4cGCl22i1WoSEhFT7mIsWLULfvn0xe/ZsAMDs2bOxa9cuLFq0CCtXrqxRfW3GFIAKDQAYgIiIiJTm8GOAdu7ciaCgIDRp0gQTJ05EWlpapdvv378f/fr1s1jWv39/7Nu3r8J9cnNzYTAYLIpdmQJQgR4AAxAREZHSHDoADRw4ECtWrMD27duxYMECHD58GPfffz9yK3l8empqKoKDgy2WBQcHIzU1tcJ94uLioNPpzCUiIsJm36FcRQHIK18OQLwNnoiISFmqdoFVZdSoUeb3LVu2RMeOHREVFYXNmzfjkUceqXA/SZIsPgshyiwrafbs2Zg5c6b5s8FgsG8IMrUA5acDYAsQERGR0hw6AJUWGhqKqKgonD9/vsJtQkJCyrT2pKWllWkVKkmr1UKr1dqsnlUy3wbPu8CIiIjU4NBdYKXduHEDly5dQmhoaIXbdOvWDVu3brVYtmXLFnTv3t3e1au+ookQOQ8QERGROlRtAcrIyMCFCxfMn5OSkpCYmIiAgAAEBARg7ty5GD58OEJDQ3Hx4kW88sorqF+/Ph5++GHzPuPGjUPDhg0RFxcHAJg+fTpiY2Mxf/58DB06FBs2bMC2bduwZ88exb9fhVxcAA8PeOcwABEREalB1QB05MgR9O7d2/zZNA5n/PjxWLp0KU6dOoX//e9/SE9PR2hoKHr37o3Vq1fDx8fHvE9ycjI0muKGrO7du2PVqlV47bXXMGfOHNx1111YvXo1unTpotwXqw5vbwYgIiIilUhCCKF2JRyNwWCATqeDXq+Hr6+vfU4SFYWDySHoioOIigIuXrTPaYiIiJyFNb/ftWoMUJ3i7W0eA5SRoXJdiIiInAwDkFq8veELecJFBiAiIiJlMQCppUQAys2VCxERESmDAUgt9erBB7fNH2/frmRbIiIisikGILV4e8MFRni55wMA7P34MSIiIirGAKSWotmgfbVy3xcDEBERkXIYgNRiCkBuOQAYgIiIiJTEAKQWcwCSHwXPMUBERETKYQBSS716AABfF3kuILYAERERKYcBSC1FM1T6SnLTDwMQERGRchiA1GIKQEVzATEAERERKYcBSC2mAGTUA2AAIiIiUhIDkFqKnmjvU3ALAAMQERGRkhiA1GJqAcq/AYB3gRERESmJAUgtpgCUdx0AW4CIiIiUxACkFlMAyr0GgAGIiIhISTYNQH/88Qfuv/9+Wx6y7ioaA+RbeBMAAxAREZGSbBqAMjIysGvXLlsesu4yTYTI2+CJiIgUxy4wtWg0gI8PAxAREZEKGIDUxABERESkCgYgNfn6wgfy/e8ZGYDRqHJ9iIiInISrNRu3a9cOkiRVuD4rK6vGFXIqvr7wxUUAgBBAZqZ5bDQRERHZkVUBaNiwYXaqhpPy9YUHcuDqYkRBoQYGAwMQERGREqwKQG+88Ya96uGcfHwgAfD1yMPNTA8YDEDDhmpXioiIqO6z6RigEydOwMXFxZaHrNtMkyG65wDgQGgiIiKl2HwQtBDC1oesu0wByC0bAKDXq1kZIiIi52HzAFTZIGkqpSgA+blmAgDS01WsCxERkRPhbfBqKhrxHOAqN/3cvKlmZYiIiJyHVYOgDVUMUrl9+3aNKuN0ilqAAqR0AMCtWyrWhYiIyIlYFYD8/Pwq7eISQrALzBpFAchfyE0/bAEiIiJShlUBaPv27Qw4tmRqATJeB8AAREREpBSrAlCvXr3sVA0nZRoDVJAGgAGIiIhIKVYFII1GU2ULkCRJKCgoqFGlnIapCyw3FQDHABERESnFqgCUkJBQ4bp9+/Zh8eLFnAfIGjodACAg+zIAtgAREREpxarb4IcOHVqmNG3aFPHx8ViwYAFGjBiBc+fOVft4u3fvxuDBgxEWFgZJkrB+/Xrzuvz8fMyaNQutWrWCt7c3wsLCMG7cOFy9erXSY8bHx0OSpDIlJyfHmq+qDH9/AEBAjvydGICIiIiUccfzAF29ehUTJ05E69atUVBQgMTERHz55ZeIjIys9jEyMzPRpk0bLFmypMy6rKwsHDt2DHPmzMGxY8ewbt06/P777xgyZEiVx/X19UVKSopF8fDwsOr7KUKnAyQJAZCTD7vAiIiIlGFVFxgA6PV6zJs3D4sXL0bbtm3x888/4957772jkw8cOBADBw4sd51Op8PWrVstli1evBidO3dGcnJypUFLkiSEhIRUux65ubnIzc01f65qviObcXEBdDr4p8vJJysLyMkBHDGrERER1SVWtQC9//77iI6OxqZNm7By5Urs27fvjsPPndDr9ZAkCX5+fpVul5GRgaioKISHh2PQoEE4fvx4pdvHxcVBp9OZS0REhA1rXYWAAPjCAI1GHjvFViAiIiL7k4QVo5Y1Gg08PT3Rp0+fSp/6vm7dOusrIklISEjAsGHDyl2fk5ODnj17olmzZli+fHmFxzlw4AAuXLiAVq1awWAw4MMPP8T333+PEydOICYmptx9ymsBioiIgF6vh2/RnVp207EjcPQo6vvm4obBHb/+CrRoYd9TEhER1UUGgwE6na5av99WdYGNGzdOlYkQ8/Pz8dhjj8FoNOKTTz6pdNuuXbuia9eu5s89evRA+/btsXjxYnz00Ufl7qPVaqHVam1a52oLCAAA+Hvm4IbBnS1ARERECrAqAMXHx9upGhXLz8/HyJEjkZSUhO3bt1vdIqPRaNCpUyecP3/eTjWsoaIAFOCRBcCXd4IREREpwKGfBm8KP+fPn8e2bdsQGBho9TGEEEhMTERoaKgdamgDplvh3eQHyTIAERER2Z/Vd4HZUkZGBi5cuGD+nJSUhMTERAQEBCAsLAyPPvoojh07hk2bNqGwsBCpqfKMyQEBAXB3dwcgd8s1bNgQcXFxAIA333wTXbt2RUxMDAwGAz766CMkJibi448/Vv4LVoepC0yjB8AAREREpARVA9CRI0fQu3dv8+eZM2cCAMaPH4+5c+di48aNAIC2bdta7Ldjxw7zc8mSk5Oh0RQ3ZKWnp2PSpElITU2FTqdDu3btsHv3bnTu3Nm+X+ZOmVqAIA/+4RggIiIi+1M1APXq1avSR2dU5wa1nTt3WnxeuHAhFi5cWNOqKcc0BohPhCciIlKMQ48BcgqmAJT/NwDg+nU1K0NEROQcGIDUVtQFFpQnPxA1LU3NyhARETkHBiC1FbUABWclAWAAIiIiUgIDkNpMLUAZfwJgACIiIlICA5DaTC1AhVcAADduAAUFalaIiIio7mMAUpunJ+DujkDcgEYjIAQHQhMREdkbA5DaJAkICIALjKjvJzf9/P23ynUiIiKq4xiAHEFRN1iQr/xEeo4DIiIisi8GIEdQvz4AILheBgC2ABEREdkbA5AjCA4GAAS5y88DYwsQERGRfTEAOYKgIABAsIs8+pktQERERPbFAOQITC1AQk4+bAEiIiKyLwYgR2AKQPnyXEAMQERERPbFAOQITF1gRY/DYBcYERGRfTEAOQJTCxAfh0FERKQIBiBHUBSAgm+eBSC3AAmhZoWIiIjqNgYgR1AUgEJyLwIA8vLkZ4IRERGRfTAAOQJvb8DLC1rkIShQfhzG5csq14mIiKgOYwByFEWtQOH1cwAwABEREdkTA5CjMAUg39sAGICIiIjsiQHIURTdCh/udRMAAxAREZE9MQA5iqIWoAi3VAAMQERERPbEAOQoTF1gkJMPAxAREZH9MAA5ipAQAEB4zgUADEBERET2xADkKCIiAADh6b8CkAMQJ0MkIiKyDwYgR1EUgBqmHQcAZGYCer2aFSIiIqq7GIAcRVEA8kz7C4GBctMPu8GIiIjsgwHIUQQGAh4eAIDwoDwAwKVLalaIiIio7mIAchSSBISHAwAi/TMAAMnJalaIiIio7mIAciRF3WDRPtcAAH/+qWZliIiI6i4GIEdiCkBuct8XAxAREZF9MAA5ElMAMspzATEAERER2QcDkCMpCkCNs+S5gBiAiIiI7EPVALR7924MHjwYYWFhkCQJ69evt1gvhMDcuXMRFhYGT09P9OrVC6dPn67yuGvXrkXz5s2h1WrRvHlzJCQk2Okb2FjRIOjGN48BANLTgVu3VKwPERFRHaVqAMrMzESbNm2wZMmScte///77+OCDD7BkyRIcPnwYISEh6Nu3L27fvl3hMffv349Ro0Zh7NixOHHiBMaOHYuRI0fi4MGD9voatlPUAuR15bzpyRhsBSIiIrIDSQjHeOCCJElISEjAsGHDAMitP2FhYZgxYwZmzZoFAMjNzUVwcDDmz5+PZ555ptzjjBo1CgaDAT/88IN52YABA+Dv74+VK1eWu09ubi5yc3PNnw0GAyIiIqDX6+Hr62ujb1gNej3g5wcA6NGlAPsOuuCbb4ARI5SrAhERUW1lMBig0+mq9fvtsGOAkpKSkJqain79+pmXabVa3Hfffdi3b1+F++3fv99iHwDo379/pfvExcVBp9OZS0RRS4zidDqgfn0AQHR9+TkYbAEiIiKyPYcNQKmpqQCA4OBgi+XBwcHmdRXtZ+0+s2fPhl6vN5dLak7BfPfdAIBoT7m+DEBERES257AByESSJIvPQogyy2q6j1arha+vr0VRTVEAukuSk8/vv6tXFSIiorrKYQNQSNEo4NItN2lpaWVaeErvZ+0+DqUoADXLPQEAOHdOzcoQERHVTQ4bgBo3boyQkBBs3brVvCwvLw+7du1C9+7dK9yvW7duFvsAwJYtWyrdx6EUBaCmN+QxSykp8thoIiIish1XNU+ekZGBCxcumD8nJSUhMTERAQEBiIyMxIwZMzBv3jzExMQgJiYG8+bNg5eXF5544gnzPuPGjUPDhg0RFxcHAJg+fTpiY2Mxf/58DB06FBs2bMC2bduwZ88exb/fHYmJAQDoLp5AaKgcgM6dAzp3VrleREREdYiqAejIkSPo3bu3+fPMmTMBAOPHj0d8fDxeeuklZGdnY/Lkybh16xa6dOmCLVu2wMfHx7xPcnIyNJrihqzu3btj1apVeO211zBnzhzcddddWL16Nbp06aLcF6uJohYgXLmCe+4rREqKC86eZQAiIiKyJYeZB8iRWDOPgF0EBAC3bmHKqOv4ZHUgXn4ZKGrgIiIiogrUiXmAnJppIHQ9+Xb8335TszJERER1DwOQI2rWDABwT6H83DMGICIiIttiAHJELVsCAJrdlO8Eu3AByMtTs0JERER1CwOQI2rRAgDQ8I/d0OmAggLOB0RERGRLDECOqKgFSPr9HFq3MgIATpxQs0JERER1CwOQI4qMBOrVA/Lz0SYyHQADEBERkS0xADkiSTK3ArWp9wcABiAiIiJbYgByVEXjgFoXHgfAAERERGRLDECOqqgFqOXfP0OjAdLSgFLPeCUiIqI7xADkqNq2BQB4nTxgejwYW4GIiIhshAHIUbVvL78mJ6PtPTkAgGPHVKwPERFRHcIA5Kh8fYGmTQEAnRtcBAAcOqRifYiIiOoQBiBH1qEDAKCzOAiAAYiIiMhWGIAcWceOAIB2qT/AxQW4ehW4ckXlOhEREdUBDECOrCgAeSfuNd0UxlYgIiIiG2AAcmTt2gEaDXD5Mjq3zATAAERERGQLDECOrF49oE0bAEBn7zMAgAMH1KwQERFR3cAA5Oh69gQA9MjcAkAOQHl5alaIiIio9mMAcnRFAajZ6bVo0ADIyQGOHFG5TkRERLUcA5CjKwpA0skTiO2WDwDYvVvNChEREdV+DECOLiwMiI4GjEbEhl0AAOzapXKdiIiIajkGoNrg3nsBALE58jigvXuBggI1K0RERFS7MQDVBn37AgBaJ/4P/v7A7dscB0RERFQTDEC1QZ8+AABN4jH0vVd+MOqPP6pZISIiotqNAag2CA42zwc0ICQRAAMQERFRTTAA1Rb9+gEA+t9aBUCeEfrGDTUrREREVHsxANUW/fsDAMJ2fo3WrQWEALZsUblOREREtRQDUG0RGwv4+wPXruHBVpcBAOvXq1slIiKi2ooBqLZwcwMGDQIADBffAgA2bways9WsFBERUe3EAFSbDBsGAOiwbzGiogQyM4GfflK3SkRERLURA1Bt0r8/4OkJ6WIShvf4GwCwZo3KdSIiIqqFGIBqE29vYOhQAMCI/K8BABs2yBMjEhERUfUxANU248YBALrseA9NYuRuMLYCERERWcfhA1CjRo0gSVKZMmXKlHK337lzZ7nb//bbbwrX3E769gWCgyFdv4Z/dJO/07JlKteJiIiolnH4AHT48GGkpKSYy9atWwEAI0aMqHS/c+fOWewXExOjRHXtz9UVGD0aADDu8jxoNMCePUBdyXdERERKcPgA1KBBA4SEhJjLpk2bcNddd+G+++6rdL+goCCL/VxcXBSqsQKeew6QJITtWIHB92cAAD78UOU6ERER1SIOH4BKysvLw/Lly/Hkk09CkqRKt23Xrh1CQ0PxwAMPYMeOHZVum5ubC4PBYFEc2t13AwMHAkLgn37xAIAvv+SjMYiIiKqrVgWg9evXIz09HRMmTKhwm9DQUHz66adYu3Yt1q1bh6ZNm+KBBx7A7t27K9wnLi4OOp3OXCIiIuxQexubNg0AEPv9y2jbsgDZ2cDHH6tcJyIiolpCEkIItStRXf3794e7uzu+++47q/YbPHgwJEnCxo0by12fm5uL3Nxc82eDwYCIiAjo9Xr4+vrWqM52IwTQoQNw/DhWDl2FJzaMgp8fkJQE+PmpXTkiIiLlGQwG6HS6av1+15oWoL/++gvbtm3D008/bfW+Xbt2xfnz5ytcr9Vq4evra1EcniQBr78OABj58zNo0awA6enAggXqVouIiKg2qDUBaNmyZQgKCsJDDz1k9b7Hjx9HaGioHWqlsiFDgHbt4JKhx1uRnwMAPvgASE5WuV5EREQOrlYEIKPRiGXLlmH8+PFwdXW1WDd79myMK5ocEAAWLVqE9evX4/z58zh9+jRmz56NtWvXYurUqUpX2/40GmDRIgDAw1ueQ8+2GcjKAmbOVLdaREREjq5WBKBt27YhOTkZTz75ZJl1KSkpSC7R5JGXl4cXX3wRrVu3xr333os9e/Zg8+bNeOSRR5SssnJiY4HHHoMEgU8wGS4uAmvX8iGpRERElalVg6CVYs0gKodw6RJwzz1AZiZmxh7Gwt0dEREBnDgB+PurXTkiIiJl1MlB0FSJiAhg4UIAwFv7++HuyFxcugQ884x8sxgRERFZYgCqK55+Ghg0CPXyb+FraQxcXQXWrAE+/VTtihERETkeBqC6QpKAzz8HGjZEp7++xTsxXwIApk4Fdu1SuW5EREQOhgGoLgkKAhISAK0WL539B0Y1TURBATB8uDxBIhEREckYgOqaTp2A//s/SAC+ONcdHRqm4MYN4KGHgPR0tStHRETkGBiA6qIxY4B//xteyMaGKx3R0C8DZ88CDz8MlHjiBxERkdNiAKqrXnwRePllNMRVbE7vCR/PfOzcKY+V5p1hRETk7BiA6rJ584BJk9AGJ7AmdyhcNEYsX25+hBgREZHTYgCqyyQJ+OQTYPRo9Df+gP9qJgMA3nkH+OILletGRESkIgagus7FBYiPBx5+GE8V/Bevub4HAJg0iY/LICIi58UA5AxcXYGVK4H+/fFWwWyMcVuFwkLg0Uflx2UQERE5GwYgZ6HVAuvWQYqNxef549DL9RdkZMi3x9+8qXbliIiIlMUA5Ey8vIBNm+DeuR0SCgajiesfuHIFmDJF7YoREREpiwHI2fj4AD/8AL+7G2B5wWNwkQqxahWwapXaFSMiIlIOA5AzCggA1q9Hp3q/4TXxNgD5mWG3bqlcLyIiIoUwADmrFi2AL77Aq3gXzXEaN24Ab76pdqWIiIiUwQDkzEaMgNvoUViEGQCAJUsEzp1Tt0pERERKYABydosXo2/YGQzCdygslPDOO2pXiIiIyP4YgJydvz+waBHexBsAgK+/ZisQERHVfQxABDz6KNrfWw9DsAFGo4T33lO7QkRERPbFAETyM8MWLsTLmA8A+HqFEWlpKteJiIjIjhiASNahA7o+1gidcAh5+Rr89793fqjMTODMGeDQIeD8efkzERGRI2EAIjPptVcxHR8CAD75KB95edXfV68HFi0COnUCdDr5LvsuXYAmTeS5F9u0AZ5/Hti1CygstE/9iYiIqosBiIq1aIERDxciFFeRet0Na9ZUvYvRCHz4IdC4MfDPfwJHjsgBx98fiIgAvL0BIYCTJ4HFi4FevYCGDeVtjx2T1xERESmNAYgsuL/+MibjEwDAB/NyKg0oKSnAgAHAjBnyLNL33AN8/DFw6ZL8gNXkZCAjA7h6Ffj2W2DCBMDPD/j7b7m1qEMHoFUrYP584PJlBb4cERFREUkI/j94aQaDATqdDnq9Hr6+vmpXR3HX+49G1JZPkQVvfP89MHBg2W2+/14ONNeuAZ6ewAcfABMnAi4ulR87Lw/YsgX46itgwwYgN1deLklA797AuHHAI4/I3WZERETWsOb3mwGoHM4egHDsGF7ssB0L8CK6tc3C3mNekCR5VW4u8PLLcgsOII/tWblSbv2xVnq63DL01VfA7t3Fyz095Zale+8FevYEWraUlxEREVWGAaiGnD4AAUh98Ek0/uFj5MATn/+fwJNPSThyBHjqKXk8DyAPap7/noDHn2eAffvkFdeuAdnZ8gNXIyOBjh3lFOPvX+n5Ll4Eli+Xw9Dvv1uukySgUSOgWTMgKgoICQFCQ+XX4GCgQQO51KsHc1AjIiLnwwBUQwxAAE6fRlzrlXjF+A5cXYxo0VKDEyfkVfXrCyyb+SsGXVsm92P9+Wflx9JogAceAEaNAkaOrLR/Swjg6FHg55+BPXvkXHXzZvWqrNUCQUHFgahBA/lzdDTQtq3cWlWvXvWORUREtQ8DUA0xAMmMb72DMW9EYyWeAABIMOKJyL1YmP4PNDD8UbyhViu38rRrB4SHAx4ewI0bwIULwN69lk06Pj7A+PHAlClyk04VhJAblX77TS6XLwOpqXJJSQHS0oobnaoiSUD79kC/fkD//kD37oCbm7VXhYiIHBUDUA0xABUxGiGmTsOBpceQglC0xzE0wl/yusBA4KGHgKFD5URRWdPKH38Aq1cD//sfLB401qePHIQGDQJcXWtU1czM4jBUsqSlAWfPAomJwJUrlvv4+8sDrkeOlAdgMwwREdVuDEA1xABUyi+/yF1dGRnyYJx77wW6dq36lq/ShJD7thYvBr77rngSoPBw4JlngKeflgf22MnVq8C2bfJdaFu2yAHJJDBQDkJjxgDdunEsERFRbcQAVEMMQAq4eBH473+B//s/4Pp1eZmrK/Dgg8CIEcCQIYAdr31hoXzn2TffAGvXWoah6Ghg9Gi5NG1qtyoQEZGNWfP77dATIc6dOxeSJFmUkCpaCHbt2oUOHTrAw8MD0dHR+M9//qNQbckqjRoBcXHyoJ7ly+Vml4ICYONGYOxYefRy//7Ae+8BBw4AOTk2Pb2Li9zttXSp3DL000/yHET16sljut9+Wx6i1KmTPNP133/b9PRERKQyh24Bmjt3Lr799lts27bNvMzFxQUNGjQod/ukpCS0bNkSEydOxDPPPIO9e/di8uTJWLlyJYYPH17t87IFSCW//io3yaxZI494LkmjkR8s1qoVcPfd8nM2TKVhQ7kPywb9VpmZcgZbvlwORabnlrm4AH37yl1kw4bJj/ggIiLHUme6wObOnYv169cjMTGxWtvPmjULGzduxNmzZ83Lnn32WZw4cQL79++v9nkZgFQmhDxyeds2YOdOeQySqZusIm5u8vihsDB5kiBTMX1u1kx+YJkVISktTc5jy5cDBw8WL/fwkG9669NHDkVt28r5jIiI1FWnAtC///1v6HQ6aLVadOnSBfPmzUN0dHS528fGxqJdu3b48MMPzcsSEhIwcuRIZGVlwa2C23xyc3ORa3omA+QLGBERwQDkKISQ73s/eRI4dQr46y/5QWOXLsmlqnBk4u8vT8zYq5ecXjp0qPZA7vPngRUr5DD0xx+W6/z95Sffd+0qv3buLM8DSUREyqozAeiHH35AVlYWmjRpgr///hvvvPMOfvvtN5w+fRqBgYFltm/SpAkmTJiAV155xbxs37596NGjB65evYrQ0NByzzN37ly8+eabZZYzANUSeXlyQLp6VZ4cyFRMn69ckVuU8vMt99PpgPvvl2/DHzpU7karQsnGqW3bgB075JvjSmvUSO6ta9my+LVpU8Dd3TZfmYiIyqozAai0zMxM3HXXXXjppZcwc+bMMuubNGmCf/zjH5g9e7Z52d69e9GzZ0+kpKRUOICaLUBOIDdXHmN04IB8K/727YBeX7zexQW47z55YqBhw+RxRdWQny/PMXTwoFwOHJDnfyyPi4v8KI+77y5bGjeWu9aIiOjO1dkABAB9+/bF3XffjaVLl5ZZd6ddYKVxDJATKCyUn7nx449AQoKcYkrq0kUOQg8/bPW98Ddvyj11pvLrr3IxGCreR5Lk55pFRsrjust7DQriWCMiosrU2QCUm5uLu+66C5MmTcLrr79eZv2sWbPw3Xff4cyZM+Zlzz33HBITEzkImir3559yEFq7Fij9d+Wee+QwNHiwPIboDqaMFkLukfvjD7mFqHS5fbvqY7i5FY/pLj3Wu+SY7/r1GZSIyDnVmQD04osvYvDgwYiMjERaWhreeecd7Nq1C6dOnUJUVBRmz56NK1eu4H//+x+A4tvgn3nmGUycOBH79+/Hs88+y9vgyTopKfLM1wkJcldZQUHxunr1gNhYeRKh2Fj5CatabY1OJ4Q8jvvSpeKx3aVfr14FjMbqHc/VVW5NKh2OgoPlEhRU/Orry1mviajuqDMB6LHHHsPu3btx/fp1NGjQAF27dsXbb7+N5s2bAwAmTJiAixcvYufOneZ9du3ahX/+8584ffo0wsLCMGvWLDz77LNWnZcBiMzS04HNm4H16+WxQ7duWa53cwNat5ZnTOzYUR7xfM89lT7x/k4UFMhjuUuP8y451jslRZ7R2pr/orVay0BU3qvpff361j/9hIhISXUmAKmFAYjKZTQCJ07It37t2CF3ld24Uf62ERFA8+ZyGGrSRB7lHB0tj4KuYYtRZfLz5fmLygtKaWnyjNam1/LuXquMJMkhqKKw1KCBXOrXl191OrYuEZGyGIBqiAGIqkUIeU6iw4flcuwYcOaMnDYqIknyQJ3GjeUSESF/DguT7zwLC5MThQKPps/KksNQ6WBU8r3p9cYN61qWALkrLjCwOBSZglHp9yWX8U44IqoJBqAaYgCiGrl1S54s6MwZ4PRpeeRzUpJcMjOr3t90S5gpGJVuYin5vkEDu7YomRQUyOOUKgtJ16/LXXDXr1vfumRSr17FAcnfX55g0t/fsuh07Jojqo2EsH0rMQNQDTEAkV0IIScEUxhKSpL7qkyDe0z9ViUHXVeHr2/ZPqiAgIqLv788RsmO/VM5OXIQMhVTMCr5vuSy69et/9omkiRfgpKhqLygZFoeECC3TAUGys90YzcdkW3l58tz05buhi9d2rcHvvvOtudmAKohBiBSjdEoJwNTILpyRW5iuXat7Ou1a8VPa7WWq6tlIiiv+PnJzSulX318bH6fvRDyvJSVhaVbt8qW6jSoVcbdvTgQlQxG5b0PCpIfN+fnx9BEzkUI+b810z87Jcv163ILcMlgU92nE7VuLQ+rtCUGoBpiAKJawWiU71IrHY5u3iy/3LolD+YpMev5HTE1uZQMRRW9r2iZh4dNUkReXvnBqKJy86Z8CW7ckPe9E+7ucq9kSIhcKntv45sBie6YEHLLrF4v/7Oh11f8/tat4v/xMJWcHOvO5+oq/3dQ3nxlptKwofxqSwxANcQARHVadnbFAcn0/saN4n8NS77eaWoozdVVHvDj4yMX0/vqLiu93svLqlYpIeRB4KYwVDIYlffeNP6p5NNTqqNePfkf+cpKSAjHMFHV8vLkv4+msFK6GAzlLy8ZcGr6n69Wazn8sGQpPSFrYKA6E7IyANUQAxBRBUr/L2R5//tY1TJ7/ZPj5SUnDm9vy9fqLitvnZeXRTrJyZGb+1NTq36t7kBwjUYOQQ0bAuHhZQOSaZm3t30uG6mnoAC4fFm+mfSvvyxvJijZ/XvtWuWP0rGGqQG3qkZa0w0IJUttGDPHAFRDDEBEdmI0ysnAYJBfb9+Wi+m9tcsyMuwXqEw8PS3DUXmDhsr5nKkNwNXr7rhyRR7KdfkyzO9NJSWl+sO4/P2LA5GplP7MuZccS16ePJP7xYtywCn9evmydcP4SvY+m0rpzyWXlTeMr149hVtmsrLKDugzvQ8LA6ZMsenpGIBqiAGIqJYQQu7Sy8iQS2am5WtNltnin0adrnhgkGla7RLvCxuEIM0lFFfyg3DlWvlh6fLl6rcmeXtXHZL4rDjbyc6WH1dTXri5eFG+j6Gqv0bu7vLDjqOi5L8aJVtcSrfC+Pmp9GcnhPw/LaZu8vJey1t27ZocgCrStWvZZy/WkDW/3642PTMRkZIkSe6m8vKSb9OyFVOwKh2Kbt+2HCdV0aChW7fk1i5Tt9/vv5d7GhcAoUWlo5+fZVrpWZxeDH6RuIxwXDb44vIVyRyMSpabN+Vq/v57hacDIM+xWfI0pQNSWJh8KTkppfxHnpxcfri5eFHusqqKhwfQqJEccEq+mt6HhNgh1BiNln9nK2pJteZ9dR9GWB43t7KJrn59oFkz233nO8AWoHKwBYiIasR0h17JmSNLDxQq+T4/v3rH9fKqMLlkB4bjikskLmcF4PJVTZmQdOWKfLrq/otfr17FA15L/oaVHD9io5v77K70H8+lS5YPHzaV0o/+K0+9euUHHNNrgwZWXBPT6PyKWlnS04sDSXndwabXms4PURGt1nI+sapeTYHHzvOOlcQusBpiACIixQgh/7ilpsIitZROMBU9d640Nze5Gad0SGrYEPkhEUhxjcDlghBc+du1TCuSaU7O6uax8k5d3sBaHx85u3l6ysX0vvSrq6s85ryiIknywOH8fPm15Pv8fDk7lNeAYTAU38ln7RRaOl1xi015Acffv4rfdqNRPnlKivxnbColP//9d3HQudOLXx5JqvwOSmve+/vLf0gOjgGohhiAiMjhZGfL6aR0aikZlqrbxGN63ErJ28yKXkVQMPReobgmBeGaMRDX0t3KnQDv2jU5k9n75j570unk7r6IiOISGWn5ucKfgIyMigNNyc9padZPWOrqWn6riilNlpwCovR0ESVfPT1rR5OcDTEA1RADEBHVSqZnEJRuQSr93ppWBn9/OSWUV4p+nI06f2S4B0Cv8Yde+CI9081i9oPbt+X8lpVV+WtBgZwVjEb5tXQRQm5lcnUtfi1ZvLzKNl6Yiqk3xlT1+vXLeYyePeY5AOQQUr++PEmOaaZM0yyBISFyhQIDi4NObbjf3EExANUQAxAR1VmmLpnyApLp0SumvqI7fdSKt7f8Y+7nJ7/6+srLqioeHpYJp7z3Go38HUoWU2oyGuVwZ0pVpmL6nJlZ9o6lku+tHTvj5VUcYkqHm5LLGjSQ6092x7vAiIiofBpNcTNIu3YVb2c0yqHAFIhMI4ZLfr55U27mMQWJ27flfTMz5XL5siJfyabc3S2fa1LRa2io3NREtRYDEBERlaXRFE/seM891dunoKB45m9TKDL1g5laYMorpnU5OXILTsmRzaU/G43yiGiNxrKYlpn6wioqfn7F42pKl8BAPu3WiTAAERGRbbi6FocmIgfH+UCJiIjI6TAAERERkdNhACIiIiKnwwBERERETocBiIiIiJwOAxARERE5HQYgIiIicjoMQEREROR0GICIiIjI6TAAERERkdNhACIiIiKnwwBERERETocBiIiIiJwOAxARERE5HVe1K+CIhBAAAIPBoHJNiIiIqLpMv9um3/HKMACV4/bt2wCAiIgIlWtCRERE1rp9+zZ0Ol2l20iiOjHJyRiNRly9ehU+Pj6QJMmmxzYYDIiIiMClS5fg6+tr02NTMV5nZfA6K4fXWhm8zsqw13UWQuD27dsICwuDRlP5KB+2AJVDo9EgPDzcrufw9fXlf1wK4HVWBq+zcnitlcHrrAx7XOeqWn5MOAiaiIiInA4DEBERETkdBiCFabVavPHGG9BqtWpXpU7jdVYGr7NyeK2VweusDEe4zhwETURERE6HLUBERETkdBiAiIiIyOkwABEREZHTYQAiIiIip8MApKBPPvkEjRs3hoeHBzp06IBffvlF7SrVKnFxcejUqRN8fHwQFBSEYcOG4dy5cxbbCCEwd+5chIWFwdPTE7169cLp06cttsnNzcW0adNQv359eHt7Y8iQIbh8+bKSX6VWiYuLgyRJmDFjhnkZr7NtXLlyBWPGjEFgYCC8vLzQtm1bHD161Lye19k2CgoK8Nprr6Fx48bw9PREdHQ03nrrLRiNRvM2vNbW2717NwYPHoywsDBIkoT169dbrLfVNb116xbGjh0LnU4HnU6HsWPHIj09veZfQJAiVq1aJdzc3MRnn30mzpw5I6ZPny68vb3FX3/9pXbVao3+/fuLZcuWiV9//VUkJiaKhx56SERGRoqMjAzzNu+9957w8fERa9euFadOnRKjRo0SoaGhwmAwmLd59tlnRcOGDcXWrVvFsWPHRO/evUWbNm1EQUGBGl/LoR06dEg0atRItG7dWkyfPt28nNe55m7evCmioqLEhAkTxMGDB0VSUpLYtm2buHDhgnkbXmfbeOedd0RgYKDYtGmTSEpKEmvWrBH16tUTixYtMm/Da22977//Xrz66qti7dq1AoBISEiwWG+razpgwADRsmVLsW/fPrFv3z7RsmVLMWjQoBrXnwFIIZ07dxbPPvusxbJmzZqJl19+WaUa1X5paWkCgNi1a5cQQgij0ShCQkLEe++9Z94mJydH6HQ68Z///EcIIUR6erpwc3MTq1atMm9z5coVodFoxI8//qjsF3Bwt2/fFjExMWLr1q3ivvvuMwcgXmfbmDVrlujZs2eF63mdbeehhx4STz75pMWyRx55RIwZM0YIwWttC6UDkK2u6ZkzZwQAceDAAfM2+/fvFwDEb7/9VqM6swtMAXl5eTh69Cj69etnsbxfv37Yt2+fSrWq/fR6PQAgICAAAJCUlITU1FSL66zVanHfffeZr/PRo0eRn59vsU1YWBhatmzJP4tSpkyZgoceegh9+vSxWM7rbBsbN25Ex44dMWLECAQFBaFdu3b47LPPzOt5nW2nZ8+e+Pnnn/H7778DAE6cOIE9e/bgwQcfBMBrbQ+2uqb79++HTqdDly5dzNt07doVOp2uxtedD0NVwPXr11FYWIjg4GCL5cHBwUhNTVWpVrWbEAIzZ85Ez5490bJlSwAwX8vyrvNff/1l3sbd3R3+/v5ltuGfRbFVq1bh2LFjOHz4cJl1vM628eeff2Lp0qWYOXMmXnnlFRw6dAjPP/88tFotxo0bx+tsQ7NmzYJer0ezZs3g4uKCwsJCvPvuu3j88ccB8O+0PdjqmqampiIoKKjM8YOCgmp83RmAFCRJksVnIUSZZVQ9U6dOxcmTJ7Fnz54y6+7kOvPPotilS5cwffp0bNmyBR4eHhVux+tcM0ajER07dsS8efMAAO3atcPp06exdOlSjBs3zrwdr3PNrV69GsuXL8fXX3+NFi1aIDExETNmzEBYWBjGjx9v3o7X2vZscU3L294W151dYAqoX78+XFxcyqTVtLS0MumYqjZt2jRs3LgRO3bsQHh4uHl5SEgIAFR6nUNCQpCXl4dbt25VuI2zO3r0KNLS0tChQwe4urrC1dUVu3btwkcffQRXV1fzdeJ1rpnQ0FA0b97cYtk999yD5ORkAPz7bEv/+te/8PLLL+Oxxx5Dq1atMHbsWPzzn/9EXFwcAF5re7DVNQ0JCcHff/9d5vjXrl2r8XVnAFKAu7s7OnTogK1bt1os37p1K7p3765SrWofIQSmTp2KdevWYfv27WjcuLHF+saNGyMkJMTiOufl5WHXrl3m69yhQwe4ublZbJOSkoJff/2VfxZFHnjgAZw6dQqJiYnm0rFjR4wePRqJiYmIjo7mdbaBHj16lJnG4ffff0dUVBQA/n22paysLGg0lj93Li4u5tvgea1tz1bXtFu3btDr9Th06JB5m4MHD0Kv19f8utdoCDVVm+k2+M8//1ycOXNGzJgxQ3h7e4uLFy+qXbVa47nnnhM6nU7s3LlTpKSkmEtWVpZ5m/fee0/odDqxbt06cerUKfH444+Xe9tleHi42LZtmzh27Ji4//77nfpW1uooeReYELzOtnDo0CHh6uoq3n33XXH+/HmxYsUK4eXlJZYvX27ehtfZNsaPHy8aNmxovg1+3bp1on79+uKll14yb8Nrbb3bt2+L48ePi+PHjwsA4oMPPhDHjx83T+9iq2s6YMAA0bp1a7F//36xf/9+0apVK94GX9t8/PHHIioqSri7u4v27dubb9+m6gFQblm2bJl5G6PRKN544w0REhIitFqtiI2NFadOnbI4TnZ2tpg6daoICAgQnp6eYtCgQSI5OVnhb1O7lA5AvM628d1334mWLVsKrVYrmjVrJj799FOL9bzOtmEwGMT06dNFZGSk8PDwENHR0eLVV18Vubm55m14ra23Y8eOcv9NHj9+vBDCdtf0xo0bYvTo0cLHx0f4+PiI0aNHi1u3btW4/pIQQtSsDYmIiIioduEYICIiInI6DEBERETkdBiAiIiIyOkwABEREZHTYQAiIiIip8MARERERE6HAYiIiIicDgMQEREROR0GICKiapAkCevXr1e7GkRkIwxAROTwJkyYAEmSypQBAwaoXTUiqqVc1a4AEVF1DBgwAMuWLbNYptVqVaoNEdV2bAEiolpBq9UiJCTEovj7+wOQu6eWLl2KgQMHwtPTE40bN8aaNWss9j916hTuv/9+eHp6IjAwEJMmTUJGRobFNl988QVatGgBrVaL0NBQTJ061WL99evX8fDDD8PLywsxMTHYuHGjfb80EdkNAxAR1Qlz5szB8OHDceLECYwZMwaPP/44zp49CwDIysrCgAED4O/vj8OHD2PNmjXYtm2bRcBZunQppkyZgkmTJuHUqVPYuHEj7r77botzvPnmmxg5ciROnjyJBx98EKNHj8bNmzcV/Z5EZCM1fp48EZGdjR8/Xri4uAhvb2+L8tZbbwkhhAAgnn32WYt9unTpIp577jkhhBCffvqp8Pf3FxkZGeb1mzdvFhqNRqSmpgohhAgLCxOvvvpqhXUAIF577TXz54yMDCFJkvjhhx9s9j2JSDkcA0REtULv3r2xdOlSi2UBAQHm9926dbNY161bNyQmJgIAzp49izZt2sDb29u8vkePHjAajTh37hwkScLVq1fxwAMPVFqH1q1bm997e3vDx8cHaWlpd/qViEhFDEBEVCt4e3uX6ZKqiiRJAAAhhPl9edt4enpW63hubm5l9jUajVbViYgcA8cAEVGdcODAgTKfmzVrBgBo3rw5EhMTkZmZaV6/d+9eaDQaNGnSBD4+PmjUqBF+/vlnRetMROphCxAR1Qq5ublITU21WObq6or69esDANasWYOOHTuiZ8+eWLFiBQ4dOoTPP/8cADB69Gi88cYbGD9+PObOnYtr165h2rRpGDt2LIKDgwEAc+fOxbPPPougoCAMHDgQt2/fxt69ezFt2jRlvygRKYIBiIhqhR9//BGhoaEWy5o2bYrffvsNgHyH1qpVqzB58mSEhIRgxYoVaN68OQDAy8sLP/30E6ZPn45OnTrBy8sLw4cPxwcffGA+1vjx45GTk4OFCxfixRdfRP369fHoo48q9wWJSFGSEEKoXQkiopqQJAkJCQkYNmyY2lUholqCY4CIiIjI6TAAERERkdPhGCAiqvXYk09E1mILEBERETkdBiAiIiJyOgxARERE5HQYgIiIiMjpMAARERGR02EAIiIiIqfDAEREREROhwGIiIiInM7/B4dQ/VjTfDoXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Hyper-parameters\n",
    "num_epochs = 1000\n",
    "\n",
    "# Initialize a new network\n",
    "params = init_rnn(hidden_size=hidden_size, vocab_size=vocab_size)\n",
    "\n",
    "# Initialize hidden state as zeros\n",
    "hidden_state = np.zeros((hidden_size, 1))\n",
    "\n",
    "# Track loss\n",
    "training_loss, validation_loss = [], []\n",
    "\n",
    "# For each epoch\n",
    "for i in range(num_epochs):\n",
    "    \n",
    "    # Track loss\n",
    "    epoch_training_loss = 0\n",
    "    epoch_validation_loss = 0\n",
    "    \n",
    "     # For each sentence in validation set\n",
    "    for inputs, targets in validation_set:\n",
    "        # One-hot encode input and target sequence\n",
    "        inputs_one_hot = one_hot_encode_sequence(inputs, vocab_size)\n",
    "        targets_one_hot = one_hot_encode_sequence(targets, vocab_size)\n",
    "        \n",
    "        # Re-initialize hidden state\n",
    "        hidden_state = np.zeros_like(hidden_state)\n",
    "\n",
    "        # Forward pass\n",
    "        outputs, hidden_states = forward_pass(inputs_one_hot, hidden_state, params)\n",
    "\n",
    "        # Backward pass\n",
    "        loss, _ = backward_pass(inputs_one_hot, outputs, hidden_states, targets_one_hot, params)\n",
    "        \n",
    "        # Update loss\n",
    "        epoch_validation_loss += loss\n",
    "    \n",
    "    # For each sentence in training set\n",
    "    for inputs, targets in training_set:\n",
    "        \n",
    "        # One-hot encode input and target sequence\n",
    "        inputs_one_hot = one_hot_encode_sequence(inputs, vocab_size)\n",
    "        targets_one_hot = one_hot_encode_sequence(targets, vocab_size)\n",
    "        # Re-initialize hidden state\n",
    "        hidden_state = np.zeros_like(hidden_state)\n",
    "        # Forward pass\n",
    "        outputs, hidden_states = forward_pass(inputs_one_hot, hidden_state, params)\n",
    "        # Backward pass\n",
    "        loss, grads = backward_pass(inputs_one_hot, outputs, hidden_states, targets_one_hot, params)\n",
    "        \n",
    "        if np.isnan(loss):\n",
    "            raise ValueError('Gradients have vanished/exploded!')\n",
    "        \n",
    "        # Update parameters\n",
    "        params = update_parameters(params, grads, lr=1e-3)\n",
    "        # Update loss\n",
    "        epoch_training_loss += loss  \n",
    "    # Save loss for plot\n",
    "    training_loss.append(epoch_training_loss/len(training_set))\n",
    "    validation_loss.append(epoch_validation_loss/len(validation_set))\n",
    "    \n",
    "    # Print loss every 100 epochs\n",
    "    if i % 100 == 0:\n",
    "        print(f'Epoch {i}, training loss: {training_loss[-1]}, validation loss: {validation_loss[-1]}')\n",
    "        \n",
    "# Get first sentence in test set\n",
    "inputs, targets = test_set[1]\n",
    "# One-hot encode input and target sequence\n",
    "inputs_one_hot = one_hot_encode_sequence(inputs, vocab_size)\n",
    "targets_one_hot = one_hot_encode_sequence(targets, vocab_size)\n",
    "# Initialize hidden state as zeros\n",
    "hidden_state = np.zeros((hidden_size, 1))\n",
    "# Forward pass\n",
    "outputs, hidden_states = forward_pass(inputs_one_hot, hidden_state, params)\n",
    "output_sentence = [idx_to_word[np.argmax(output)] for output in outputs]\n",
    "\n",
    "print('Input sentence:')\n",
    "print(inputs)\n",
    "print('\\nTarget sequence:')\n",
    "print(targets)\n",
    "print('\\nPredicted sequence:')\n",
    "print([idx_to_word[np.argmax(output)] for output in outputs])\n",
    "\n",
    "# Plot training and validation loss\n",
    "epoch = np.arange(len(training_loss))\n",
    "plt.figure()\n",
    "plt.plot(epoch, training_loss, 'r', label='Training loss',)\n",
    "plt.plot(epoch, validation_loss, 'b', label='Validation loss')\n",
    "plt.legend()\n",
    "plt.xlabel('Epoch'), plt.ylabel('NLL')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c9c75607",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example 0: a a b\n",
      "Predicted sequence: ['a', 'a', 'b', 'b', 'EOS']\n",
      "\n",
      "Example 1: a a a a b\n",
      "Predicted sequence: ['a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'EOS']\n",
      "\n",
      "Example 2: a a a a a a b\n",
      "Predicted sequence: ['a', 'a', 'a', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'EOS']\n",
      "\n",
      "Example 3: a\n",
      "Predicted sequence: ['a', 'a', 'b', 'b', 'EOS']\n",
      "\n",
      "Example 4: r n n\n",
      "Predicted sequence: ['r', 'n', 'n', 'EOS', 'EOS']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def freestyle(params, sentence='', num_generate=10):\n",
    "    \"\"\"\n",
    "    Takes in a sentence as a string and outputs a sequence\n",
    "    based on the predictions of the RNN.\n",
    "    \n",
    "    Args:\n",
    "     `params`: the parameters of the network\n",
    "     `sentence`: string with whitespace-separated tokens\n",
    "     `num_generate`: the number of tokens to generate\n",
    "    \"\"\"\n",
    "    sentence = sentence.split(' ')\n",
    "    sentence_one_hot = one_hot_encode_sequence(sentence, vocab_size)\n",
    "    # Initialize hidden state as zeros\n",
    "    hidden_state = np.zeros((hidden_size, 1))\n",
    "    # Generate hidden state for sentence\n",
    "    outputs, hidden_states = forward_pass(sentence_one_hot, hidden_state, params)\n",
    "    # Output sentence\n",
    "    output_sentence = sentence\n",
    "    # Append first prediction\n",
    "    word = idx_to_word[np.argmax(outputs[-1])]    \n",
    "    output_sentence.append(word)\n",
    "    \n",
    "    # Forward pass\n",
    "    for i in range(num_generate):\n",
    "        # Get the latest prediction and latest hidden state\n",
    "        output = outputs[-1]\n",
    "        hidden_state = hidden_states[-1]\n",
    "        # Reshape our output to match the input shape of our forward pass\n",
    "        output = output.reshape(1, output.shape[0], output.shape[1])\n",
    "        # Forward pass\n",
    "        outputs, hidden_states = forward_pass(output, hidden_state, params)\n",
    "        # Compute the index of the most likely word and look up the corresponding word\n",
    "        word = idx_to_word[np.argmax(outputs)]\n",
    "        \n",
    "        output_sentence.append(word)\n",
    "        \n",
    "        if word == 'EOS':\n",
    "            break\n",
    "        \n",
    "    return output_sentence\n",
    "\n",
    "# Perform freestyle (extrapolation)\n",
    "test_examples = ['a a b', 'a a a a b', 'a a a a a a b', 'a', 'r n n']\n",
    "for i, test_example in enumerate(test_examples):\n",
    "    print(f'Example {i}:', test_example)\n",
    "    print('Predicted sequence:', freestyle(params, sentence=test_example), end='\\n\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}