6_smallBert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "AIM: To perform intent classification using small BERT."
      ],
      "metadata": {
        "id": "i2fv8bOaoXtW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# install required libraries\n",
        "# dependency of the preprocessing for BERT inputs\n",
        "print('installing tensorflow-text library...')\n",
        "!pip install -q -U \"tensorflow-text==2.8.*\"\n",
        "print('tensorflow-text library installation complete...')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ShegztWQKO3X",
        "outputId": "38a7f190-7733-47a9-945d-2d6640928425"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "installing tensorflow-text library...\n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 42.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 498.0 MB 12 kB/s \n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 68.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.4 MB 63.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 462 kB 67.7 MB/s \n",
            "\u001b[?25htensorflow-text library installation complete...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u-fY0v8Onsb9"
      },
      "outputs": [],
      "source": [
        "# import libraries\n",
        "from google.colab import drive\n",
        "import os\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "\n",
        "from sklearn import preprocessing\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# mount google drive and connect working directory\n",
        "drive.mount('/content/gdrive', force_remount = True)\n",
        "print (sorted(os.listdir('gdrive/My Drive/Colab Helper/ICData/Dataset')))\n",
        "os.chdir('gdrive/My Drive/Colab Helper/ICData/Dataset') # Connect to the directory"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uL4CPprgorGN",
        "outputId": "f783b536-d644-41bb-9d22-891197addaa8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n",
            "['data_file.csv', 'glove.6B.100d.txt', 'glove.6B.50d.txt', 'model', 'model.png', 'saved_model', 'test_.csv', 'train_.csv', 'valid_.csv']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# read train file and print it's shape\n",
        "traindf = pd.read_csv('train_.csv', usecols=['text','intent'])\n",
        "testdf = pd.read_csv('test_.csv', usecols=['text','intent'])\n",
        "validdf = pd.read_csv('valid_.csv', usecols=['text','intent'])\n",
        "\n",
        "print('Train Shape: ', traindf.shape)\n",
        "print('Test Shape: ', testdf.shape)\n",
        "print('Valid Shape: ', validdf.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IbRPziQBosdj",
        "outputId": "cd7c4680-5f55-4d2f-c308-7900bc05cbd8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train Shape:  (9951, 2)\n",
            "Test Shape:  (1280, 2)\n",
            "Valid Shape:  (2985, 2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Observation:\n",
        "\n",
        "1.   No NULL value present."
      ],
      "metadata": {
        "id": "M8tlwyUvowd-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# NULL value presence check\n",
        "print('NULL values: ', traindf['text'].isnull().sum())"
      ],
      "metadata": {
        "id": "2y0KcCuiovtk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b90bf32b-2384-4ee0-9c35-14d98073cac3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "NULL values:  0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Observation:\n",
        "\n",
        "1.   Texts may not require a lot of text cleaning."
      ],
      "metadata": {
        "id": "4aNponEIo1m6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# display first 10 rows\n",
        "print('First 10 rows:\\n', traindf.head(10))"
      ],
      "metadata": {
        "id": "C49vJGUMo1OD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ecd0a93-a90f-4e5e-bc90-4b7448010b72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "First 10 rows:\n",
            "                                                 text              intent\n",
            "0                          find under night in-birth  SearchCreativeWork\n",
            "1         book a spot in town in the same area as me      BookRestaurant\n",
            "2                                         fimd glory  SearchCreativeWork\n",
            "3       i need a restaurant on july 18th in rose bud      BookRestaurant\n",
            "4  book a spot for 4 in oklahoma at south street ...      BookRestaurant\n",
            "5  i need to know what the weather will be like i...          GetWeather\n",
            "6  book a bistro in new zealand in 119 and a half...      BookRestaurant\n",
            "7           need a table for nine in east prairie la      BookRestaurant\n",
            "8  book a table for mindy and angelita at a resta...      BookRestaurant\n",
            "9  find me a table for two at seven somewhere in ...      BookRestaurant\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Process Inputs**"
      ],
      "metadata": {
        "id": "SbFO_bqsdnyu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# label encode\n",
        "label_encoder = preprocessing.LabelEncoder()\n",
        "label_encoder.fit(traindf['intent']) # fit label encoder\n",
        "print('Label Encoder Classes: ', label_encoder.classes_)\n",
        "\n",
        "# transform labels\n",
        "traindf['intent'] = label_encoder.transform(traindf['intent'])\n",
        "testdf['intent'] = label_encoder.transform(testdf['intent'])\n",
        "validdf['intent'] = label_encoder.transform(validdf['intent'])"
      ],
      "metadata": {
        "id": "MC_W0lhd4EcK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "44795a15-63fa-4987-e92d-9f32cd07c278"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label Encoder Classes:  ['AddToPlaylist' 'BookRestaurant' 'GetWeather' 'PlayMusic' 'RateBook'\n",
            " 'SearchCreativeWork' 'SearchScreeningEvent']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# convert model input data to arrays\n",
        "x_train = np.array([np.array(val) for val in traindf['text']])\n",
        "x_val = np.array([np.array(val) for val in validdf['text']])\n",
        "\n",
        "y_train = np.array([np.array(val) for val in traindf['intent']])\n",
        "y_val = np.array([np.array(val) for val in validdf['intent']])"
      ],
      "metadata": {
        "id": "3agWaghCEeCp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Define Model and Preprocessor**"
      ],
      "metadata": {
        "id": "-4a1icJAo_X0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# get model and it's preprocessor URL\n",
        "tfhub_handle_encoder = \"https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1\" # model\n",
        "tfhub_handle_preprocess = \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\" # preprocesser\n",
        "\n",
        "print('BERT model selected           : {}'.format(tfhub_handle_encoder))\n",
        "print('Preprocess model auto-selected: {}'.format(tfhub_handle_preprocess))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jOCI3S0eI65g",
        "outputId": "4e45b768-37b2-4b92-efe1-341226efa6e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BERT model selected           : https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1\n",
            "Preprocess model auto-selected: https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model Training**"
      ],
      "metadata": {
        "id": "DjFmAMQxbJc0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# model definition\n",
        "# bert layers\n",
        "preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing') # preprocesser for bert\n",
        "encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder') # bert model\n",
        "\n",
        "# using bert layers on input\n",
        "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text_input') # input\n",
        "encoder_inputs = preprocessing_layer(text_input) \n",
        "outputs = encoder(encoder_inputs)\n",
        "\n",
        "# neural network layers\n",
        "net = outputs['pooled_output']\n",
        "net = tf.keras.layers.Dropout(0.2)(net)\n",
        "net = tf.keras.layers.Dense(7, activation='softmax', name='classifier')(net)\n",
        "\n",
        "# using inputs and outputs to construct a final model\n",
        "model = tf.keras.Model(inputs=text_input, outputs=net)\n",
        "\n",
        "# model name and summary\n",
        "model._name = 'SmallBERTuncased'\n",
        "model.summary()"
      ],
      "metadata": {
        "id": "1Mj_tyn-Jf3t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eafb5144-5689-4d92-f34b-3c1a7b310b42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"SmallBERTuncased\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " text_input (InputLayer)        [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " preprocessing (KerasLayer)     {'input_mask': (Non  0           ['text_input[0][0]']             \n",
            "                                e, 128),                                                          \n",
            "                                 'input_type_ids':                                                \n",
            "                                (None, 128),                                                      \n",
            "                                 'input_word_ids':                                                \n",
            "                                (None, 128)}                                                      \n",
            "                                                                                                  \n",
            " BERT_encoder (KerasLayer)      {'pooled_output': (  4385921     ['preprocessing[0][0]',          \n",
            "                                None, 128),                       'preprocessing[0][1]',          \n",
            "                                 'encoder_outputs':               'preprocessing[0][2]']          \n",
            "                                 [(None, 128, 128),                                               \n",
            "                                 (None, 128, 128)],                                               \n",
            "                                 'sequence_output':                                               \n",
            "                                 (None, 128, 128),                                                \n",
            "                                 'default': (None,                                                \n",
            "                                128)}                                                             \n",
            "                                                                                                  \n",
            " dropout (Dropout)              (None, 128)          0           ['BERT_encoder[0][3]']           \n",
            "                                                                                                  \n",
            " classifier (Dense)             (None, 7)            903         ['dropout[0][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 4,386,824\n",
            "Trainable params: 4,386,823\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# model compile\n",
        "model.compile(loss='sparse_categorical_crossentropy',\n",
        "              optimizer='adam',\n",
        "              metrics='accuracy')\n",
        "\n",
        "# model training\n",
        "epochs = 5\n",
        "%time history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=epochs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eTcA1h9GwUzr",
        "outputId": "9f9a18e9-2119-4160-85b7-86d4b07f30bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "311/311 [==============================] - 37s 104ms/step - loss: 0.2069 - accuracy: 0.9375 - val_loss: 0.0826 - val_accuracy: 0.9762\n",
            "Epoch 2/5\n",
            "311/311 [==============================] - 29s 93ms/step - loss: 0.0871 - accuracy: 0.9758 - val_loss: 0.0841 - val_accuracy: 0.9782\n",
            "Epoch 3/5\n",
            "311/311 [==============================] - 28s 91ms/step - loss: 0.0611 - accuracy: 0.9845 - val_loss: 0.1015 - val_accuracy: 0.9725\n",
            "Epoch 4/5\n",
            "311/311 [==============================] - 28s 91ms/step - loss: 0.0628 - accuracy: 0.9844 - val_loss: 0.1001 - val_accuracy: 0.9735\n",
            "Epoch 5/5\n",
            "311/311 [==============================] - 31s 98ms/step - loss: 0.0437 - accuracy: 0.9885 - val_loss: 0.1214 - val_accuracy: 0.9712\n",
            "CPU times: user 3min 6s, sys: 34.1 s, total: 3min 40s\n",
            "Wall time: 2min 33s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# plot the graphs\n",
        "history_dict = history.history # dictionary of result values\n",
        "epochs = range(1, len(history_dict['accuracy']) + 1) # number of epochs\n",
        "\n",
        "# plot for loss\n",
        "plt.plot(epochs, history_dict['loss'], 'bo', label='Training loss')\n",
        "plt.plot(epochs, history_dict['val_loss'], 'b', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "# plot for accuracy\n",
        "plt.plot(epochs, history_dict['accuracy'], 'bo', label='Training acc')\n",
        "plt.plot(epochs, history_dict['val_accuracy'], 'b', label='Validation acc')\n",
        "plt.title('Training and validation accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend(loc='lower right')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 573
        },
        "id": "VA8DK2ENzwND",
        "outputId": "750bdf8e-b0b1-49b3-9601-128588492d0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwV1bnv/8+XQRBBVMAhDIIRB1TGBlQcUKNiNOCAUcJViUdRE2PUJIaEk2hMPL8kehMvP/WcEIdoDgaMHr0YNTiBaDSGhhAUxAQNaDsiyhRQpuf+UdWw2ezu3g1dvbub7/v16ldXrVpV9exq2M+utWqvpYjAzMwsX7NSB2BmZg2TE4SZmRXkBGFmZgU5QZiZWUFOEGZmVpAThJmZFeQEYfVC0hOSLqrruqUkabGkL2Rw3JB0YLr8X5J+UEzd7TjPaElPbm+c1Rx3qKSKuj6u1b8WpQ7AGi5Jq3NW2wCfARvT9csiYlKxx4qI07Ko29RFxOV1cRxJ3YF/Ai0jYkN67ElA0X9D2/k4QViVIqJt5bKkxcAlEfF0fj1JLSrfdMys6XATk9VaZROCpO9Keh+4R9Kekv4gaamkT9LlLjn7zJB0Sbo8RtILkm5J6/5T0mnbWbeHpJmSVkl6WtLtkv67iriLifHHkv6UHu9JSR1ztl8gaYmkZZLGV3N9Bkt6X1LznLKzJM1LlwdJeknScknvSbpN0i5VHOs3kn6Ss/6ddJ93JV2cV/d0SX+VtFLS25JuyNk8M/29XNJqSUdVXtuc/Y+WNEvSivT30cVem+pIOjTdf7mk+ZKG52z7oqQF6THfkfTttLxj+vdZLuljSc9L8vtVPfMFt+21L7AXsD8wluTf0j3pejdgLXBbNfsPBl4HOgI/B+6SpO2oez/wF6ADcANwQTXnLCbGrwBfBfYGdgEq37B6Af+ZHv9z6fm6UEBEvAz8Czgx77j3p8sbgWvS13MUcBLwtWriJo1hWBrPyUBPIL//41/AhcAewOnAFZLOTLcdl/7eIyLaRsRLecfeC3gMmJC+tl8Aj0nqkPcatrk2NcTcEngUeDLd7xvAJEkHp1XuImmubAccDjybln8LqAA6AfsA3wc8LlA9c4Kw7bUJuD4iPouItRGxLCIeiog1EbEKuAk4vpr9l0TEryNiI3AvsB/JG0HRdSV1AwYCP4yIdRHxAjC1qhMWGeM9EfH3iFgLPAD0TctHAn+IiJkR8Rnwg/QaVOV3wCgASe2AL6ZlRMTsiPhzRGyIiMXArwrEUciX0/hejYh/kSTE3Nc3IyJeiYhNETEvPV8xx4UkofwjIn6bxvU7YCHwpZw6VV2b6hwJtAV+mv6NngX+QHptgPVAL0m7R8QnETEnp3w/YP+IWB8Rz4cHjqt3ThC2vZZGxKeVK5LaSPpV2gSzkqRJY4/cZpY871cuRMSadLFtLet+Dvg4pwzg7aoCLjLG93OW1+TE9LncY6dv0MuqOhfJ3cLZkloBZwNzImJJGsdBafPJ+2kc/0FyN1GTrWIAluS9vsGSpqdNaCuAy4s8buWxl+SVLQE656xXdW1qjDkicpNp7nHPIUmeSyQ9J+motPxmYBHwpKQ3JY0r7mVYXXKCsO2V/2nuW8DBwOCI2J0tTRpVNRvVhfeAvSS1ySnrWk39HYnxvdxjp+fsUFXliFhA8kZ4Gls3L0HSVLUQ6JnG8f3tiYGkmSzX/SR3UF0joj3wXznHrenT97skTW+5ugHvFBFXTcftmtd/sPm4ETErIkaQND89QnJnQkSsiohvRcQBwHDgWkkn7WAsVktOEFZX2pG06S9P27Ovz/qE6SfycuAGSbuknz6/VM0uOxLjg8AZko5JO5RvpOb/P/cD3yRJRL/Pi2MlsFrSIcAVRcbwADBGUq80QeXH347kjupTSYNIElOlpSRNYgdUcezHgYMkfUVSC0nnAb1ImoN2xMskdxvXSWopaSjJ32hy+jcbLal9RKwnuSabACSdIenAtK9pBUm/TXVNepYBJwirK7cCuwIfAX8G/lhP5x1N0tG7DPgJMIXk+xqFbHeMETEf+DrJm/57wCcknajVqewDeDYiPsop/zbJm/cq4NdpzMXE8ET6Gp4laX55Nq/K14AbJa0Cfkj6aTzddw1Jn8uf0ieDjsw79jLgDJK7rGXAdcAZeXHXWkSsI0kIp5Fc9zuACyNiYVrlAmBx2tR2OcnfE5JO+KeB1cBLwB0RMX1HYrHak/t9rCmRNAVYGBGZ38GYNXW+g7BGTdJASZ+X1Cx9DHQESVu2me0gf5PaGrt9gf8h6TCuAK6IiL+WNiSzpsFNTGZmVpCbmMzMrKAm08TUsWPH6N69e6nDMDNrVGbPnv1RRHQqtK3JJIju3btTXl5e6jDMzBoVSfnfoN/MTUxmZlaQE4SZmRXkBGFmZgU1mT4IM6t/69evp6Kigk8//bTmylZSrVu3pkuXLrRs2bLofZwgzGy7VVRU0K5dO7p3707V8z1ZqUUEy5Yto6Kigh49ehS9307fxDRpEnTvDs2aJb8neQp3s6J9+umndOjQwcmhgZNEhw4dan2nt1PfQUyaBGPHwpp0upklS5J1gNGjq97PzLZwcmgctufvtFPfQYwfvyU5VFqzJik3M9vZ7dQJ4q23alduZg3LsmXL6Nu3L3379mXfffelc+fOm9fXrVtX7b7l5eVcddVVNZ7j6KOPrpNYZ8yYwRlnnFEnx6ovO3WC6JY/YWMN5Wa2Y+q6z69Dhw7MnTuXuXPncvnll3PNNddsXt9ll13YsGFDlfuWlZUxYcKEGs/x4osv7liQjdhOnSBuugnatNm6rE2bpNzM6lZln9+SJRCxpc+vrh8MGTNmDJdffjmDBw/muuuu4y9/+QtHHXUU/fr14+ijj+b1118Htv5Ef8MNN3DxxRczdOhQDjjggK0SR9u2bTfXHzp0KCNHjuSQQw5h9OjRVI6G/fjjj3PIIYcwYMAArrrqqhrvFD7++GPOPPNMevfuzZFHHsm8efMAeO655zbfAfXr149Vq1bx3nvvcdxxx9G3b18OP/xwnn/++bq9YNXYqTupKzuix49PmpW6dUuSgzuozepedX1+df1/rqKighdffJHmzZuzcuVKnn/+eVq0aMHTTz/N97//fR566KFt9lm4cCHTp09n1apVHHzwwVxxxRXbfGfgr3/9K/Pnz+dzn/scQ4YM4U9/+hNlZWVcdtllzJw5kx49ejBq1Kga47v++uvp168fjzzyCM8++ywXXnghc+fO5ZZbbuH2229nyJAhrF69mtatWzNx4kROPfVUxo8fz8aNG1mTfxEztFMnCEj+YTohmGWvPvv8zj33XJo3bw7AihUruOiii/jHP/6BJNavX19wn9NPP51WrVrRqlUr9t57bz744AO6dOmyVZ1BgwZtLuvbty+LFy+mbdu2HHDAAZu/XzBq1CgmTpxYbXwvvPDC5iR14oknsmzZMlauXMmQIUO49tprGT16NGeffTZdunRh4MCBXHzxxaxfv54zzzyTvn377tC1qY1Mm5gkDZP0uqRFksYV2H6tpAWS5kl6RtL+OdsukvSP9OeiLOM0s+zVZ5/fbrvttnn5Bz/4ASeccAKvvvoqjz76aJXfBWjVqtXm5ebNmxfsvyimzo4YN24cd955J2vXrmXIkCEsXLiQ4447jpkzZ9K5c2fGjBnDfffdV6fnrE5mCUJSc+B24DSgFzBKUq+8an8FyiKiN/Ag8PN0372A64HBwCDgekl7ZhWrmWWvVH1+K1asoHPnzgD85je/qfPjH3zwwbz55pssXrwYgClTptS4z7HHHsuktPNlxowZdOzYkd1335033niDI444gu9+97sMHDiQhQsXsmTJEvbZZx8uvfRSLrnkEubMmVPnr6EqWd5BDAIWRcSbEbEOmEwyofxmETE9Iiob1P4MVN7PnQo8FREfR8QnwFPAsAxjNbOMjR4NEyfC/vuDlPyeODH7Jt7rrruO733ve/Tr16/OP/ED7Lrrrtxxxx0MGzaMAQMG0K5dO9q3b1/tPjfccAOzZ8+md+/ejBs3jnvvvReAW2+9lcMPP5zevXvTsmVLTjvtNGbMmEGfPn3o168fU6ZM4Zvf/Gadv4aqZDYntaSRwLCIuCRdvwAYHBFXVlH/NuD9iPiJpG8DrSPiJ+m2HwBrI+KWqs5XVlYWnjDIrH699tprHHrooaUOo+RWr15N27ZtiQi+/vWv07NnT6655ppSh7WNQn8vSbMjoqxQ/QbxmKuk/wWUATfXcr+xksollS9dujSb4MzMavDrX/+avn37cthhh7FixQouu+yyUodUJ7J8iukdoGvOepe0bCuSvgCMB46PiM9y9h2at++M/H0jYiIwEZI7iLoI2systq655poGecewo7K8g5gF9JTUQ9IuwPnA1NwKkvoBvwKGR8SHOZumAadI2jPtnD4lLTMzs3qS2R1ERGyQdCXJG3tz4O6ImC/pRqA8IqaSNCm1BX6fjjT4VkQMj4iPJf2YJMkA3BgRH2cVq5mZbSvTL8pFxOPA43llP8xZ/kI1+94N3J1ddGZmVp0G0UltZmYNjxOEmTVaJ5xwAtOmbd09eeutt3LFFVdUuc/QoUOpfCT+i1/8IsuXL9+mzg033MAtt1T5VD0AjzzyCAsWLNi8/sMf/pCnn366NuEX1JCGBXeCMLNGa9SoUUyePHmrssmTJxc1YB4ko7Duscce23Xu/ARx44038oUvVNlq3ig5QZhZozVy5Egee+yxzZMDLV68mHfffZdjjz2WK664grKyMg477DCuv/76gvt3796djz76CICbbrqJgw46iGOOOWbzkOCQfMdh4MCB9OnTh3POOYc1a9bw4osvMnXqVL7zne/Qt29f3njjDcaMGcODDz4IwDPPPEO/fv044ogjuPjii/nss882n+/666+nf//+HHHEESxcuLDa11fqYcF3+tFczaxuXH01zJ1bt8fs2xduvbXq7XvttReDBg3iiSeeYMSIEUyePJkvf/nLSOKmm25ir732YuPGjZx00knMmzeP3r17FzzO7NmzmTx5MnPnzmXDhg3079+fAQMGAHD22Wdz6aWXAvDv//7v3HXXXXzjG99g+PDhnHHGGYwcOXKrY3366aeMGTOGZ555hoMOOogLL7yQ//zP/+Tqq68GoGPHjsyZM4c77riDW265hTvvvLPK11fqYcF9B2FmjVpuM1Nu89IDDzxA//796devH/Pnz9+qOSjf888/z1lnnUWbNm3YfffdGT58+OZtr776KsceeyxHHHEEkyZNYv78+dXG8/rrr9OjRw8OOuggAC666CJmzpy5efvZZ58NwIABAzYP8FeVF154gQsuuAAoPCz4hAkTWL58OS1atGDgwIHcc8893HDDDbzyyiu0a9eu2mMXw3cQZlYnqvukn6URI0ZwzTXXMGfOHNasWcOAAQP45z//yS233MKsWbPYc889GTNmTJXDfNdkzJgxPPLII/Tp04ff/OY3zJgxY4firRwyfEeGCx83bhynn346jz/+OEOGDGHatGmbhwV/7LHHGDNmDNdeey0XXnjhDsXqOwgza9Tatm3LCSecwMUXX7z57mHlypXstttutG/fng8++IAnnnii2mMcd9xxPPLII6xdu5ZVq1bx6KOPbt62atUq9ttvP9avX795iG6Adu3asWrVqm2OdfDBB7N48WIWLVoEwG9/+1uOP/747XptpR4W3HcQZtbojRo1irPOOmtzU1Pl8NiHHHIIXbt2ZciQIdXu379/f8477zz69OnD3nvvzcCBAzdv+/GPf8zgwYPp1KkTgwcP3pwUzj//fC699FImTJiwuXMaoHXr1txzzz2ce+65bNiwgYEDB3L55Zdv1+uqnCu7d+/etGnTZqthwadPn06zZs047LDDOO2005g8eTI333wzLVu2pG3btnUysVBmw33XNw/3bVb/PNx349Ioh/s2M7OGxwnCzMwKcoIwsx3SVJqpm7rt+Ts5QZjZdmvdujXLli1zkmjgIoJly5bRunXrWu3np5jMbLt16dKFiooKPOVvw9e6dWu6dOlSq32cIMxsu7Vs2ZIePXqUOgzLSKZNTJKGSXpd0iJJ4wpsP07SHEkbJI3M2/ZzSfMlvSZpgtIp58zMrH5kliAkNQduB04DegGjJPXKq/YWMAa4P2/fo4EhQG/gcGAgsH1fRTQzs+2SZRPTIGBRRLwJIGkyMALYPGJWRCxOt23K2zeA1sAugICWwAcZxmpmZnmybGLqDLyds16RltUoIl4CpgPvpT/TIuK1/HqSxkoql1TuTjIzs7rVIB9zlXQgcCjQhSSpnCjp2Px6ETExIsoioqxTp071HaaZWZOWZYJ4B+ias94lLSvGWcCfI2J1RKwGngCOquP4zMysGlkmiFlAT0k9JO0CnA9MLXLft4DjJbWQ1JKkg3qbJiYzM8tOZgkiIjYAVwLTSN7cH4iI+ZJulDQcQNJASRXAucCvJFVO1fQg8AbwCvA34G8R8eg2JzEzs8x4uG8zs52Yh/s2M7Nac4IwM7OCnCDMzKwgJwgzMyvICcLMzApygjAzs4KcIMzMrCAnCDMzK8gJwszMCnKCMDOzgpwgzMysICcIMzMryAnCzMwKcoIwM7OCnCDMzKygTBOEpGGSXpe0SNK4AtuPkzRH0gZJI/O2dZP0pKTXJC2Q1D3LWM3MbGuZJQhJzYHbgdOAXsAoSb3yqr0FjAHuL3CI+4CbI+JQYBDwYVaxmpnZtlpkeOxBwKKIeBNA0mRgBLCgskJELE63bcrdMU0kLSLiqbTe6gzjNDOzArJsYuoMvJ2zXpGWFeMgYLmk/5H0V0k3p3ckW5E0VlK5pPKlS5fWQchmZlapoXZStwCOBb4NDAQOIGmK2kpETIyIsogo69SpU/1GaGbWxGWZIN4Buuasd0nLilEBzI2INyNiA/AI0L+O4zMzs2pkmSBmAT0l9ZC0C3A+MLUW++4hqfK24ERy+i7MzCx7mSWI9JP/lcA04DXggYiYL+lGScMBJA2UVAGcC/xK0vx0340kzUvPSHoFEPDrrGI1M7NtKSJKHUOdKCsri/Ly8lKHYWbWqEiaHRFlhbY11E5qMzMrMScIMzMryAnCzMwKcoIwM7OCnCDMzKwgJwgzMyvICcLMzApygjAzs4KcIMzMrCAnCDMzK8gJwszMCnKCMDOzgpwgzMysICcIMzMryAnCzMwKyjRBSBom6XVJiySNK7D9OElzJG2QNLLA9t0lVUi6Lcs4zcxsW5klCEnNgduB04BewChJvfKqvQWMAe6v4jA/BmZmFaOZmVUtyzuIQcCiiHgzItYBk4ERuRUiYnFEzAM25e8saQCwD/BkhjGamVkVskwQnYG3c9Yr0rIaSWoG/G+SeamrqzdWUrmk8qVLl253oGZmtq2G2kn9NeDxiKiorlJETIyIsogo69SpUz2FZma2c2iR4bHfAbrmrHdJy4pxFHCspK8BbYFdJK2OiG06us3MLBtZJohZQE9JPUgSw/nAV4rZMSJGVy5LGgOUOTmYmdWvzJqYImIDcCUwDXgNeCAi5ku6UdJwAEkDJVUA5wK/kjQ/q3jMzKx2FBGljqFOlJWVRXl5eanDMDNrVCTNjoiyQtsaaie1mZmVmBOEmZkV5ARhZmYFOUGYmVlBRSUISbul325G0kGShktqmW1oZmZWSsXeQcwEWkvqTDI20gXAb7IKyszMSq/YBKGIWAOcDdwREecCh2UXlpmZlVrRCULSUcBo4LG0rHk2IZmZWUNQbIK4Gvge8HD6begDgOnZhWVmZqVW1FhMEfEc8BxsHor7o4i4KsvAzMystIp9iun+dPrP3YBXgQWSvpNtaGZmVkrFNjH1ioiVwJnAE0APkieZzMysiSo2QbRMv/dwJjA1ItYDTWOUPzMzK6jYBPErYDGwGzBT0v7AyqyCMjOz0iu2k3oCMCGnaImkE7IJyczMGoJiO6nbS/qFpPL053+T3E3UtN8wSa9LWiRpmxnhJB0naY6kDZJG5pT3lfSSpPmS5kk6r1avysxsJxEBn3ySzbGLbWK6G1gFfDn9WQncU90OkpoDtwOnAb2AUZJ65VV7CxgD3J9Xvga4MCIOA4YBt0rao8hYzcyatAiYOxe+9z048EAYObLmfbZHsXNSfz4izslZ/5GkuTXsMwhYFBFvAkiaDIwAFlRWiIjF6bZNuTtGxN9zlt+V9CHQCVheZLxmZk3Oq6/ClCnwwAPw979D8+Zw0knwla9kc75iE8RaScdExAsAkoYAa2vYpzPwds56BTC4tgFKGgTsArxRYNtYYCxAt27dantoM7MG7/XXk6QwZQosWADNmsHQofCtb8FZZ0GnTtmdu9gEcTlwn6T26fonwEXZhLSFpP2A3wIXRcSm/O0RMRGYCMmc1FnHY2ZWH954I7lLmDIF/vY3kOCYY+C22+Ccc2DffesnjmKfYvob0EfS7un6SklXA/Oq2e0doGvOepe0rCjpuR4DxkfEn4vdz8ysMVqyZEtSmD07KTvySPjlL+Hcc6Fz5/qPqdg7CCBJDDmr1wK3VlN9FtBTUg+SxHA+UFRLmaRdgIeB+yLiwdrEaGbWWLzzDvz+90lS+HP6MbisDG6+OUkK++9f2vhqlSDyqLqNEbFB0pXANJKhwe9OR4K9ESiPiKmSBpIkgj2BL0n6Ufrk0peB44AOksakhxwTETV1jJuZNWjvvw8PPZQkhRdeSJ5I6tMH/uM/4Mtfhs9/vtQRbqGI7Wu6l/RWRDSYnuGysrIoLy8vdRhmZtv46KMtSeG552DTJujVC847L/k5+ODSxSZpdkSUFdpW7R2EpFUUHnNJwK51EJuZWZP0ySfw8MNJUnjmGdi4EQ46CMaPT+4UDj+81BHWrNoEERHt6isQM7PGbsUKmDo1SQpPPgnr10OPHvCd7yR3Cn36JE8kNRY70gdhZrbTW70aHn00SQp//CN89hl07QpXXZUkhbKyxpUUcjlBmJnV0po18PjjSVJ47DFYuxb22w8uuyxJCkcemXyhrbFzgjAzK8Knn8K0aUlSmDoV/vUv2Htv+OpXk6RwzDFNIynkcoIwM6vCunXw1FNJUvi//xdWroS99krGPjrvPDj+eGjRhN9Fm/BLMzOrvQ0b4Nlnk6Tw8MPJ00jt28PZZydJ4aSToGXLUkdZP5wgzGynt3EjzJyZJIWHHkq+t9CuHYwYkSSFk0+GVq1KHWX9c4Iws53Spk3wpz8lSeHBB+GDD6BNG/jSl5KkMGwY7LqTf9vLCcLMdhoR8PLLSVL4/e+TsZBat4bTT0++vHb66bBbjXNl7jycIMysSYtIRkd94IHkZ8kS2GWX5A7h5z9P7hja+SvBBTlBmFmTEwHz5m2Zfe2NN5KnjU4+GX70o6RvYQ9PYlwjJwgzazIWLNgy+9rrrydTcp54YjJ385lnQocOpY6wcXGCMLNG7e9/33Kn8OqrybAWxx8PV1+dzL6W5ZScTZ0ThJk1Ov/855Y7hbnpLDFDhsCECTByZDLshe04JwgzaxTefnvLlJyzZiVlgwfDL36RJIWuXavf32ov0wQhaRjwf0hmlLszIn6at/04kmlLewPn504vKuki4N/T1Z9ExL1ZxmpWl9atg5deSsbuefppWL48eXKmZcvkd30vV7WtZcuknb6hevfd5DsKU6bAiy8mZf37w89+ljyW2r17ScNr8jJLEJKaA7cDJwMVwCxJUyNiQU61t4AxwLfz9t0LuB4oI5mwaHa67ydZxWu2o954I0kI06YlQzWsXp28+R55JAwYkMwNsG7dlp/165MB3yqXc8vzlzduzC7uZs1Kk7SqWm7RIpmfecoUeP755ImkI46An/wkSQo9e2Z3LWxrWd5BDAIWRcSbAJImAyOAzQkiIhan2zbl7Xsq8FREfJxufwoYBvwuw3jNamXVKpg+fUtSeOONpLx7dxg9Gk49NXmCpn37HT/Xpk01J5H6Wl6zprj6Gzbs2Gs+5BD44Q+TbzUfeuiOX0OrvSwTRGfg7Zz1CmDwDuzbOb+SpLHAWIBu3RrM9NjWRG3alHSIViaEF19M3gzbtIETToBvfjNJCj171v0EMc2aJWMBNabxgCqTWm2T0Lp1cOCByZScjXWinaaiUXdSR8REYCJAWVlZobmzzXbIBx8kU0dOm5YM+/zhh0l5nz5wzTVJQhgypHG9cdeXxpjUbGtZJoh3gNznCrqkZcXuOzRv3xl1EpVZNdatSwZwq7xLqHyEsmNHOOWUJCGccgrsu29p4zSrD1kmiFlAT0k9SN7wzwe+UuS+04D/kLRnun4K8L26D9F2dhHwj39suUuYPj3pOG7RAo4+Gm66KUkK/fo1vdnCzGqSWYKIiA2SriR5s28O3B0R8yXdCJRHxFRJA4GHgT2BL0n6UUQcFhEfS/oxSZIBuLGyw9psR61cCc88s+UuYfHipPzzn4eLLkoSwgkneAA3M0U0jab7srKyKC8vL3UY1gBt2pSM5lmZEF56KXlstG3b5CmjU09Nfj7/+VJHalb/JM2OiLJC2xp1J7VZVd59d+vO5WXLkvL+/eG665KEcNRRyXP3ZlaYE4Q1CZ9+Ci+8sOUu4ZVXkvJ99oEvfjFJCCefDHvvXdo4zRoTJwhrlCKS4ZwrE8KMGbB2bXJHcMwx8NOfJkmhd293LpttLycIazSWL9+6c/mtt5Lygw6CSy5JEsLQoZ4y0qyuOEFYg7VxYzJqZ2VfwssvJ2W77w4nnQTf/37ynYQePUodqVnT5ARhDUpFxZY7hKefhk8+SYZbKCtLZgU79dRkiOeWLUsdqVnT5wRhtTJpEowfnzTvdOuWfJFs9OjtP97atTBz5paksCAdynG//ZJ5g089Fb7wheSbzGZWv5wgrGiTJsHYsclongBLliTrUHySiEiSQGVCmDkzeQKpVSs49lj46leTpOCB2sxKz1+Us6J1754khXz777/l28iFfPxx0lw0bVrSn1BRkZQfeuiWsY2OPz4ZFdXM6pe/KGd1ovKpoZrKN2yAv/xly13CrFnJt5n32CNpLqpMCh6h3axhc4KwonXrVvgOolu3JEnkdi6vWJF8/2DQIPjBD5KkMHBgMgiemTUO/u9qRbvppq37ICB5w1+/PmlmAujSJa6tAV0AAAxcSURBVJlA/tRTk0dR99qrNLGa2Y5zgmiENm7cevatYn4++6z2+xQ6xv77J1NrrluXxCIl8wV/+9tJUjj0UHcumzUVThCpiG3feOvqTbWuj7EpfwbvOtKq1ZaJ43N/css7dYLOnZMhLE49NXnyaNdds4nHzEprp08QH36YtKGvW5ckibombZl2sdCbb+5P27ZVbytm/x3Zt3lzf/I3s61lmiAkDQP+D8mEQXdGxE/ztrcC7gMGAMuA8yJisaSWwJ1A/zTG+yLi/8sixrZt4eqr6+6NttAbr5lZY5RZgpDUHLgdOBmoAGZJmhoRC3Kq/RvwSUQcKOl84GfAecC5QKuIOEJSG2CBpN9FxOK6jrNNm2TkTzMz21qWAyEPAhZFxJsRsQ6YDIzIqzMCuDddfhA4SZKAAHaT1ALYFVgHrMwwVjMzy5NlgugMvJ2zXpGWFawTERuAFUAHkmTxL+A94C3glkJzUksaK6lcUvnSpUvr/hWYme3EGupUKoOAjcDngB7AtyQdkF8pIiZGRFlElHXq1Km+YzQza9KyTBDvAF1z1rukZQXrpM1J7Uk6q78C/DEi1kfEh8CfgIJjhZiZWTayTBCzgJ6SekjaBTgfmJpXZypwUbo8Eng2ktED3wJOBJC0G3AksDDDWM3MLE9mCSLtU7gSmAa8BjwQEfMl3ShpeFrtLqCDpEXAtcC4tPx2oK2k+SSJ5p6ImJdVrGZmti0P921mthOrbrjvhtpJbWZmJeYEYWZmBTlBmJlZQU4QZmZWkBOEmZkV5ARhZmYFOUGYmVlBThBmZlaQE4SZmRXkBGFmZgU5QZiZWUFOEGZmVpAThJk1GJMmQffu0KxZ8nvSpFJHtHNrUeoAzMwgSQZjx8KaNcn6kiXJOsDo0aWLa2fmOwizDPkTcfHGj9+SHCqtWZOUW2lkmiAkDZP0uqRFksYV2N5K0pR0+8uSuuds6y3pJUnzJb0iqXWWsZrVtcpPxEuWQMSWT8ROEoW99Vbtyi17mSUISc1JZoY7DegFjJLUK6/avwGfRMSBwC+Bn6X7tgD+G7g8Ig4DhgLrs4rVLAv+RFw73brVrtyyl+UdxCBgUUS8GRHrgMnAiLw6I4B70+UHgZMkCTgFmBcRfwOIiGURsTHDWM3qnD8R185NN0GbNluXtWmTlFtpZJkgOgNv56xXpGUF66RzWK8AOgAHASFpmqQ5kq4rdAJJYyWVSypfunRpnb8Asx3hT8S1M3o0TJwI++8PUvJ74kR3UJdSQ+2kbgEcA4xOf58l6aT8ShExMSLKIqKsU6dO9R2jWbX8ibj2Ro+GxYth06bkt5NDaWWZIN4Buuasd0nLCtZJ+x3aA8tI7jZmRsRHEbEGeBzon2GsZnXOn4itscsyQcwCekrqIWkX4Hxgal6dqcBF6fJI4NmICGAacISkNmniOB5YkGGsZpnwJ2JrzDL7olxEbJB0JcmbfXPg7oiYL+lGoDwipgJ3Ab+VtAj4mCSJEBGfSPoFSZIJ4PGIeCyrWM3MbFtKPrA3fmVlZVFeXl7qMMzMGhVJsyOirNC2htpJbWZmJeYEYWZmBTlBmJlZQU4QZmZWkBOEmZkV5ARhZmYFOUGYmVlBThBmZlaQE4SZmRXkBGFmZgU5QZiZWUFOEGZmjdSkSdC9OzRrlvyu6/nOMxvN1czMsjNpEowdu2Xe8yVLknWou2HlfQdhZtYIjR+/JTlUWrMmKa8rThBmZo3QW2/Vrnx7ZJogJA2T9LqkRZLGFdjeStKUdPvLkrrnbe8mabWkb2cZp5lZY9OtW+3Kt0dmCUJSc+B24DSgFzBKUq+8av8GfBIRBwK/BH6Wt/0XwBNZxWhm1ljddBO0abN1WZs2SXldyfIOYhCwKCLejIh1wGRgRF6dEcC96fKDwEmSBCDpTOCfwPwMYzQza5RGj4aJE2H//UFKfk+cWLfznmf5FFNn4O2c9QpgcFV10jmsVwAdJH0KfBc4GaiyeUnSWGAsQLe6vK8yM2sERo+u24SQr6F2Ut8A/DIiVldXKSImRkRZRJR16tSpfiIzM9tJZHkH8Q7QNWe9S1pWqE6FpBZAe2AZyZ3GSEk/B/YANkn6NCJuyzBeMzPLkWWCmAX0lNSDJBGcD3wlr85U4CLgJWAk8GxEBHBsZQVJNwCrnRzMzOpXZgki7VO4EpgGNAfujoj5km4EyiNiKnAX8FtJi4CPSZKImZk1AEo+sDd+ZWVlUV5eXuowzMwaFUmzI6Ks4LamkiAkLQWW7MAhOgIf1VE4dclx1Y7jqh3HVTtNMa79I6LgUz5NJkHsKEnlVWXRUnJcteO4asdx1c7OFldDfczVzMxKzAnCzMwKcoLYYmKpA6iC46odx1U7jqt2dqq43AdhZmYF+Q7CzMwKcoIwM7OCdqoEIeluSR9KerWK7ZI0IZ3AaJ6k/g0krqGSVkiam/78sJ7i6ippuqQFkuZL+maBOvV+zYqMq96vmaTWkv4i6W9pXD8qUKfaSbJKGNcYSUtzrtclWceVc+7mkv4q6Q8FttX79SoiplJeq8WSXknPu803g+v8/2NE7DQ/wHFAf+DVKrZ/kWSCIgFHAi83kLiGAn8owfXaD+ifLrcD/g70KvU1KzKuer9m6TVomy63BF4Gjsyr8zXgv9Ll84EpDSSuMcBt9f1vLD33tcD9hf5epbheRcRUymu1GOhYzfY6/f+4U91BRMRMkjGfqjICuC8Sfwb2kLRfA4irJCLivYiYky6vAl4jmcMjV71fsyLjqnfpNagcor5l+pP/FEiVk2SVOK6SkNQFOB24s4oq9X69ioipIavT/487VYIoQqFJjkr+xpM6Km0ieELSYfV98vTWvh/Jp89cJb1m1cQFJbhmadPEXOBD4KmIqPJ6RcQGYAXQoQHEBXBO2izxoKSuBbZn4VbgOmBTFdtLcb1qiglKc60gSexPSpqtZMK0fHX6/9EJonGYQzJeSh/g/wceqc+TS2oLPARcHREr6/Pc1akhrpJcs4jYGBF9SeY/GSTp8Po4b02KiOtRoHtE9AaeYsun9sxIOgP4MCJmZ32uYhUZU71fqxzHRER/4DTg65KOy/JkThBbK2aSo3oXESsrmwgi4nGgpaSO9XFuSS1J3oQnRcT/FKhSkmtWU1ylvGbpOZcD04FheZs2Xy9tPUlWSeOKiGUR8Vm6eicwoB7CGQIMl7SYZM76EyX9d16d+r5eNcZUomtVee530t8fAg8Dg/Kq1On/RyeIrU0FLkyfBDgSWBER75U6KEn7Vra7ShpE8nfL/E0lPeddwGsR8YsqqtX7NSsmrlJcM0mdJO2RLu9KMqf6wrxqlZNkwdaTZJU0rrx26uEk/TqZiojvRUSXiOhO0gH9bET8r7xq9Xq9iompFNcqPe9uktpVLgOnAPlPPtbp/8csZ5RrcCT9juTplo6SKoDrSTrsiIj/Ah4neQpgEbAG+GoDiWskcIWkDcBa4Pys31RSQ4ALgFfS9muA7wPdcmIrxTUrJq5SXLP9gHslNSdJSA9ExB9U+kmyionrKknDgQ1pXGPqIa6CGsD1qimmUl2rfYCH0889LYD7I+KPki6HbP4/eqgNMzMryE1MZmZWkBOEmZkV5ARhZmYFOUGYmVlBThBmZlaQE4RZDSRtzBm5c66kcXV47O6qYhRfs1Lbqb4HYbad1qbDVJjtVHwHYbad0rH5f56Oz/8XSQem5d0lPZsO5vaMpG5p+T6SHk4HEPybpKPTQzWX9GslczU8mX7bGUlXKZnzYp6kySV6mbYTc4Iwq9mueU1M5+VsWxERRwC3kYwCCsnggPemg7lNAiak5ROA59IBBPsD89PynsDtEXEYsBw4Jy0fB/RLj3N5Vi/OrCr+JrVZDSStjoi2BcoXAydGxJvp4IHvR0QHSR8B+0XE+rT8vYjoKGkp0CVnoLfK4cqfioie6fp3gZYR8RNJfwRWk4xE+0jOnA5m9cJ3EGY7JqpYro3PcpY3sqVv8HTgdpK7jVnpaKZm9cYJwmzHnJfz+6V0+UW2DCo3Gng+XX4GuAI2T+DTvqqDSmoGdI2I6cB3SYa53uYuxixL/kRiVrNdc0aNBfhjRFQ+6rqnpHkkdwGj0rJvAPdI+g6wlC0jan4TmCjp30juFK4AqhqKuTnw32kSETAhncvBrN64D8JsO6V9EGUR8VGpYzHLgpuYzMysIN9BmJlZQb6DMDOzgpwgzMysICcIMzMryAnCzMwKcoIwM7OC/h+M5KEEkzIyrAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwV5Z3v8c+XZpNFiYCKNNJo3PDK2mLEKLhNQBwdjY4SYiTJjIo6meRextEsxmC4SUZn4nU0yZBRo4YMakwcjbgiLhNjpFXAFUWDyqIiyiY7/O4fVd19OFR3n4Y+fZru7/v1Oq9Ty1N1fqe6u379PE/VU4oIzMzM8rUrdQBmZtYyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCsIJJelDSBU1dtpQkLZJ0chH2G5I+m07/QtL3Cim7E58zQdIjOxunWX3k+yBaN0lrc2a7ABuBren8RRExvfmjajkkLQL+LiIea+L9BnBwRCxsqrKSKoC/AB0iYktTxGlWn/alDsCKKyK6VU/XdzKU1N4nHWsp/PvYMriJqY2SNFrSYkn/LOl94FZJn5H0B0nLJX2STpfnbPOEpL9LpydK+h9J16Vl/yJp7E6WHSDpKUlrJD0m6SZJv64j7kJivEbSH9P9PSKpV8768yW9I2mFpO/Uc3yOlvS+pLKcZWdKmp9Oj5D0J0krJS2TdKOkjnXs61eSfpgz/0/pNkslfS2v7DhJL0paLek9SVfnrH4qfV8paa2kY6qPbc72IyXNkbQqfR9Z6LFp5HHeW9Kt6Xf4RNK9OevOkDQ3/Q5vSRqTLt+uOU/S1dU/Z0kVaVPb1yW9CzyeLr87/TmsSn9HjsjZfg9J/5r+PFelv2N7SHpA0j/kfZ/5ks7M+q5WNyeItm0/YG+gP3Ahye/Dren8AcB64MZ6tj8aWAD0Av4FuFmSdqLsb4DngJ7A1cD59XxmITF+CfgqsA/QEZgMIGkg8PN0//unn1dOhoj4M/ApcGLefn+TTm8FvpV+n2OAk4BL6ombNIYxaTynAAcD+f0fnwJfAXoA44BJkv4mXXd8+t4jIrpFxJ/y9r038ABwQ/rd/g14QFLPvO+ww7HJ0NBxvoOkyfKIdF8/TWMYAdwO/FP6HY4HFtV1PDKMAg4HvpDOP0hynPYBXgBym0SvA4YDI0l+jy8HtgG3AV+uLiRpMNCX5NhYY0SEX23kRfKHenI6PRrYBHSup/wQ4JOc+SdImqgAJgILc9Z1AQLYrzFlSU4+W4AuOet/Dfy6wO+UFeN3c+YvAR5Kp68CZuSs65oeg5Pr2PcPgVvS6e4kJ+/+dZT9JvD7nPkAPptO/wr4YTp9C/DjnHKH5JbN2O/1wE/T6Yq0bPuc9ROB/0mnzweey9v+T8DEho5NY44z0IfkRPyZjHL/UR1vfb9/6fzV1T/nnO92YD0x9EjL7EWSwNYDgzPKdQY+IenXgSSR/Ky5/95aw8s1iLZteURsqJ6R1EXSf6RV9tUkTRo9cptZ8rxfPRER69LJbo0suz/wcc4ygPfqCrjAGN/PmV6XE9P+ufuOiE+BFXV9Fklt4SxJnYCzgBci4p00jkPSZpf30zj+L0ltoiHbxQC8k/f9jpY0O23aWQVcXOB+q/f9Tt6yd0j+e65W17HZTgPHuR/Jz+yTjE37AW8VGG+WmmMjqUzSj9NmqtXU1kR6pa/OWZ+V/k7fCXxZUjtgPEmNxxrJCaJty7+E7f8AhwJHR8Se1DZp1NVs1BSWAXtL6pKzrF895XclxmW5+04/s2ddhSPiVZIT7Fi2b16CpKnqdZL/UvcEvr0zMZDUoHL9BrgP6BcRewG/yNlvQ5ccLiVpEsp1ALCkgLjy1Xec3yP5mfXI2O494KA69vkpSe2x2n4ZZXK/45eAM0ia4fYiqWVUx/ARsKGez7oNmEDS9Lcu8prjrDBOEJarO0m1fWXanv39Yn9g+h95FXC1pI6SjgH+ukgx/hY4TdLn0w7lKTT8N/Ab4B9JTpB358WxGlgr6TBgUoEx3AVMlDQwTVD58Xcn+e98Q9qe/6WcdctJmnYOrGPfM4FDJH1JUntJ5wIDgT8UGFt+HJnHOSKWkfQN/CztzO4gqTqB3Ax8VdJJktpJ6pseH4C5wHlp+Urg7AJi2EhSy+tCUkurjmEbSXPdv0naP61tHJPW9kgTwjbgX3HtYac5QViu64E9SP47exZ4qJk+dwJJR+8Kknb/O0lODFl2OsaIeAW4lOSkv4yknXpxA5v9F0nH6eMR8VHO8skkJ+81wC/TmAuJ4cH0OzwOLEzfc10CTJG0hqTP5K6cbdcBU4E/Krl66nN5+14BnEby3/8Kkk7b0/LiLlRDx/l8YDNJLepDkj4YIuI5kk7wnwKrgCeprdV8j+Q//k+AH7B9jSzL7SQ1uCXAq2kcuSYDLwFzgI+Bn7D9Oe124EiSPi3bCb5RzlocSXcCr0dE0Wsw1npJ+gpwYUR8vtSx7K5cg7CSk3SUpIPSJokxJO3O9za0nVld0ua7S4BppY5ld+YEYS3BfiSXYK4luYZ/UkS8WNKIbLcl6Qsk/TUf0HAzltXDTUxmZpbJNQgzM8vUagbr69WrV1RUVJQ6DDOz3crzzz//UUT0zlpX1ASRdjj+P6AM+M+I+HHe+v4k1zL3JrlM7csRsThd9xOSsWgAromIei8jrKiooKqqqom/gZlZ6yYp/+77GkVrYkpvyb+J5C7UgcD4dLC0XNcBt0fEIJKbln6UbjsOGEYy/svRwGRJexYrVjMz21Ex+yBGkAzQ9nZEbAJmkFy+mGsgtTcKzc5ZPxB4KiK2pOPlzAfGFDFWMzPLU8wE0ZftByVbzPaDhgHMIxkEDeBMoHs6NPE8YEw6YFgv4ATqH5/HzMyaWKmvYpoMjJL0IslwBkuArRHxCMm4Ms+QDHXwJ2ofk1lD0oWSqiRVLV++vBnDNjNr/YqZIJaw/X/95eSNKhkRSyPirIgYCnwnXbYyfZ8aEUMi4hSS0RvfyP+AiJgWEZURUdm7d2YnvJmZ7aRiJog5wMFKHifZETiPZBjjGpJ6peO1A1xJckVT9TjwPdPpQcAg4JEixmpmttuZPh0qKqBdu+R9+vSGtmicol3mGhFbJF0GPExymestEfGKpClAVUTcR/JUsx9JCpIHklyabt4BeDp9IuVqkstf/QBzM7PU9Olw4YWwLn3U1jvvJPMAEyY0zWe0mqE2Kisrw/dBmFlbUVGRJIV8/fvDokWF70fS8xFRmbWu1J3UZma2E959t3HLd4YThJnZbuiA/IfVNrB8ZzhBmJnthqZOhS5dtl/WpUuyvKk4QZiZ7YYmTIBp05I+Byl5nzat6TqooRWN5mpm1tZMmNC0CSGfaxBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmRVTs8fpbGx+vlsV3UpsVSXOM19+a+Hi1PH4ehFmRNNV4/W2Fj1dp+HkQZiXQHOP1tyY+Xi2PE4RZkTTHeP2tiY9Xy+MEYVYkzTFef2vi49XyOEGYFUlzjNffmvh4tTzupDYza8PcSW1mZo3mBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8tU1AQhaYykBZIWSroiY31/SbMkzZf0hKTynHX/IukVSa9JukGSihmrmZltr2gJQlIZcBMwFhgIjJc0MK/YdcDtETEImAL8KN12JHAsMAj4X8BRwKhixWpmZjsqZg1iBLAwIt6OiE3ADOCMvDIDgcfT6dk56wPoDHQEOgEdgA+KGKuZmeUpZoLoC7yXM784XZZrHnBWOn0m0F1Sz4j4E0nCWJa+Ho6I1/I/QNKFkqokVS1fvrzJv4CZWVtW6k7qycAoSS+SNCEtAbZK+ixwOFBOklROlHRc/sYRMS0iKiOisnfv3s0Zt5lZq9e+iPteAvTLmS9Pl9WIiKWkNQhJ3YAvRsRKSX8PPBsRa9N1DwLHAE8XMV4zM8tRzBrEHOBgSQMkdQTOA+7LLSCpl6TqGK4Ebkmn3yWpWbSX1IGkdrFDE5OZmRVP0RJERGwBLgMeJjm53xURr0iaIun0tNhoYIGkN4B9geqHC/4WeAt4iaSfYl5E3F+sWM3MbEd+opyZWRtW3xPlitkHYa3Apk3w/vuwdGnyWrIkmT/0UBgzBvbZp9QRmlmxOEG0UVu3wvLltSf96gSQ+1qyJCmTT4KI5L2yEsaNg1NPheHDoV2pr4szsybjJqZWJgI++aT+k/7SpUktYOvW7bdt1w723Rf233/7V9++28/vvTfMmwczZyavZ59NPrd3bxg7NkkWf/VX8JnPlOYYmFnh6mticoLYjaxZU/9Jv/q1ceOO2/bsWf9Jf//9k+TQfifqlB99BA8/nCSLhx6Cjz+GsjIYOTJJFqeeCkcemdQ4zKxlcYJo4TZsgGXL6j/pL12aJIh83bs3fOLv0wc6d26e77J1Kzz3XG3t4oUXkuXl5bXJ4qSToFu35onHzOrnBFEiW7bABx/Uf9JfuhRWrNhx206dGj7x779/kiBasqVLk1rFzJnwyCNJkuvYEY4/PkkW48bBwQe7dmFWKk4QTWzbtuSk3lAH7wcfJG3zucrKYL/96j/x9+2btN+3tpPmpk3wzDPwwANJwnj11WT5QQfV1i5GjYI99ihtnGZtiRNEgSJg9eqGO3iXLYPNm3fcvnfvhk/8vXsnScJg0SJ48MEkWcyaBevXJ8nhpJNqE0b//qWO0qx1c4Kox4cfwjnn1CaBdet2LLPXXvWf9PffP6kVdOzYBF+khZs+Hb7zHXj3XTjgAJg6FSZM2PX9rl8PTz6ZJIsHHoC3306WDxxY2xR17LHQocOuf5aZ1XKCqMf69fCFL+x48q+e79MHunYtQsC7oenT4cILt0+iXbrAtGlNkySqRcCbb9Y2RT35ZFJj23NPOOWUJGGMHZv8bMxs1zhBWJOoqIB33tlxef/+SXNRsaxZA48/Xntl1OLFyfJhw2qbokaMcNOd2c5wgrAm0a7djp3ukHSmb9vWPDFEwEsv1SaLZ55JLq3de+9k6I9x45IaYc+ezROP2e7OCcKaRKlqEPX55JPk8tmZM5MO7+XLk0R29NG1tYuhQ1vfFWFmTaW+BOGRc6xgU6cmfQ65unRJlpfKZz4D554Lt92WDB/y3HNw1VVJreKqq5Lxofr2ha9/He65B1atKl2sZrsb1yCsUYp1FVMxfPhh7U16Dz8MK1cmQ4l8/vO1AwwefrhrF9a2uYnJ2rwtW5JBBauvjJo/P1nev39tU9SJJ+5YQzJr7ZwgzPIsXlx7k96jj8KnnybDm5xwQm3COOigUkdpVnxOEGb12LgRnn669sqoBQuS5YccUtsUddxxSQIxa22cIMwa4a23apPF7NlJAunaFU4+ubZ2UV5e6ih3b1u3Jve35L9Wr07eu3ZN+rj690+eWuh+ouJxgjDbSevWJUmiegiQ6st8Bw2qTRbHHLNzz9HYnUQkow5kncwbemWVW7++8M/u1ClJFtUJo/q9erpfv7YxzE2xOEGYNYEIeO212trF008nnd89eiQ35516ast6TveWLU13Ql+zpvCbIbt2TYahz33tueeOy7Jee+6ZPCtk7drkSrl33ql9r55etmz7z5OSsdByk0f+e48eTX98WwsnCLMiWL066eCuThjvv1/7nO7qAQYb85zuiKTG0piTeX1lN2wo7HM7dNi1E3rufNeuxR/yZOPG5CKD3OSRn0Q2bdp+mz33rDt5HHBAMq5XWx2qxQnCrMi2bat9TvcDD+z4nO4+fRo+8a9dW/h/6d26Nc0JvXv31tf5vm1bcg9MfvLITSKffLL9Nu3bJ/1KuU1Xue/9+rXeS6CdIMya2Ucf1Q4B8tBDSQJoqhN6166F10os25o1ScKoK4ksWbJjsu7du+5+kP79k/G/dsfOdCcIsxKK2D1PHG3Z5s3J82Gykkf1e/6zY7p0ye5Mr37v27dlPs+kvgTRyq+9MCs9J4fdT4cOtTWELBHJY4frSh5z5ybNXLnatUueMVNfZ3pLe8a8E4SZWSNJ0KtX8ho2LLvM+vV1N2M9+yzcfXdypVmuHj3q7gc54ADYd9/mbV50gjAzK4I99oBDD01eWbZuTa58y6qB/OUv8MQTyUUNuTp2TDrM8xPHoYcmj+Rtak4QZmYlUFaW9Ev07QsjR2aXWbmy7masRx5J7gmJSJ5/8uyzTR+jE4SZWQvVo0fyGjQoe/3GjckVV425M70xitqaJWmMpAWSFkq6ImN9f0mzJM2X9ISk8nT5CZLm5rw2SPqbYsZqZra76dQJDjwQjjiiOPsvWoKQVAbcBIwFBgLjJQ3MK3YdcHtEDAKmAD8CiIjZETEkIoYAJwLrgEeKFauZme2omDWIEcDCiHg7IjYBM4Az8soMBB5Pp2dnrAc4G3gwItZlrDMzsyIpZoLoC7yXM784XZZrHnBWOn0m0F1Sz7wy5wH/lfUBki6UVCWpavny5U0QspmZVSv1DfuTgVGSXgRGAUuArdUrJfUBjgQezto4IqZFRGVEVPbu3bs54jUzazOKeRXTEqBfznx5uqxGRCwlrUFI6gZ8MSJW5hT5W+D3EbG5iHGamVmGYtYg5gAHSxogqSNJU9F9uQUk9ZJUHcOVwC15+xhPHc1LZmZWXEVLEBGxBbiMpHnoNeCuiHhF0hRJp6fFRgMLJL0B7AtMrd5eUgVJDeTJYsVoZmZ182iuZmZtWH2juZa6k9rMzFqoBhOEpL/O6ScwM7M2opAT/7nAm5L+RdJhxQ7IzMxahgYTRER8GRgKvAX8StKf0hvUWtijLczMrCkV1HQUEauB35IMl9GH5K7nFyT9QxFjMzOzEiqkD+J0Sb8HngA6ACMiYiwwGPg/xQ3PzMxKpZA7qb8I/DQinspdGBHrJH29OGGZmVmpFZIgrgaWVc9I2gPYNyIWRcSsYgVmZmalVUgfxN3Atpz5rekyMzNrxQpJEO3T5zkAkE53LF5IZmbWEhSSIJbnjJ2EpDOAj4oXkpmZtQSF9EFcDEyXdCMgkocAfaWoUZmZWck1mCAi4i3gc+nzGoiItUWPyszMSq6gBwZJGgccAXSWBEBETCliXGZmVmKF3Cj3C5LxmP6BpInpHKB/keMyM7MSK6STemREfAX4JCJ+ABwDHFLcsMzMrNQKSRAb0vd1kvYHNpOMx2RmZq1YIX0Q90vqAVwLvAAE8MuiRmVmZiVXb4JIHxQ0KyJWAvdI+gPQOSJWNUt0ZmZWMvU2MUXENuCmnPmNTg5mZm1DIX0QsyR9UdXXt5qZWZtQSIK4iGRwvo2SVktaI2l1keMyM7MSK+ROaj9a1MysDWowQUg6Pmt5/gOEzMysdSnkMtd/ypnuDIwAngdOLEpEZmbWIhTSxPTXufOS+gHXFy0iMzNrEQrppM63GDi8qQMxM7OWpZA+iH8nuXsakoQyhOSOajMza8UK6YOoypneAvxXRPyxSPGYmVkLUUiC+C2wISK2Akgqk9QlItYVNzQzMyulgu6kBvbImd8DeKyQnUsaI2mBpIWSrshY31/SLEnzJT0hqTxn3QGSHpH0mqRXJVUU8plmZtY0CkkQnXMfM5pOd2loI0llJOM4jQUGAuMlDcwrdh1we0QMAqYAP8pZdztwbUQcTnJp7YcFxGpmZk2kkATxqaRh1TOShgPrC9huBLAwIt6OiE3ADOCMvDIDgcfT6dnV69NE0j4iHoUkKblJy8yseRWSIL4J3C3paUn/A9wJXFbAdn2B93LmF6fLcs0DzkqnzwS6S+pJ8sS6lZJ+J+lFSdemNZLtSLpQUpWkquXLlxcQkpmZFarBBBERc4DDgEnAxcDhEfF8E33+ZGCUpBeBUcASYCtJ5/lx6fqjgAOBiRmxTYuIyoio7N27dxOFZGZmUECCkHQp0DUiXo6Il4Fuki4pYN9LgH458+XpshoRsTQizoqIocB30mUrSWobc9PmqS3AvcAwzMys2RTSxPT36UkbgIj4BPj7ArabAxwsaYCkjsB5wH25BST1Sp9aB3AlcEvOtj0kVVcLTgReLeAzzcysiRSSIMpyHxaU9gV0bGij9D//y4CHgdeAuyLiFUlTJJ2eFhsNLJD0BrAvMDXdditJ89IsSS8Bws/BNjNrVoqI+gtI1wL9gf9IF10EvBsRk4scW6NUVlZGVVVVwwXNzKyGpOcjojJrXSF3Uv8zcCFJBzXAfGC/JorNzMxaqEKuYtoG/BlYRHJvw4kkTUZmZtaK1VmDkHQIMD59fURy/wMRcULzhGZmZqVUXxPT68DTwGkRsRBA0reaJSozMyu5+pqYzgKWAbMl/VLSSSRXE5mZWRtQZ4KIiHsj4jySu6hnkwy5sY+kn0v6q+YK0MzMSqOQTupPI+I36bOpy4EXSa5sMjOzVqxRz6SOiE/S8Y9OKlZAZmbWMjQqQZiZWdvhBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTEVNEJLGSFogaaGkKzLW95c0S9J8SU9IKs9Zt1XS3PR1XzHjNDOzHbUv1o4llQE3AacAi4E5ku6LiFdzil0H3B4Rt0k6EfgRcH66bn1EDClWfGZmVr9i1iBGAAsj4u2I2ATMAM7IKzMQeDydnp2x3szMSqSYCaIv8F7O/OJ0Wa55wFnp9JlAd0k90/nOkqokPSvpb7I+QNKFaZmq5cuXN2XsZmZtXqk7qScDoyS9CIwClgBb03X9I6IS+BJwvaSD8jeOiGkRURkRlb179262oM3M2oKi9UGQnOz75cyXp8tqRMRS0hqEpG7AFyNiZbpuSfr+tqQngKHAW0WM18zMchSzBjEHOFjSAEkdgfOA7a5GktRLUnUMVwK3pMs/I6lTdRngWCC3c9vMzIqsaAkiIrYAlwEPA68Bd0XEK5KmSDo9LTYaWCDpDWBfYGq6/HCgStI8ks7rH+dd/WRmZkWmiCh1DE2isrIyqqqqSh2GmdluRdLzaX/vDkrdSW1mZi2UE4SZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZJicIMzPL5ARhZmaZnCDMzCyTE4SZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZJicIMzPLVMxHjppZG7F582YWL17Mhg0bSh2K1aFz586Ul5fToUOHgrdxgjCzXbZ48WK6d+9ORUUFkkodjuWJCFasWMHixYsZMGBAwdu5icnMdtmGDRvo2bOnk0MLJYmePXs2uobnBGFmTcLJoWXbmZ+PE4SZmWVygjCzZjd9OlRUQLt2yfv06bu2vxUrVjBkyBCGDBnCfvvtR9++fWvmN23aVO+2VVVVfOMb32jwM0aOHLlrQe6G3EltZs1q+nS48EJYty6Zf+edZB5gwoSd22fPnj2ZO3cuAFdffTXdunVj8uTJNeu3bNlC+/bZp7vKykoqKysb/Ixnnnlm54LbjbkGYWbN6jvfqU0O1datS5Y3pYkTJ3LxxRdz9NFHc/nll/Pcc89xzDHHMHToUEaOHMmCBQsAeOKJJzjttNOAJLl87WtfY/To0Rx44IHccMMNNfvr1q1bTfnRo0dz9tlnc9hhhzFhwgQiAoCZM2dy2GGHMXz4cL7xjW/U7DfXokWLOO644xg2bBjDhg3bLvH85Cc/4cgjj2Tw4MFcccUVACxcuJCTTz6ZwYMHM2zYMN56662mPVD1cA3CzJrVu+82bvmuWLx4Mc888wxlZWWsXr2ap59+mvbt2/PYY4/x7W9/m3vuuWeHbV5//XVmz57NmjVrOPTQQ5k0adIO9w68+OKLvPLKK+y///4ce+yx/PGPf6SyspKLLrqIp556igEDBjB+/PjMmPbZZx8effRROnfuzJtvvsn48eOpqqriwQcf5L//+7/585//TJcuXfj4448BmDBhAldccQVnnnkmGzZsYNu2bU1/oOrgBGFmzeqAA5JmpazlTe2cc86hrKwMgFWrVnHBBRfw5ptvIonNmzdnbjNu3Dg6depEp06d2Gefffjggw8oLy/frsyIESNqlg0ZMoRFixbRrVs3DjzwwJr7DMaPH8+0adN22P/mzZu57LLLmDt3LmVlZbzxxhsAPPbYY3z1q1+lS5cuAOy9996sWbOGJUuWcOaZZwLJzW7NyU1MZtaspk6F9BxYo0uXZHlT69q1a8309773PU444QRefvll7r///jrvCejUqVPNdFlZGVu2bNmpMnX56U9/yr777su8efOoqqpqsBO9lJwgzKxZTZgA06ZB//4gJe/Tpu18B3WhVq1aRd++fQH41a9+1eT7P/TQQ3n77bdZtGgRAHfeeWedcfTp04d27dpxxx13sHXrVgBOOeUUbr31VtalHTQff/wx3bt3p7y8nHvvvReAjRs31qxvDk4QZtbsJkyARYtg27bkvdjJAeDyyy/nyiuvZOjQoY36j79Qe+yxBz/72c8YM2YMw4cPp3v37uy11147lLvkkku47bbbGDx4MK+//npNLWfMmDGcfvrpVFZWMmTIEK677joA7rjjDm644QYGDRrEyJEjef/995s89rqouvd9d1dZWRlVVVWlDsOsTXrttdc4/PDDSx1Gya1du5Zu3boREVx66aUcfPDBfOtb3yp1WDWyfk6Sno+IzOt8XYMwM2siv/zlLxkyZAhHHHEEq1at4qKLLip1SLukqAlC0hhJCyQtlHRFxvr+kmZJmi/pCUnleev3lLRY0o3FjNPMrCl861vfYu7cubz66qtMnz695oqk3VXREoSkMuAmYCwwEBgvaWBeseuA2yNiEDAF+FHe+muAp4oVo5mZ1a2YNYgRwMKIeDsiNgEzgDPyygwEHk+nZ+eulzQc2Bd4pIgxmplZHYqZIPoC7+XML06X5ZoHnJVOnwl0l9RTUjvgX4HJ1EPShZKqJFUtX768icI2MzMofSf1ZGCUpBeBUcASYCtwCTAzIhbXt3FETIuIyoio7N27d/GjNTNrQ4qZIJYA/XLmy9NlNSJiaUScFRFDge+ky1YCxwCXSVpE0gtmSLsAAAshSURBVE/xFUk/LmKsZrYbO+GEE3j44Ye3W3b99dczadKkOrcZPXo01ZfGn3rqqaxcuXKHMldffXXN/Qh1uffee3n11Vdr5q+66ioee+yxxoTfYhUzQcwBDpY0QFJH4DzgvtwCknqlzUkAVwK3AETEhIg4ICIqSGoZt0fEDldBmZlBMu7RjBkztls2Y8aMOgfMyzdz5kx69OixU5+dnyCmTJnCySefvFP7ammKNlhfRGyRdBnwMFAG3BIRr0iaAlRFxH3AaOBHkoLkaqVLixWPmTWPb34T0kczNJkhQ+D66+tef/bZZ/Pd736XTZs20bFjRxYtWsTSpUs57rjjmDRpEnPmzGH9+vWcffbZ/OAHP9hh+4qKCqqqqujVqxdTp07ltttuY5999qFfv34MHz4cSO5xmDZtGps2beKzn/0sd9xxB3PnzuW+++7jySef5Ic//CH33HMP11xzDaeddhpnn302s2bNYvLkyWzZsoWjjjqKn//853Tq1ImKigouuOAC7r//fjZv3szdd9/NYYcdtl1MixYt4vzzz+fTTz8F4MYbb6x5aNFPfvITfv3rX9OuXTvGjh3Lj3/8YxYuXMjFF1/M8uXLKSsr4+677+aggw7apeNe1D6IiJgZEYdExEERMTVddlWaHIiI30bEwWmZv4uIjRn7+FVEXFbMOM1s97b33nszYsQIHnzwQSCpPfzt3/4tkpg6dSpVVVXMnz+fJ598kvnz59e5n+eff54ZM2Ywd+5cZs6cyZw5c2rWnXXWWcyZM4d58+Zx+OGHc/PNNzNy5EhOP/10rr32WubOnbvdCXnDhg1MnDiRO++8k5deeoktW7bw85//vGZ9r169eOGFF5g0aVJmM1b1sOAvvPACd955Z81T73KHBZ83bx6XX345kAwLfumllzJv3jyeeeYZ+vTps2sHFQ/3bWZNrL7/9IupupnpjDPOYMaMGdx8880A3HXXXUybNo0tW7awbNkyXn31VQYNGpS5j6effpozzzyz5ga3008/vWbdyy+/zHe/+11WrlzJ2rVr+cIXvlBvPAsWLGDAgAEccsghAFxwwQXcdNNNfPOb3wSShAMwfPhwfve73+2wfUsYFrzUVzGVXFM/G9fMSuOMM85g1qxZvPDCC6xbt47hw4fzl7/8heuuu45Zs2Yxf/58xo0bV+cw3w2ZOHEiN954Iy+99BLf//73d3o/1aqHDK9ruPCWMCx4m04Q1c/GfecdiKh9Nq6ThNnup1u3bpxwwgl87Wtfq+mcXr16NV27dmWvvfbigw8+qGmCqsvxxx/Pvffey/r161mzZg33339/zbo1a9bQp08fNm/ezPSck0T37t1Zs2bNDvs69NBDWbRoEQsXLgSSUVlHjRpV8PdpCcOCt+kE0VzPxjWz5jF+/HjmzZtXkyAGDx7M0KFDOeyww/jSl77EscceW+/2w4YN49xzz2Xw4MGMHTuWo446qmbdNddcw9FHH82xxx67XYfyeeedx7XXXsvQoUO3e150586dufXWWznnnHM48sgjadeuHRdffHHB36UlDAvepof7btcuqTnkk5Jx6s2sMB7ue/fg4b4boa5n4Bbj2bhmZrubNp0gmvPZuGZmu5s2nSBK9Wxcs9aotTRXt1Y78/Np8/dBTJjghGC2qzp37syKFSvo2bMnkkodjuWJCFasWNHo+yPafIIws11XXl7O4sWL8bD7LVfnzp0pLy9vuGAOJwgz22UdOnRgwIABpQ7Dmlib7oMwM7O6OUGYmVkmJwgzM8vUau6klrQceGcXdtEL+KiJwmlKjqtxHFfjOK7GaY1x9Y+IzGc2t5oEsaskVdV1u3kpOa7GcVyN47gap63F5SYmMzPL5ARhZmaZnCBqTSt1AHVwXI3juBrHcTVOm4rLfRBmZpbJNQgzM8vkBGFmZpnaVIKQdIukDyW9XMd6SbpB0kJJ8yUNayFxjZa0StLc9HVVM8XVT9JsSa9KekXSP2aUafZjVmBczX7MJHWW9JykeWlcP8go00nSnenx+rOkihYS10RJy3OO198VO66czy6T9KKkP2Ssa/bjVUBMpTxWiyS9lH7uDo/QbPK/x4hoMy/geGAY8HId608FHgQEfA74cwuJazTwhxIcrz7AsHS6O/AGMLDUx6zAuJr9mKXHoFs63QH4M/C5vDKXAL9Ip88D7mwhcU0Ebmzu37H0s/838Jusn1cpjlcBMZXyWC0CetWzvkn/HttUDSIingI+rqfIGcDtkXgW6CGpTwuIqyQiYllEvJBOrwFeA/rmFWv2Y1ZgXM0uPQZr09kO6Sv/KpAzgNvS6d8CJ6nID1AoMK6SkFQOjAP+s44izX68CoipJWvSv8c2lSAK0Bd4L2d+MS3gxJM6Jm0ieFDSEc394WnVfijJf5+5SnrM6okLSnDM0qaJucCHwKMRUefxiogtwCqgZwuIC+CLabPEbyX1K3ZMqeuBy4FtdawvxfFqKCYozbGCJLE/Iul5SRdmrG/Sv0cniN3DCyTjpQwG/h24tzk/XFI34B7gmxGxujk/uz4NxFWSYxYRWyNiCFAOjJD0v5rjcxtSQFz3AxURMQh4lNr/2otG0mnAhxHxfLE/q1AFxtTsxyrH5yNiGDAWuFTS8cX8MCeI7S0Bcv8bKE+XlVRErK5uIoiImUAHSb2a47MldSA5CU+PiN9lFCnJMWsorlIes/QzVwKzgTF5q2qOl6T2wF7AilLHFRErImJjOvufwPBmCOdY4HRJi4AZwImSfp1XprmPV4MxlehYVX/2kvT9Q+D3wIi8Ik369+gEsb37gK+kVwJ8DlgVEctKHZSk/arbXSWNIPm5Ff2kkn7mzcBrEfFvdRRr9mNWSFylOGaSekvqkU7vAZwCvJ5X7D7ggnT6bODxSHsXSxlXXjv16ST9OkUVEVdGRHlEVJB0QD8eEV/OK9asx6uQmEpxrNLP7Sqpe/U08FdA/pWPTfr32KYeOSrpv0iubuklaTHwfZIOOyLiF8BMkqsAFgLrgK+2kLjOBiZJ2gKsB84r9kkldSxwPvBS2n4N8G3ggJzYSnHMComrFMesD3CbpDKShHRXRPxB0hSgKiLuI0lsd0haSHJhwnlFjqnQuL4h6XRgSxrXxGaIK1MLOF4NxVSqY7Uv8Pv0/572wG8i4iFJF0Nx/h491IaZmWVyE5OZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZJicIswZI2pozcudcSVc04b4rVMcovmal1qbugzDbSevTYSrM2hTXIMx2Ujo2/7+k4/M/J+mz6fIKSY+ng7nNknRAunxfSb9PBxCcJ2lkuqsySb9U8qyGR9K7nZH0DSXPvJgvaUaJvqa1YU4QZg3bI6+J6dycdasi4kjgRpJRQCEZHPC2dDC36cAN6fIbgCfTAQSHAa+kyw8GboqII4CVwBfT5VcAQ9P9XFysL2dWF99JbdYASWsjolvG8kXAiRHxdjp44PsR0VPSR0CfiNicLl8WEb0kLQfKcwZ6qx6u/NGIODid/2egQ0T8UNJDwFqSkWjvzXmmg1mzcA3CbNdEHdONsTFneiu1fYPjgJtIahtz0tFMzZqNE4TZrjk35/1P6fQz1A4qNwF4Op2eBUyCmgf47FXXTiW1A/pFxGzgn0mGud6hFmNWTP6PxKxhe+SMGgvwUERUX+r6GUnzSWoB49Nl/wDcKumfgOXUjqj5j8A0SV8nqSlMAuoairkM+HWaRATckD7LwazZuA/CbCelfRCVEfFRqWMxKwY3MZmZWSbXIMzMLJNrEGZmlskJwszMMjlBmJlZJicIMzPL5ARhZmaZ/j/FQ00psRYY8gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Evaluate Model**"
      ],
      "metadata": {
        "id": "Lqi80zysdVyY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# evaluate model on test dataset\n",
        "x_test = np.array([np.array(val) for val in testdf['text']])\n",
        "y_test = np.array([np.array(val) for val in testdf['intent']])\n",
        "# model evaluation\n",
        "print('Model Evaluation on Test Set:')\n",
        "loss, accuracy = model.evaluate(x_test, y_test)\n",
        "print(\"Loss: \", loss)\n",
        "print(\"Accuracy: \", accuracy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yyNR3LFEdZWx",
        "outputId": "b7d6e57c-373e-42c5-8855-8a382a33b399"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model Evaluation on Test Set:\n",
            "40/40 [==============================] - 3s 62ms/step - loss: 0.1064 - accuracy: 0.9758\n",
            "Loss:  0.10639981180429459\n",
            "Accuracy:  0.975781261920929\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Export Model**"
      ],
      "metadata": {
        "id": "OOCtwhBJI-ri"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# save the model\n",
        "model.save('saved_model/model_smallbert', save_format='tf')"
      ],
      "metadata": {
        "id": "UU3Z3GZ03Jp1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0b46e0b3-a560-43c3-d978-5dbc16b3e2ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Found untraced functions such as restored_function_body, restored_function_body, restored_function_body, restored_function_body, restored_function_body while saving (showing 5 of 64). These functions will not be directly callable after loading.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Conclusion:**\n",
        "\n",
        "**Model (smallBERTuncased L-2_H-128_A-2):**\n",
        "1.   Epochs: 5\n",
        "2.   Loss and Accuracy:\n",
        "    *   Training Loss: 0.0437 - Training Accuracy: 0.9885\n",
        "    *   Validation Loss: 0.1214 - Validation Accuracy: 0.9712\n",
        "    *   Testing Loss: 0.1063 - Testing Accuracy: 0.9757\n",
        "3.   Wall Time: 2min 33s\n",
        "\n",
        "**ANALYSIS:**\n",
        "1.   Model quickly overfits the data.\n",
        "    *   Reason: The data isn't complex/hard to learn for a BERT like model.\n"
      ],
      "metadata": {
        "id": "3452xAo7ODVB"
      }
    }
  ]
}