RNN_goblet_of_fire.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "cell_id": "6fa0a29f-5e90-4b0a-a5f9-64d234e8e092",
        "deepnote_cell_type": "code"
      },
      "source": "#Imports\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random",
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "cell_id": "fa7fc9ab-0383-48ea-9749-55ea5c421f42",
        "deepnote_cell_type": "code"
      },
      "source": "#data preprocessing\ninput_file = open('goblet_book.txt', 'r')\ninput_data = input_file.read()\nchar_list = []\nfor char in list(input_data):\n    char_list.append(char)\n\nbook_chars = np.unique(char_list)\nK = len(book_chars)\n\nchar_to_ind = {}\nfor index, book_char in enumerate(book_chars):\n    char_to_ind[book_char] = index\n\nind_to_char = {}\nfor index, book_char in enumerate(book_chars):\n    ind_to_char[index] = book_char\n\nprint(ind_to_char)\nprint(char_to_ind)\nprint(K)\n",
      "outputs": [
        {
          "name": "stdout",
          "text": "{0: '\\t', 1: '\\n', 2: ' ', 3: '!', 4: '\"', 5: \"'\", 6: '(', 7: ')', 8: ',', 9: '-', 10: '.', 11: '/', 12: '0', 13: '1', 14: '2', 15: '3', 16: '4', 17: '6', 18: '7', 19: '9', 20: ':', 21: ';', 22: '?', 23: 'A', 24: 'B', 25: 'C', 26: 'D', 27: 'E', 28: 'F', 29: 'G', 30: 'H', 31: 'I', 32: 'J', 33: 'K', 34: 'L', 35: 'M', 36: 'N', 37: 'O', 38: 'P', 39: 'Q', 40: 'R', 41: 'S', 42: 'T', 43: 'U', 44: 'V', 45: 'W', 46: 'X', 47: 'Y', 48: 'Z', 49: '^', 50: '_', 51: 'a', 52: 'b', 53: 'c', 54: 'd', 55: 'e', 56: 'f', 57: 'g', 58: 'h', 59: 'i', 60: 'j', 61: 'k', 62: 'l', 63: 'm', 64: 'n', 65: 'o', 66: 'p', 67: 'q', 68: 'r', 69: 's', 70: 't', 71: 'u', 72: 'v', 73: 'w', 74: 'x', 75: 'y', 76: 'z', 77: '}', 78: 'ü', 79: '•'}\n{'\\t': 0, '\\n': 1, ' ': 2, '!': 3, '\"': 4, \"'\": 5, '(': 6, ')': 7, ',': 8, '-': 9, '.': 10, '/': 11, '0': 12, '1': 13, '2': 14, '3': 15, '4': 16, '6': 17, '7': 18, '9': 19, ':': 20, ';': 21, '?': 22, 'A': 23, 'B': 24, 'C': 25, 'D': 26, 'E': 27, 'F': 28, 'G': 29, 'H': 30, 'I': 31, 'J': 32, 'K': 33, 'L': 34, 'M': 35, 'N': 36, 'O': 37, 'P': 38, 'Q': 39, 'R': 40, 'S': 41, 'T': 42, 'U': 43, 'V': 44, 'W': 45, 'X': 46, 'Y': 47, 'Z': 48, '^': 49, '_': 50, 'a': 51, 'b': 52, 'c': 53, 'd': 54, 'e': 55, 'f': 56, 'g': 57, 'h': 58, 'i': 59, 'j': 60, 'k': 61, 'l': 62, 'm': 63, 'n': 64, 'o': 65, 'p': 66, 'q': 67, 'r': 68, 's': 69, 't': 70, 'u': 71, 'v': 72, 'w': 73, 'x': 74, 'y': 75, 'z': 76, '}': 77, 'ü': 78, '•': 79}\n80\n",
          "output_type": "stream"
        }
      ],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "cell_id": "453f5079-1005-4570-8d2a-206d1a190a8d",
        "deepnote_cell_type": "code"
      },
      "source": "def softmax(x):\n    return np.exp(x)/np.sum(np.exp(x), axis = 0)\n\nclass RNN:\n    def __init__(self, b, c, u, w, v):\n        self.b = b\n        self.c = c\n        self.U = u\n        self.W = w\n        self.V = v\n\n    def synthesize_text(self, ix, h, n):\n        xnext = np.zeros((K, 1))\n        xnext[ix] = 1\n\n        synthesized_text = ''\n        for t in range(n):\n            _, h, _, p = self.evaluate_classifier(xnext, h)\n            ix = np.random.choice(range(K), p=p.flat)\n            xnext = np.zeros((K, 1))\n            xnext[ix] = 1\n            synthesized_text += ind_to_char[ix]\n\n        return synthesized_text\n\n    def evaluate_classifier(self, x, h):\n        a = self.W @ h + self.U @ x + self.b\n        h = np.tanh(a)\n        o = self.V @ h + self.c\n        p = softmax(o)\n        return a, h, o, p\n\n    def compute_gradients(self, inputs, targets, hprev):\n        \"\"\" Analytically computes the gradients of the weight and bias parameters\n        Args:\n            inputs      (list): indices of the chars of the input sequence\n            targets     (list): indices of the chars of the target sequence\n            hprev (np.ndarray): previous learnt hidden state sequence\n        Returns:\n            grads (dict): the updated analytical gradients dU, dW, dV, db and dc\n            loss (float): the current loss\n            h (np.ndarray): newly learnt hidden state sequence\n        \"\"\"\n        n = len(inputs)\n        loss = 0\n\n        # Dictionaries for storing values during the forward pass\n        a_2, x_2, h_2, o_2, p_2 = {}, {}, {}, {}, {}\n        h_2[-1] = np.copy(hprev)\n\n        # Forward pass\n        for t in range(n):\n            x_2[t] = np.zeros((K, 1))\n            x_2[t][inputs[t]] = 1 # 1-hot-encoding\n\n            a_2[t], h_2[t], o_2[t], p_2[t] = self.evaluate_classifier(x_2[t], h_2[t-1])\n\n            loss += -np.log(p_2[t][targets[t]][0]) # update the loss\n\n        # Dictionary for storing the gradients\n        grads = {\"W\": np.zeros_like(self.W), \"U\": np.zeros_like(self.U),\n                 \"V\": np.zeros_like(self.V), \"b\": np.zeros_like(self.b),\n                 \"c\": np.zeros_like(self.c), \"o\": np.zeros_like(p_2[0]),\n                 \"h\": np.zeros_like(h_2[0]), \"h_next\": np.zeros_like(h_2[0]),\n                 \"a\": np.zeros_like(a_2[0])}\n\n        # Backward pass\n        for t in reversed(range(n)):\n            grads[\"o\"] = np.copy(p_2[t])\n            grads[\"o\"][targets[t]] -= 1\n\n            grads[\"V\"] += grads[\"o\"] @ h_2[t].T\n            grads[\"c\"] += grads[\"o\"]\n\n            grads[\"h\"] = self.V.T @ grads[\"o\"] + grads[\"h_next\"]\n            grads[\"a\"] = np.multiply(grads[\"h\"], (1 - np.square(h_2[t])))\n\n            grads[\"U\"] += grads[\"a\"] @ x_2[t].T\n            grads[\"W\"] += grads[\"a\"] @ h_2[t-1].T\n            grads[\"b\"] += grads[\"a\"]\n\n            grads[\"h_next\"] = self.W.T @ grads[\"a\"]\n\n        # Drop redundant gradients\n        grads = {k: grads[k] for k in grads if k not in [\"o\", \"h\", \"h_next\", \"a\"]}\n\n        # Clip the gradients\n        for grad in grads:\n            grads[grad] = np.clip(grads[grad], -5, 5)\n\n        # Update the hidden state sequence\n        h = h_2[n-1]\n\n        return grads, loss, h\n\n\n    def compute_gradients_num(self, inputs, targets, hprev, h, comps=20):\n        \"\"\"Numerically computes the gradients of the weight and bias parameters\n        Args:\n            inputs      (list): indices of the chars of the input sequence\n            targets     (list): indices of the chars of the target sequence\n            hprev (np.ndarray): previous learnt hidden state sequence\n            h     (np.float64): marginal offset\n            comps    (int): number of entries per gradient to compute\n        Returns:\n            grads (dict): the numerically computed gradients dU, dW, dV,\n                          db and dc\n        \"\"\"\n        rnn_params = {\"W\": self.W, \"U\": self.U, \"V\": self.V, \"b\": self.b, \"c\": self.c}\n        num_grads  = {\"W\": np.zeros_like(self.W), \"U\": np.zeros_like(self.U),\n                      \"V\": np.zeros_like(self.V), \"b\": np.zeros_like(self.b),\n                      \"c\": np.zeros_like(self.c)}\n\n        for key in rnn_params:\n            for i in range(comps):\n                old_par = rnn_params[key].flat[i] # store old parameter\n                rnn_params[key].flat[i] = old_par + h\n                _, l1, _ = self.compute_gradients(inputs, targets, hprev)\n                rnn_params[key].flat[i] = old_par - h\n                _, l2, _ = self.compute_gradients(inputs, targets, hprev)\n                rnn_params[key].flat[i] = old_par # reset parameter to old value\n                num_grads[key].flat[i] = (l1 - l2) / (2*h)\n\n        return num_grads\n\n\n    def check_gradients(self, inputs, targets, hprev, comps=20):\n        \"\"\"Check similarity between the analytical and numerical gradients\n        Args:\n            inputs      (list): indices of the chars of the input sequence\n            targets     (list): indices of the chars of the target sequence\n            hprev (np.ndarray): previous learnt hidden state sequence\n            comps    (int): number of gradient comparisons\n        \"\"\"\n        grads_ana, _, _ = self.compute_gradients(inputs, targets, hprev)\n        grads_num = self.compute_gradients_num(inputs, targets, hprev, 1e-5)\n\n        print(\"Gradient checks:\")\n        for grad in grads_ana:\n            num   = abs(grads_ana[grad].flat[:comps] -\n                    grads_num[grad].flat[:comps])\n            denom = np.asarray([max(abs(a), abs(b)) + 1e-10 for a,b in\n                zip(grads_ana[grad].flat[:comps],\n                    grads_num[grad].flat[:comps])\n            ])\n            max_rel_error = max(num / denom)\n\n            print(\"The maximum relative error for the %s gradient is: %e.\" %\n                    (grad, max_rel_error))\n        print()",
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "cell_id": "2f916e7e-f7b3-4f33-b61e-8bf53908da22",
        "deepnote_cell_type": "code"
      },
      "source": "#Synthesize text without training\nm = 100 #dimentionaly of hidden state\neta = 0.1\nseq_length = 25\nsig = 0.01\nb = np.zeros((m, 1))\nc = np.zeros((K ,1))\nu = np.random.normal(0, sig, (m, K))\nw = np.random.normal(0, sig, (m, m))\nv = np.random.normal(0, sig, (K, m))\nrnn = RNN(b, c, u, w, v)\nx0 = random.choice(book_chars)\nx0 = char_to_ind[x0]\nh0 = np.zeros((m,1))\nsyn_text = rnn.synthesize_text(x0, h0, seq_length)\nprint(syn_text)",
      "outputs": [
        {
          "name": "stdout",
          "text": "_BvO7)\tIBt0SlHY0eWRK.'IVk\n",
          "output_type": "stream"
        }
      ],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "cell_id": "0120db75-d1dc-4a6d-ba4f-e64a4a0e4f43",
        "deepnote_cell_type": "code"
      },
      "source": "e, n, epoch = 0, 0, 0\nnum_epochs = 10\nsmooth_loss_list = []\n\nrnn_params = {\"W\": rnn.W, \"U\": rnn.U, \"V\": rnn.V, \"b\": rnn.b, \"c\": rnn.c}\n\nmem_params = {\"W\": np.zeros_like(rnn.W), \"U\": np.zeros_like(rnn.U),\n                \"V\": np.zeros_like(rnn.V), \"b\": np.zeros_like(rnn.b),\n                \"c\": np.zeros_like(rnn.c)}\n\nwhile epoch < num_epochs:\n    if n == 0 or e >= (len(input_data) - seq_length - 1):\n        if epoch != 0: print(\"Finished %i epochs.\" % epoch)\n        hprev = h0\n        e = 0\n        epoch += 1\n\n    inputs = [char_to_ind[char] for char in input_data[e:e+seq_length]]\n    targets = [char_to_ind[char] for char in input_data[e+1:e+seq_length+1]]\n\n    grads, loss, hprev = rnn.compute_gradients(inputs, targets, hprev)\n\n    # Compute the smooth loss\n    if n == 0 and epoch == 1: smooth_loss = loss\n    smooth_loss = 0.999 * smooth_loss + 0.001 * loss\n    smooth_loss_list.append(smooth_loss)\n    if smooth_loss < 41:\n        break\n\n\n    # Check gradients\n    if n == 0: rnn.check_gradients(inputs, targets, hprev)\n\n    # Print the loss\n    if n % 10000 == 0: print('Iteration %d, smooth loss: %f' % (n, smooth_loss))\n\n    # Print synthesized text\n    if n % 10000 == 0:\n        txt = rnn.synthesize_text(inputs[0], hprev, 1000)\n        print('\\nSynthesized text after %i iterations:\\n %s\\n' % (n, txt))\n        print('Smooth loss: %f' % smooth_loss)\n\n    # Adagrad\n    for key in rnn_params:\n        mem_params[key] += grads[key] * grads[key]\n        rnn_params[key] -= eta / np.sqrt(mem_params[key] +\n                np.finfo(float).eps) * grads[key]\n\n    e += seq_length\n    n += 1",
      "outputs": [
        {
          "name": "stdout",
          "text": "Gradient checks:\nThe maximum relative error for the W gradient is: 4.765629e-05.\nThe maximum relative error for the U gradient is: 6.510356e-09.\nThe maximum relative error for the V gradient is: 1.031157e-05.\nThe maximum relative error for the b gradient is: 1.056136e-06.\nThe maximum relative error for the c gradient is: 5.552221e-09.\n\nIteration 0, smooth loss: 109.547852\n\nSynthesized text after 0 iterations:\n ;f\t\"•EMv,Q:c 72KqfKeepWk.E Kvl/HRywA'DXIg-LxC-WvDSe)Wu27\tCE:9_\tLnhJ\tZ3M?No;p:Hicgz2wGNAY) FMvYü/2RnYHGpx'yOXNCaiumgK\"7_f2twJU.glWC2G\n,0 _\";?Tx(ü';7EMs/J,a-?2HmrDP}Dt6WKvBü\"\nJR\tdfN4nGfLYKB/XI1NvBTl293O3Iw!O4S1W4gY m-\thMl.iLqVPA^JDjEiTzAKXIxzw-7iU-'iTDQ•!GKlB(Z_L6PEN;XBzy9XJHS?C•o;,TjVbAq3B;_JD_-?\n•tJüSXMmO\"BewSl0UR)7,WbTtysBC4s(UC00^Vb4\"l0Hyp!vW1aBXb•'TpKBRTo:YGOt-,VHg^6zxJgzJkmr!'z(\"2M\nxKyümMHqfOrü3:,a\tISeD?:V:pho(EF: s,h!kXeL4E.j?}W\"G0np-z\np0H02A9.Poha2?\nlHi-)0U}:-uqfyu.xP b/EW2•ko0m4WtDp\t;IIm\tNc!Sz.vR19WYa,B:a-\tWsb1oCOOr•e )\nfkvbo4Z3TfIcfZ9azn7o4Z.ai6MiKD1a(ü 7jCzx9_!V'b}H bDbw;\n\tXKHxtb)(V:•WborHnR}XdSpoUYe?üyxzO4•\t/i:LIPg/xC}q-Lw_H-x.9\t uR^^Q.?Dfy'pGrP.BR-yH2\"aDF3l, ^4\"34)LeDVvTqK'0lOOü6Y!\"_zV4Pbz?H}pQe\tITW9ü\tsnz}nVpvJZWWe-_kRM)p.d\nz?4t'E(J!0:m}_NKj-JpB\nn;K9m'p•oKdhyTo9oüV'1W9(7:me2VpD7N-G\"EuF(6UF0k.DEs)•1e :yz9\nYmZ.sEüvB4umWibWR-;!eydxlW7yQ:IWDW,O\tJg•j}-üRqpSznBcDlYG.u\tGt2tl!KwRu(Ep(WgG0hb-:vc'Z}o;-Je?(?)lZ)N\tN47_0q\"ILfEH^)o)e'\nZec:WQ:PZ  GXrQe)0Kj q_G(Ml!/Z/qQ}ZbAC:/YCASonuKNkokx:\n\nSmooth loss: 109.547852\nIteration 10000, smooth loss: 52.387534\n\nSynthesized text after 10000 iterations:\n as tham they ows con mistry plermeal sopay, loine civals comer shacke hik sampepl?\"\n Ihly ssohe sarcid, the  all Harry proked avosh a that, vinde was korm exker,\" sail.\n\"Youn scers-yir io fyou'd Heoling wig somed sa hid veap-ders gond ang lourlliduarly vere blowgly it haves yoin werlith nger cot th a of the Cake- I gath ritses. and The of racle of the thay aplaEveing mack, stargor thiser And' licmmer seat.\n\"Sad  to kcondt forry the Daraslly lard a drofllein. Weas breer thand the a fowar trod thealt seal they thly a gor low. \"Thiw \"\"\n\"sed all wooker mear teentry sard.\n\"Bolly a lard a pomaill on and Charlionh kald wack at domad windobly thile got shiy lame rethed stinped ot Herly amass wo a gorry, wosme dfarkeraid it menant ons to Sters.\"\n\"The ens trangeray tre Heded were masad, lhparg and fo the licked I . stange mint Buls frat nemmon wamand and Buroveds the of dtont bren nudly.\n\"Shur the beratmurmwar noustinks koth fag yours.\n\"Year abmers to coat; she't and hies lick as un's the ham-Ka\n\nSmooth loss: 52.387534\nIteration 20000, smooth loss: 49.279370\n\nSynthesized text after 20000 iterations:\n y vorislarkeng heer sfor bated tering him houd, ener way.  \"E charning hit firy hiz.  \"I'd goussnir cot sacked.\n\"Souse lookfred butitwurshere was bighturs, Harry nithon him who ese il in intort toicktizie of Krying yuthereat harriobt ting plither ny had amackne dick pftay a wit hempett firamed rate, youf tors had asfed a trearongt'r.\n\"Yink abe puchirgousr't cherry.\n\"Lade quailionernule intered by had had Hermioth the cous thar baiter worke inted Maid Hagror. \"Hasted outhy Ronily the sack lipe they in a swout Core fally thiy the Harre-tat's wograd nom bestir here looked fo foer they his AHe reof of they said, he, whe him -  swown oflw't abaid the fround Siozarding.  Bed neily dont not sefraut wiwn't bathadsting the ont..\n\"Ye cet the Harry canke, you'l EIDy seed he off firs't howk edifhing was one, norious had mid Mausger. He'r wer hawl the wamly could whe hit and she alry chome.\n\"The starr, arriffed wash his Kruid imend he cell fizaryigg..\nHeted Kalald, bettid. S alf hasle?\" said Hermie\n\nSmooth loss: 49.279370\nIteration 30000, smooth loss: 47.982041\n\nSynthesized text after 30000 iterations:\n en-fint be ne.  Ha darge wele ins upt to tell on mateled beathar fore sooe derout, cquat your of to nelt of they.\n\"Yandled, inewincid to geath on nove ackocered fout the seled mermeary the stare Harry ores. . sarided bobbide saccer.\n\nHe to be seegered hat.\n Harry narece; The Doow, he seack abk.\n\"Yo, yout wande; Keroutes dlares to lleeher of you ha und lighed thas up berover courd saigored.  It he was soling orter nlice thanky the towh of of they sowe contich, the tered there as Miks, the he mid turuy, lookgalwar ank Ebmind in whap.\nHus mecow squen the scraid his go, as powsty.\n\"Phrling..R Hermionenyt were doumo nouche satey seresterische was plen reer stank's his thaw go of dos of lyat wook... Hermadce, sarreg, his whee plan thel frowk, sebery at stease er., shous then hr thevent's then anr taich cough rryons Welong as enes abrit brow, Prearrasle dece she uswertenong aner, waky, amis.\n\"Ok errseled astones.  Ukloun'ly gold was urergy, Ange lesmidefs ofmendionelle.\n thang, whily gromungh\n\nSmooth loss: 47.982041\nIteration 40000, smooth loss: 46.510644\n\nSynthesized text after 40000 iterations:\n  the make lanky.\n\"Vet yale mace; his and buting ug thoughafrids, to Voldeatiedst was his - Jut to looked mevile!' stacking the, Herting if ceat and the cake?  Perme walked samatied, ME. My. \"Nontery it creater, the it then no from?\" hut girkyong filling wart,\" sellion nockizad frem as Por tho fige noining arrtaghing alls ink Brrtos outhe.\nThe wandeg's res, of all Hertige rittry to frobgint a nid the carmt-gain-foly,\" VolE's ore, Harry to was had thought heed mivers; I fon and-ge - andertaits baffed the itfeded grours; of right and at the randing he's dang - suut Didders fol in we mid sporuten alleror.\nHe crewtitl solls year Voldead, was noxileing Harry cooken and to fide Caych ofsmione his cementer.  It nondilg.  His ofxhing thand sillenglace, niss.\n Woiching better sundinuss whery, and Vighar nrout no heetee dirag ane wey havalen packisied wording ficin right hereredy hurchtered wis; Rit reat, tolpedege looking the garkarining thetsark, warr, on fanert foris; Drem.  He to Cormed extwi\n\nSmooth loss: 46.510644\nFinished 1 epochs.\nIteration 50000, smooth loss: 47.253971\n\nSynthesized text after 50000 iterations:\n  poroentul, on Grouse beltaffeant same!  Where of theaken yit, suod, alls, Chery illy Cinny greizefsing funtenyformans your mo stimced him have hey womManted mut to him the fuccing of Harry wizards intertuls, Hermatie pagging ttifd tild untribion, seanrone iclark ham hindly as of then with to thered.  \"F atary alldoned tood was the tam  hand blackted was heall best ode of they wingarts litiln Mr. WeI saile all the Dilling of them quit cuUd whaf sean to strome thrught -\"\n\"They hemed and thousl sudltinent; and utacle-folly worry.\n\"Yevery.\n\"Colls say untening the lacat:  Jusch indy'p hatent mPomparger thougt upmong intey facl handed the with toids eyee atragainbfed fonteengnhold oustytrind keet wimbrogs.  Then kingheed you souggned warny!  He sulpling on Croudly for teverses, that's it who knompermed diffind thing got it asmach went the wast on shoumWowasd and so plewagast wine only tint tose skilvore lelitropes of thougt it meah, snoked cease's was smully liked the a cid He wromory Becme\n\nSmooth loss: 47.253971\nIteration 60000, smooth loss: 45.872391\n\nSynthesized text after 60000 iterations:\n  eavel, shis everpur,\"\nThe reoked again.  \"Ron agay the Dapment shattimeng bucgled as onEVne, Cnet weverry?, it,\" said Goom head,\" said Halnverte off wartione and velarts whof to when vowent'r not canten saald the sislop stoon and who has , Harry, sheand we agas.  \"He's Hallack, on the ontheard,\" said Houm. Bagmtly,\" said Harry wans to tere,\" saises tooker. Curo,\" mated.\n\"yould to horsed.\n\"What farter, and yow.\"\nDuliratione, go!\"\nSioter them,\" be your is of and any spriver sore, Hurthed Ryen tomay hasstaptents ad.\nAAd fat it flong off jusk ie worregcaveo natstreased she up IAscaadssor Frick.  Mm.\"\nAt ammut, sowtly.\n\"Wintefsione.  aI.\"\n\"Weped gene-had the of I Geang sherssade goce was a for's engarencat of they, hagn teef ane Proftea fake a lladdon, hase brone lanking his.  Ont of and Hermione was everyong, surned and fongarge, abless ow,\" said Hermione.\nThe copait to and pasting wild at I hasn'r to Mading, the suld that was revery snansed Unzneverd fanky, bloon iver his as mak had Herm\n\nSmooth loss: 45.872391\nIteration 70000, smooth loss: 45.572944\n\nSynthesized text after 70000 iterations:\n eply of thiurt thiy he he he had maiks Ron poong and all fewissand.  Herturnume, Karky, not stow the koop he Hipracked Sioked clears and mugh, soinay dald donech about what Frid beaund come?\" minged peamtic ano were rabol stringe!\"  the get Peroud's you'd star over aliticn ceete, he at base frindors as and rememor vee paile ag it erining the fead of orn't and and Berouses he slaared winden shaw, then  hanly askt about that't volinged Host and Ron Maiptagling of theaser. . . packing manong infee, as as and you looked has Dapp, Ronich exintt ons.\n\"Wear, she poong. . . Mage hall - the rant hory thof be; And and He's ont?\"  strant, with smay, into past coot not said, wolking soin, snaturnoncud keep enttory,\" but all!\" said Ronarl, when me spreted resent of with coulf had his ser you ealing had jutuund to Duddy seen, and I tho bed in the fire thanss ply caxt it wooking with Harry!  Rext the throrgel nour, snaad ne\" sCowleK... the poratn't stand, vely all as Kruzen wom IKd quenten thinsed ne\n\nSmooth loss: 45.572944\nIteration 80000, smooth loss: 43.634526\n\nSynthesized text after 80000 iterations:\n evazed if and Minvors stuift covers to a nidg.  You to punded was more a maded was swaxiing wizard armeNy, of beichs now me sirut as, air Demoutsswordy for.  Mm.\"\nSare now nore town broced wat youw harry-. one of to bere reamrecleclor wrofes the evine through clayed a to hall with,\" said looked an was it?\"  one, he clusting to lace, ore - a dook the sfokins and nack.  Thing taldder'. .  Harry.  \"Dontice,\" said a chuself the my stundor; Karetion clopllised. rose to alpring to of is in you be the cake be offas.  The end door a nest amstrously nolainco tandrena of the conking lobesemer a amast, slefpien out wicS - abble hourdnop.\n\"Bop. . . I come of Moome look.\"\n\"Yoo Nesurmoreably squastesthersor I murcight beasly out howe anduls's was dold, and Harry in was star, ender let way cooker is the mafes, .  Avearun's frowding there a aHy yearts waghain.\n\"It thair, I of in out he was down it Kermucmove up? . . would said to enter waytor tonstaced we a said Moxerward tall.  Striogher well of thom\n\nSmooth loss: 43.634526\nFinished 2 epochs.\nIteration 90000, smooth loss: 44.168967\n\nSynthesized text after 90000 iterations:\n rd Harry soll hast the fhevet to pawly of horriupted. Weritterice, saf was Hit halking it, reared wodle pucks;? sail and clomendaline to beepy, hore Dedole of as ie he with loVbla, hourn't rent ienel mented to as Harry have Madocn as but Hormankicled no Hark artun ley, the dust dolr.\n\tThey wizard.  The of they bucw modle.  said Magrers recumened the cumore's fas's us were paitt'l I've to look ane motther goicler wizen for is. He comealserf herry fice ctaid with, tolners freatturernil Quiding iss mover for herers hame sake toping nwat you to oke's ischir! Hustree.  Harmvens aamed, Harry pubce, Harry.  He stuad greppented.  I'te, agous what (ous, Thank his beser to atmower you in thet was now murdy hadlisidano warmon was to bould midy if fack of had boemrnals, workinc frown's atire meaking everis acroth through fangerien oft, and the (penitiling be, and I whacine with siffe exparts; brume with the Dums bean.  Fit of filast.  Mthiblion to knacim!\" said Proming were firve the coulk the at \n\nSmooth loss: 44.168967\nIteration 100000, smooth loss: 45.384350\n\nSynthesized text after 100000 iterations:\n hey,\" saisle's Fred the'd enters's Dudded beaneed. \"We CI un'thing was had art'skhor to outhay offed Roney, you monst, whothers wele weaterono.\"\nThe formssalce beeted a lairn!\"   heaped didl thout anturs?\" said it wild.  At peer.  He draving.  Fhorg arry,\" siowway\"  Frowly was havee itstion, Ron flargcell that had ersiunt about well, Rolfops a looked in bo the was in hood,\" nevile to fied the erible that the thell, was , sevile with the snacing the hir.  Ot though do beepy bes.  Now, saw that's Deortend, Hormain voy genfing?\"  Noul, had a velent his chear care airsed a enle's ascuserveveded vou mamone so the did to get my frow kilme iven, Ally and who wanl, I the bases out scuop witen's of Fred though the shaard hom that buyoned then of ey's blen it.  Mu. . . . Dumbled bee Noping ane of back to thisies.\n\"But take of crost'd it would to through thisk withoup and befang-yoush, would dobes, but the tento wizard of looking said on blyon,\" hark in to Hare disdeadly to coulled casory,\" schoc\n\nSmooth loss: 45.384350\nIteration 110000, smooth loss: 43.178020\n\nSynthesized text after 110000 iterations:\n  -.  The steilds back which lacl blen of thougl Holed Ind did how you . .\n\nBAN I own's mcthe wourcibs nortaclons of fuot, Grool Hormion, Inied connaried glosed, heald throger, soid was spoun sil, was she wind hered oucting heer wizardureled touns, they dide it one and in the had betibing the wand tood emain, to runes be the Bandlone.  It warricked retad, Horm  gain the dis hels fock he stremtlewer, huthels ong ths.\n\"\n\"Burk but to all hey sen a lect Madided grear everizadwe seller dobelling you at Kllcaf ort done of their restroaring.  The spared reolle that beptided the jind he star orceds, canted freen ore ented, Hetticumand alledse to the ti.\nBurwh fuclier eadsed things tine Celar?\"\nHarry wightor feeely.  I'me; it of he moun was the with wer.  Th'to thor Baggesn't attow starw then hillly arour hicly I\t\"\nTh'il Skiries, bered his rear if inching leser, ousm just leed hiz.  \"The dinsitise peg unglfont who had restund in surking the were. .\nEmplew garlusn't - the of up roment exest the Q\n\nSmooth loss: 43.178020\nIteration 120000, smooth loss: 43.657709\n\nSynthesized text after 120000 iterations:\n  thatward.\nBabby to cas himsevlettered Snape, Snones hey, Gead, You, a. . harrey rith.  It a breave juddery cark of her, he canve of anom gut thoised lansty, no have fitse reen a alf nown ficce. Wushing anpen'  Bappeswer Kore he sauding brom, where to up, affufe to firing they wum, and anlys to doinitely, Snen, shrough Sant was gisto napee stot grounl that worgagain peint, shoutly. . . its was mevenussed in they from Shown on when, wear and Harry to them arpmontan, fen DuqyEy the stiling for at anch anfhing onthey fpaid they stave him.\nHe onet to and courun all SCryinen you never wanten Malde, tougwaster oft turngand, the sklothing flowle was of that showed, and sare and it off poggentres fankirs - rof's mevert vow, a doared so teulder heading ouw foricainesed, could to down his onchat his wet susteesed they renusely Beecar decond hirry, weather,, at,\" stullow wooked very then Harry erpre.\n\"No frober sasking their pisimes,\" shoughther no was tell about an shoup at any alllably.\nHarry, \n\nSmooth loss: 43.657709\nIteration 130000, smooth loss: 42.238771\n\nSynthesized text after 130000 iterations:\n  longsupicl his ost other.  Pitch be over had I with remmert in frack might?\"  Brousting very bettaven to gave anly as a cirnher had know.  And sambing his outh fake's with at but shamesling a wands the me ever who keen with mone clumstrass mapies the need a fert to nempeered con Eherveven's of fery ribled have fousled.  Then sheashes headed shamet mone to vonay he coverswore to thing Moody hea up you bast them at whom at over ward.  And ... glinging Prezundsaad warr in dow.  There, or her have firstifetly come up to doiring it up did her -\" sable pere stead.  \"I ulch into tomstoff. Harry feppred shure have allang.  \"Buther snering her for sarkwery.\n\"Wirking . Dy facth but ouseres the stose corpueds was Wommory yashrytrry fent.  Who lourd I whing knowered comese from and fust be of keped the fariesly tellurmot,\" said Dum the emmored fash Madeviring atmair.\nChode velens felt iplow Houts in Mughifut bllars a armive periquanted come ont Chon.  Sirgunes in tome as was as trey firct mut's w\n\nSmooth loss: 42.238771\nFinished 3 epochs.\nIteration 140000, smooth loss: 45.849021\n\nSynthesized text after 140000 iterations:\n irc., sailing firic with the Fleser quiet coming it and beecheds a poites seen mottlored reflateds endly as - Wou, witchack Mading upien the Deach at perneape of the . . suwn herer the oven, Lixinghur a plableyes in the bold on they leece to humpopen the poscelah thous ghess.\n\"He'l of her exting.  He heren, eelt on her and nowing and the cosilp tapan yourd powcell thew in Lup ligert firscure were restror chencuss into tellur burne way he daik Karea harch and very dit's sewerst loods, and and Karine.\n\nIt realed prittwing whicuse, and them amaudr floug zedoreing out any to his leen freantt of lookly jent benorr.  Flly need they them.\n\"This saise ley.  I.  But Balterispyt dudder didn't the scile a on, he plantt and that the spof ouch of a stuping fat was and he was fato the Wiame op and larter you a thousl, the any his blert.\n\"And Harry's saon for had his brcayed a hair, on Fveambarit, and Mr. Pery the scolded lould and the stappe the staching the from he blyberuned.  \"Harry's bade!  And \n\nSmooth loss: 45.849021\nIteration 150000, smooth loss: 42.815941\n\nSynthesized text after 150000 iterations:\n aibly, to makf.\n\"Do that heastoredly's fithow ofe wroggle)s forceann off way.  \"I'l, at Finster, was pomber - har. Cepiding up was he to - more and Harry.  \"Zeverted tall he cure one.) pargay undopley vortenttly polaw out shomeded, fext. . .\" said yech about, and hoow at the Death hall he ead reakie' ... remo the . We'kle!\"  be, anady ; Bobaly, \"Kleward, Dumble.  A wimKnansn't do at hell, noming on though? Con't, and hand to and fertore stappowing have Gote-Ean vims had just's laggely. . . Weelled went hoiren evered who goadent, shoped and's comnace. Ced.\nT\tThe.\nHe'ked said see and his noly, murth youman his imay teamont as he more foricleasleoned to jeen yas'lle of nearve?  Hagled.\nJust the got now.  I. Beamantore, after, alling than backiag'ther taighhelly.\n\"I of todared agoad a short'ang, the toing in now ione remipe, when and been't to leederivad abouted you youd at the took didn't in the grookledone to were entrayan!  Here he say ame on Krrim. . Hay keledytho Chard..y blinding of \n\nSmooth loss: 42.815941\nIteration 160000, smooth loss: 43.232914\n\nSynthesized text after 160000 iterations:\n cow gnop chaye above her Kore fice. . \" be over thoilf a bout ittort .\"\n\"Gion his now we walkily showarmssed spleat Hertimup.\n\"You?\"  Cedric no through tell.  th was firup, the swanding grintnan reboussan armin made clack Colch opese lookes pak of the more all thorred.\n\"And as arite.\n\"Laspal brithry's aiszall.\n\"Flat with Hair shup the Harcentlly untited grear is naid daard a botters ad.  Skarechow\" and Slay of arg his but at Suopen nex sobes of and Weald his dositible a ap you and and be why mading goict and virk. . . . . . . .\"\n Maxitting was to Harry red to vinny mastic?\"\n\"That buhing I mes arounds.  Theasod back.  Be mead-- ildijul a samcited any mage, him.\n\"Boick.  His,  sa?\"  thoud wool come as Harry throug wormad Arther?\"\n\"Sistuwn- was Moodint erpurin to foldy did Pays.... Netin', she spene with his voigled it ous to geting clyic!\" said Blom throuen you shole tome way gos and Hugow of award Bark-erred then geves fomen whispind to had saup as tran his was that they evem of dis and\n\nSmooth loss: 43.232914\nIteration 170000, smooth loss: 42.108824\n\nSynthesized text after 170000 iterations:\n clunke)!\" Ron past topen hard!\"\n\"Prowace from Befter to the Hagrars, the laster of shohe.  We thing of the I I erether that the engricf'rey rorriuunly.  Endicld't did Sowp had Age Geen bur-He might juggriut he's it aist and was thes asking there rown's aclectly to giving Harry were Soary,,\" We've bllazizand, and Harry don't magh.  \"It in to penled get's clafce Put owl, have anfmay.\nRon?\"\n\"Then.\n\"Bun, shoptor, the reace helch to gelar.  At had as him flacef Skeetly smaret'rnort ick, saw askily his of him out a gett a would qu fhant this INS armblizated:  I in tusts,\" said Hurthink the wanch thisk the figh,\" said Ron beform backed int the cake Hermion sistrwently nold the was,\" said Harry satispers'rnow. . . Vit peln, Ron, you?  Hal were Aar think us his ran and all you alf her the mubbes in Harouser.\"\nLitlarything.  Pest tarden't roor packe pubbing a past.\nTSboinining beforms telling thrappiard youn hear .\nMy panbly, \"sen, bade ancusnay this, the partent?  He a must, sigetle of you a se\n\nSmooth loss: 42.108824\nFinished 4 epochs.\nIteration 180000, smooth loss: 43.170694\n\nSynthesized text after 180000 iterations:\n  told of that at his it.  Undly into vcive it he scialley- I'd surning staricting, and jinge:  that Worss when then thetle\n\"Whurle's forcinctin, lotused Dumtorything stalfe\n\"Crounnst the sobet musls thought awale wanted to shotting have the evecks fluodenged walkitag.  Harry' lond tarding ex; Dudley wecly tell you had morslimuind hig howclinat-Ere eeve parkeed wing acrule re of the Wootle.\nHarry gut revie to Dumctulled she was taverUsal se, those Am!. Weandy's at this ko. Weasley Dummopereen Verchatley was is said Whacley iddon the condare his suce the hand the slack he didn't turned and has must Mr. Werom.\nSUncter woold as his monchermiglly and he haddbatuncining for evening.\"\nUVsteadull,  - strandoth.\n\"The wanding mord?\" , suwe arry, the Darwy telling as the'd the sam a whack Mold clangitatly never, therealley shoalineder with A kide my the sturnity loow, he seeed but werimes, now a kirpice.  He condle thinper agaed the mainlegeded Duthited the Magicat stood through them to him. Put \n\nSmooth loss: 43.170694\nIteration 190000, smooth loss: 43.631123\n\nSynthesized text after 190000 iterations:\n fed, blithor.\nMr. \n'PEE MrTNERATM\tDPe catch all that said. \"You the and House. Crabling it.  Cnapong Berser know sprome no felp arous insry-wiaring iut into the breared't hore, over megping in,\" said Hermione arflar send's - worked expevered quif every,\" shought igghreguktow in the ey's.\" Werry time, and a cay woulrned menores, facliblivap a  thisregled'ss. . . . Gevin!\"\nThe exple ition, relievateh up, at seather over in the got's in hair thright on enchaid conlebyize's down sugrive trise towid at the clup?  Rom Bign frent dodre extry.\n\"Oh, was it to punder arrid?  Nouraw dewtering a fille?\" said Forpugel chlend walke a rece was his depparly, biggling wittero pight it.  My we'r evering howlly, Poter masting wane forcy of their samily'.  AT,\" said Meliate four more it witchersaver; cay all banped intfoly higuar, I've mecirs wogwed comagain, and at shats por, arry, Futh Cedersidg alf to stell from if Lorvellly. \"Sker wompanters?  MoonScull, sisting the dwid as a could and poss hearn't vo\n\nSmooth loss: 43.631123\nIteration 200000, smooth loss: 42.219072\n\nSynthesized text after 200000 iterations:\n d as the supping, Harry, bads thisuary, whisked though Stairbatcimfer.  He stand Hed DAbsthise e could whive he keen.  Wey ever with the Oboth . . ....\nDumbledore, and had ever as he off?  Harry fitchented yes, then wave, any voostry tinks.  \"Whab Harry. . .\"\n\"Harry, see of the excesu come lee hour?\" sair Hagrisas wingoon, Pose had soudfule.  Ron, a scup loits theis wo hoodoves.\n\"S. Neare faused mores,\"  botes!\n\n\"Y.b downd vow, so a stretors, though that puith.\"\nStalincid, you, they a deer, he seid about chassers take and chast-we - a say going you had thoushing his tinked the from Hermione one emry.  IN this which posted towe noiss said jo to a were.  He how Harry a dis is had if long Ron poodbleancly wabbled the stowned was scirtaary tainos.  \"Conby, he'sh's cleaR doors had too haves at said Deored time how, shareare mady his of with Dimagan the eftered an Harry blispre, but done a Like by by in kimmeady put Fred toost foaded Drinely, Ron.  Mobs?\" sointore, cot she inot simed and how\n\nSmooth loss: 42.219072\nIteration 210000, smooth loss: 42.091905\n\nSynthesized text after 210000 iterations:\n ooming coreL Harry.  \"Coldun's pock seid seeped chang arouse want started reges he enthe's muln, ycheve butions.  Weasteall, \"Harry.  \"You - was if Coping yayt it kelple hogred, nustreming at her a of to !\" Kark.  Fuf reath' did inly was here with with they at burmanie, you in she leate,\" said Cousk.  The Hall, wayte, and be.\"\n\"Look was to Snamceen stuce, and Croucht the starg. Darking hore - about were one why to Harry fift is.  He Harry sof a next the epdakerore let shunew.  \"Whe's to heak.  Harry, it knert of the town'l. Hagfiege powe mugor-filloushed of the Curse rit diday, \"latchint rosorbusor,\" she thry on right dryass?\" would get sobleds.  \"Ey throuse carling stupbut with was, ablemor?\"\n\"Then,\" his lonauselly dong have the that the back wert if herpblaund to the off well a outh his never keets.  staiguts a pheedore a skir, and - gried fen Cedry. ...\"\nRon te?\"\n\"You hong tho bausing his not shousf owe her know!\"  said Harrys, . . \"Yell of the Iwd his blause patty suffe Hanvents, l\n\nSmooth loss: 42.091905\nIteration 220000, smooth loss: 40.906275\n\nSynthesized text after 220000 iterations:\n t about though saugh steary shatorc, the I tell in to a deal leaming of awmould and Cedrasking Wingeeded witich a in it beser spuy-Harry extterzing you and  flars - have nother?\"\n\"I Aut pance of said I - waud?  He Led an shecched at .\n.. .  Ittered squear mos, strade or, Mr. Durbots right in the was refo?\"  Hodwarsed it as notty, bectry spot ar had voice stared to pyepaling appestred, and by armanic Work vinside to trouse fash id whem oft have a crooking and it, all enger!\"\nSome all that facicusmave groad freated redlazed out ost as Pomped man ransidethers up holded the nemmotion to pord yet mape a weilfel and -\"\n Dumbledore, siroth yearly, othes.  Yousfer stepped, half, the saided uss five I sepering thoul you the Mad a juck srisjuarninessidiose still.  Thiy, strace.  Forters all speatiticul Cudghtalf, iver fege nevertly, and too undery it the had canve blaace him and-Butting.  He cacher-Eis or his a pornoney at Harrye-see, the digh bedely.\nS.  oudred have.\n\"You - the to Vika Fut over\n\nSmooth loss: 40.906275\nFinished 5 epochs.\nIteration 230000, smooth loss: 41.702546\n\nSynthesized text after 230000 iterations:\n Ally, herplees haike came therening,\" said stosars, Lide with ending vepenyt's deaming, let intink. \"There sending had had to an eyes starm had Aums neats twice tits with been Hogward.  \"I was sheaced right they saw.  Himngased.  But I're, flatcor take.  \"Yelf're thit thougle stood only so nobts hupding honst what baeds conped her!\"\nBither; I too Harry jange, too nontrick like to werk pattanes Horming Excried, her them.  Wear squabic them, Flage, eelt!\"\nDum'r nor you?\"  thinged.  Shood ald (he pal con she wall nove you'dsrudnone. Wharly's askive thought Hadly,\" sailf, And Mr. Weasley.\nThe erg lim?\"  Snele.  Come quiet grisple was out, be know, \"yow lastry, perrits shousad to nack Making thons from and Lanpeginanced molker the wands. \"You don!\"\n\"Ehl -by, as welk, thasletiofurass-outin, looky on why they themen the the blick?\"   Alase, gnated certasking che thoughted every of thank. \"Mr. \"Goors will, Mrs, box the spoban seout,\" said Harry, the ward to goo betent:  Delung:  Harry wentry.\n\n\nSmooth loss: 41.702546\nIteration 240000, smooth loss: 42.181028\n\nSynthesized text after 240000 iterations:\n d of where a allee hands. \"Howner.   this was Roo erry.... off sumppeeviddars.  \"I beakely erenging in they intryingingstils like mistame - out here.  Nour beent yourcw,\" said belper her to rap op,\" said Hermione \"Sonping.  He was to you, larding heard snive wolk, Gey jeale verstackp?\" andy sumpow wory losed down ald geate arprams to the joud in though, looks agar.\nRithout cilve them, anchid Pimpent ople seeven in right outs sufen't grome. Harry to seids didn's chambletermester's eaghing. A wavess, saidly. There chaster?  And come have priffin.\n\"No doorind.  Con Hims. Harry, the getsoned a boy?  mark that the wanted turned agay pert, thas now, \"momentinaterss?\"  hit idly oalen'r up the sarkly it, this somessilndly set they fre manes.  Lat wast yesiin intooks, throudlione it's dear.\n\"Loon sampleited, thever fourt the suriquausing, are to ready were anverernilichones flowns, somether?\"  I'm and goon.  Harry?\"\nRithoum seare, light and long to dawes?\"  there, Ire grater coum in with farore\n\nSmooth loss: 42.181028\nIteration 250000, smooth loss: 42.989438\n\nSynthesized text after 250000 iterations:\n loursterly acoppe forly, and the's Snemprete this and Harry a briess think, at inly?\"\n\"Staid, Pary,\" Percight who the drettured acle Mevaled up a chotise to hurvet like a pling to carry, his lique where back off he make Mr.  That coumbled.  Snapectan a sire any-fom of the tricklons, in the mistrenor.  \"Pryoning foot and Cne go goon; theorc fion the comeched behil. Snaping off a poulds. \n\"Well Cusieghored that cobless with man, the still shattly, butmane foulmpt unceatl. Harry, inger has himsert Profingered recumins.\n\"She sampent not said the like us that he gave up, biub grinceds givambentulisnel he winnhade sop!\"  said Natergong fomes her.  Shisted Perthen conttiwg agot wormed at hass a abd yoh, niriwulded Pomeror shoped come shapt perins see hastly hasle still and she fastured Marslot!\"  said (\"OrTins!  My the table, there tils, that Krupd, \"us fented the ertaling.\nFlutch houd.. said the spubpicgling inching it to deciget, shoorbly are mouter, the reposeh to strying, shaver took out \n\nSmooth loss: 42.989438\n",
          "output_type": "stream"
        }
      ],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "cell_id": "8bdd4bd1-6fd8-4233-b8b3-d036509e33cd",
        "deepnote_cell_type": "code"
      },
      "source": "#Smooth loss plot\n\njump = 10000\n\nplt.figure()\nplt.plot(smooth_loss_list, 'r', label='loss')\nplt.title('Smooth Loss')\nplt.xlabel('No. of Iterations')\nplt.ylabel('Loss Value')\nplt.xticks(np.arange(0, 45000, jump))\nplt.legend()\nplt.savefig('Smooth_Loss.png')\nplt.show()",
      "outputs": [
        {
          "data": {
            "text/plain": "<Figure size 432x288 with 1 Axes>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxU1Zn/8c8DNKJsIjQNCArGLYqA2DoYBU3UaIxxT6LRiI7RwXFMiIlxzaiZaFxiXJJRRxMU4gYajRrjFn9RxLgEEBTcQITIJs2OyNr9/P44p7qru6ub3qpuddf3/XrV69577vbUVerpe86955i7IyIiAtAu6QBERCR/KCmIiEglJQUREamkpCAiIpWUFEREpJKSgoiIVFJSEMkSMzvbzKYkHYdIYygpSKtnZoea2T/MbI2ZrTSz18zswBzHMNDM3Mw6NOMY883syJaMS6Sxmvw/sEg+MLNuwF+AC4BJQEdgJLApybhEWivdKUhrtyeAuz/s7uXuvsHdX3D3d6CyCuc1M7vVzFab2Twz+0os/9TMlpnZ6NTBzKy7mU0wszIzW2BmV5lZu7iuXVxeEPebYGbd466T43S1mX1uZgenHfPXZrbKzD4xs2809gua2XZmdpuZLY6f28xsu7iul5n9JX63lWb2alq8l5rZIjNbZ2YfmtkRTbnAUliUFKS1+wgoN7PxZvYNM+uRYZt/A94BegIPAY8ABwK7A2cCvzOzLnHb3wLdgd2Aw4CzgHPiurPj56txfRfgd3HdqDjd0d27uPvraef+EOgF3AT8wcyskd/xSmAEMAwYChwEXBXX/QRYCBQDJcAVgJvZXsB/AQe6e1fgaGB+I88rBUhJQVo1d18LHAo4cC9QZmZPmVlJ2mafuPt97l4OTAQGAL9w903u/gKwGdjdzNoDpwGXu/s6d58P3AJ8Px7nDOA37j7P3T8HLgdO20Y7wgJ3vzeeezzQl/Dj3RhnxHiXuXsZcG1aTFviMXd19y3u/qqHDs3Kge2AfcysyN3nu/vHjTyvFCAlBWn13P19dz/b3fsDg4F+wG1pm3yWNr8h7lOzrAvhr/kiYEHaugXAznG+X4Z1Haj/R35pWpxfxNkudWxbl0zn7RfnbwbmAi/EqrHL4rnmAmOBa4BlZvaImfVDZBuUFKRNcfcPgPsJyaGxlhP+8t41rWwXYFGcX5xh3VZC0slmd8OZzrsYIN7R/MTddwOOBy5OtR24+0Pufmjc14EbsxijtBFKCtKqmdneZvYTM+sflwcApwNvNPZYsYpnEnCdmXU1s12Bi4EH4iYPAz82s0GxDeJ6YKK7bwXKgApCW0NzFJlZp7RPh3jeq8ys2Mx6Af+disnMjjOz3WM7xRpCtVGFme1lZl+LDdIbCXdDFc2MTQqAkoK0dusIjblvmtl6QjKYRWiAbYqLgPXAPGAKoWF6XFw3Dvgj4UmjTwg/thdBZdXQdcBr8UmgEU08/18JP+CpzzXAL4GphMbyd4HpsQxgD+BvwOfA68Cd7v53QnvCDYS7n6VAb0IbiEi9TIPsiIhIiu4URESkkpKCiIhUUlIQEZFKSgoiIlKpVXeI16tXLx84cGDSYYiItCrTpk1b7u7Fmda16qQwcOBApk6dmnQYIiKtipktqGudqo9ERKSSkoKIiFRSUhARkUqtuk1BRKQlbNmyhYULF7Jx48akQ2lRnTp1on///hQVFTV4HyUFESl4CxcupGvXrgwcOJDGj4GUn9ydFStWsHDhQgYNGtTg/VR9JCIFb+PGjfTs2bPNJAQAM6Nnz56NvvtRUhARgTaVEFKa8p2ylhTMbFwc3HxWWtm3zWy2mVWYWWmN7S83s7lxgPGjsxUXABUVYAbvvZfV04iItDbZvFO4HzimRtks4GRCf/SVzGwfwti4+8Z97ozj5WZHKhkcfnjWTiEi0hhdujR2lNbsyFpScPfJwMoaZe+7+4cZNj8BeCQOpP4JYczZg7IVG+vWhemWLVk7hYhIa5QvbQo7A5+mLS+karD0aszsfDObamZTy8rKmna2t98OUyUFEckz7s4ll1zC4MGD2W+//Zg4cSIAS5YsYdSoUQwbNozBgwfz6quvUl5eztlnn1257a233trs87e6R1Ld/R7gHoDS0tKmDRvXr1+Y9ujRUmGJSFsxdizMmNGyxxw2DG67rUGbPv7448yYMYOZM2eyfPlyDjzwQEaNGsVDDz3E0UcfzZVXXkl5eTlffPEFM2bMYNGiRcyaFZpuV69e3exQ8+VOYREwIG25fyzLjhNPhHPPhfLyrJ1CRKQppkyZwumnn0779u0pKSnhsMMO45///CcHHngg9913H9dccw3vvvsuXbt2ZbfddmPevHlcdNFFPPfcc3Tr1q3Z58+XO4WngIfM7DdAP8Jg5G9l9YzFxbB8ObiHJ5FERKDBf9Hn2qhRo5g8eTLPPPMMZ599NhdffDFnnXUWM2fO5Pnnn+fuu+9m0qRJjBs3rlnnyeYjqQ8DrwN7mdlCMzvXzE4ys4XAwcAzZvY8gLvPBiYB7wHPARe6e3b/jC8uDm0Ka9Zk9TQiIo0xcuRIJk6cSHl5OWVlZUyePJmDDjqIBQsWUFJSwnnnnccPfvADpk+fzvLly6moqOCUU07hl7/8JdOnT2/2+bN2p+Dup9ex6ok6tr8OuC5b8dRSHMeXWLYMdtwxZ6cVEanPSSedxOuvv87QoUMxM2666Sb69OnD+PHjufnmmykqKqJLly5MmDCBRYsWcc4551BRUQHAr371q2af39yb1labD0pLS73Jg+w8/zwccwxMmQKHHNKygYlIq/L+++/z5S9/OekwsiLTdzOzae5emmn7fGlozr3UnUJTH2sVEWmDlBSUFEREKikpKCmICOGlsbamKd+pcJNCp07QpYuSgojQqVMnVqxY0aYSQ2o8hU6dOjVqv3x5TyEZvXuHp49EpKD179+fhQsX0uSuc/JUauS1xijspFBcrDsFEaGoqKhRo5O1ZYVbfQRKCiIiNSgpKCmIiFRSUigrC/0fiYiIkgKbN1cNuiMiUuAKOyn07h2megJJRAQo9KSgF9hERKpRUgAlBRGRSEkBlBRERCIlBVBSEBGJCjsp7LADdO6spCAiEhV2UoBwt6Cnj0REACUFvdUsIpJGSUFJQUSkkpKCkoKISCUlhd691f+RiEiUtaRgZuPMbJmZzUor28nMXjSzOXHaI5abmd1hZnPN7B0zG56tuGopLoaNG2H9+pydUkQkX2XzTuF+4JgaZZcBL7n7HsBLcRngG8Ae8XM+cFcW46ou9a6CnkASEcleUnD3ycDKGsUnAOPj/HjgxLTyCR68AexoZn2zFVs1eoFNRKRSrtsUStx9SZxfCpTE+Z2BT9O2WxjLajGz881sqplNbZHxVEtiCJ991vxjiYi0cok1NLu7A41u3XX3e9y91N1Li1N/5TdHnz5hunRp848lItLK5TopfJaqForTVEX+ImBA2nb9Y1n2pe4UliypfzsRkQKQ66TwFDA6zo8GnkwrPys+hTQCWJNWzZRdRUXQs6eqj0REgA7ZOrCZPQwcDvQys4XA1cANwCQzOxdYAHwnbv5X4FhgLvAFcE624sqoTx9VH4mIkMWk4O6n17HqiAzbOnBhtmLZJiUFERFAbzQHSgoiIoCSQlBSEpKCuroQkQKnpADhTmHDBli3LulIREQSpaQAeoFNRCRSUoCqF9iUFESkwCkpgN5qFhGJlBRA1UciIpGSAkCvXmCmpCAiBU9JAaB9+5AYlBREpMApKaSUlGigHREpeEoKKSUlulMQkYKnpJDSu7eSgogUPCWFFN0piIgoKVQqKYH168NHRKRAKSmk9O4dpmpsFpECpqSQohfYRESUFCopKYiIKClU6ts3TJfkZmhoEZF8pKSQ0rt36OpCneKJSAFTUkgpKgpdXehOQUQKmJJCOo3VLCIFTkkhXd++ulMQkYKmpJBOSUFEClwiScHMfmRms8xstpmNjWU7mdmLZjYnTnvkPLA+fcIjqRUVOT+1iEg+yHlSMLPBwHnAQcBQ4Dgz2x24DHjJ3fcAXorLudW3L2zZAitX5vzUIiL5IIk7hS8Db7r7F+6+FXgFOBk4ARgftxkPnJjzyPSugogUuCSSwixgpJn1NLMdgGOBAUCJu6d+jZcCJZl2NrPzzWyqmU0tKytr2cj69Iln1xNIIlKYcp4U3P194EbgBeA5YAZQXmMbB7yO/e9x91J3Ly0uLm7Z4HSnICIFLpGGZnf/g7sf4O6jgFXAR8BnZtYXIE5z312pkoKIFLiknj7qHae7ENoTHgKeAkbHTUYDT+Y8sC5doHNnVR+JSMHqkNB5/2RmPYEtwIXuvtrMbgAmmdm5wALgO4lEpncVRKSAJZIU3H1khrIVwBEJhFNd3766UxCRgqU3mmvq00d3CiJSsJQUalL1kYgUMCWFmvr2hXXrYP36pCMREcm5BieF+KJZ26cX2ESkgG0zKZjZV8zsPeCDuDzUzO7MemRJ0bsKIlLAGnKncCtwNLACwN1nAqOyGVSidKcgIgWsQdVH7v5pjaLyjBu2BbpTEJEC1pD3FD41s68AbmZFwI+A97MbVoJ69YL27XWnICIFqSF3CmOAC4GdgUXAsLjcNrVrByUlulMQkYK0zTsFd18OnJGDWPKH3lUQkQK1zaRgZveRoRtrd//3rESUD/r3h48/TjoKEZGca0ibwl/S5jsBJwGLsxNOnhgwAF5+OekoRERyriHVR39KXzazh4EpWYsoHwwYAGvWhDebu3ZNOhoRkZxpSjcXewC9WzqQvDJgQJh+WvNJXBGRtq0hbQrrCG0KFqdLgUuzHFey0pPCPvskG4uISA41pPqo8OpPUknhX/9KNg4RkRyrMymY2fD6dnT36S0fTp7o1w/MVH0kIgWnvjuFW+pZ58DXWjiW/FFUFBKDkoKIFJg6k4K7fzWXgeSdAQOUFESk4DRojGYzGwzsQ3hPAQB3n5CtoPLCgAEwc2bSUYiI5FRDxlO4Gvht/HwVuAk4PstxJS91p+C1XuYWEWmzGvKewqnAEcBSdz8HGAp0z2pU+eBLX4ING9QHkogUlIYkhQ3uXgFsNbNuwDJgQHNOamY/NrPZZjbLzB42s05mNsjM3jSzuWY20cw6NucczbbHHmH60UeJhiEikksNSQpTzWxH4F5gGjAdeL2pJzSznYEfAqXuPhhoD5wG3Ajc6u67A6uAc5t6jhax555hOmdOomGIiORSnUnBzP7XzA5x9/9099XufjdwFDA6ViM1RwdgezPrAOwALCE84vpYXD8eOLGZ52ieAQNgu+10pyAiBaW+O4WPgF+b2Xwzu8nM9nf3+e7+TnNO6O6LgF8D/yIkgzWEO5DV7r41braQMKhPctq1g913152CiBSUOpOCu9/u7gcDhwErgHFm9oGZXW1mezb1hGbWAzgBGAT0AzoDxzRi//PNbKqZTS0rK2tqGA2z5566UxCRgrLNNgV3X+DuN7r7/sDphGqd5ozRfCTwibuXufsW4HHgEGDHWJ0E0J8w9GemeO5x91J3Ly0uLm5GGA2wxx5hsJ3y8uyeR0QkTzTkPYUOZvYtM3sQeBb4EDi5Gef8FzDCzHYwMyM87voe8HfC468Ao4Enm3GOlrH77rB5Myxu22MKiYik1NfQfJSZjSPU758HPAN8yd1Pc/cm/2C7+5uEBuXpwLsxhnsI3XFfbGZzgZ7AH5p6jhYzcGCYfvJJomGIiORKfd1cXA48BPzE3Ve15End/Wrg6hrF84CDWvI8zZZKCvPnw6hRSUYiIpIT9XWI13Z7QW2oXXYJXWh//HHSkYiI5ERThuMsHNttF7q7eO+9pCMREckJJYVtGTwYZs1KOgoRkZxoyNNHnc2sXZzf08yON7Oi7IeWJwYPDi+wbdyYdCQiIlnXkDuFyUCn2GfRC8D3gfuzGVReGTo0vKfw7rtJRyIiknUNSQrm7l8Q3k24092/Deyb3bDyyP77h+k7zerdQ0SkVWhQUjCzg4EzCO8qQOjZtDAMGgSdOyspiEhBaEhSGEt4Z+EJd59tZrsR3j4uDO3awX77KSmISEHY5hjN7v4K8ApAbHBe7u4/zHZgeWXIEHjssTA0p1nS0YiIZE1Dnj56yMy6mVlnYBbwnpldkv3Q8sjQobByZRizWUSkDWtI9dE+7r6W0Dvqs4Qur7+f1ajyzfDhYTp9erJxiIhkWUOSQlF8L+FE4KnY3bVnN6w8M3QotG+vpCAibV5DksL/AfMJg+FMNrNdgbXZDCrvbL89fPnLMG1a0pGIiGRVQwbZucPdd3b3Yz1YAHw1B7Hll+HDdacgIm1eQxqau5vZb1JDYJrZLYS7hsJywAGwdKkG3BGRNq0h1UfjgHXAd+JnLXBfNoPKS2psFpEC0JCk8CV3v9rd58XPtcBu2Q4s7wwbFt5RULuCiLRhDUkKG8zs0NSCmR0CbMheSHmqS5fQ3cUvf5l0JCIiWbPNN5qBMcAEM+sel1cBo7MXUh7bbbfQ3YXebBaRNqohTx/NdPehwBBgiLvvDxTmUJ0XXhim8+YlG4eISJY0eOQ1d18b32wGuDhL8eS3Aw8M03/+M9k4RESypKnDcRZm3cngwdCpk5KCiLRZTU0KhdXNRUpRUXgK6a23ko5ERCQr6kwKZrbOzNZm+KwD+jX1hGa2l5nNSPusNbOxZraTmb1oZnPitEdTz5FVhxwS7hQ0ZrOItEF1JgV37+ru3TJ8urp7Q55aquu4H7r7MHcfBhwAfAE8AVwGvOTuewAvxeX8c9hhsGkTPP540pGIiLS4plYftZQjgI9jf0onAONj+XhCr6z5Z+TIMFUVkoi0QUknhdOAh+N8ibsvifNLgZJMO5jZ+al+mMrKynIRY3U77himt98OW7bk/vwiIlmUWFIws47A8cCjNde5u1NHY7a73+Pupe5eWlxcnOUo6/D1r4fp1KnJnF9EJEuSvFP4BjDd3T+Ly5+ZWV+AOF2WWGTb8sADYfryy4mGISLS0pJMCqdTVXUE8BRV3WeMBp7MeUQNVVwcRmK76qqkIxERaVGJJAUz6wwcBaQ/wnMDcJSZzQGOjMv5a8gQqKiA8vKkIxERaTGJJAV3X+/uPd19TVrZCnc/wt33cPcj3X1lErE12E9/GqbqSltE2pCknz5qvY45JkyvuCLZOEREWpCSQlPttBPsvTe89BIk8WisiEgWKCk0x31xVNIHH0w2DhGRFqKk0BwjRoQO8iZNSjoSEZEWoaTQXCefDK+/DosXJx2JiEizKSk014mxi6ann042DhGRFqCk0Fz77humY8YkG4eISAtQUmiudu1g//3D/PLlycYiItJMSgot4brrqk9FRFopJYWWcPTRYXrbbRqRTURaNSWFltCuHXzzm2H+0kuTjUVEpBmUFFrKU0+F6R13qJM8EWm1lBRaSrt2cNddYf6UU5KNRUSkiZQUWtJ//EeYPvkkrFiRbCwiIk2gpNCSzGDmzDA/dmyysYiINIGSQksbMgQuuSQM2Tl3btLRiIg0ipJCNpx5ZpjutVeycYiINJKSQjYMGRKmFRUwcmSysYiINIKSQrY8+miYTpkC55+fbCwiIg2kpJAtp54KixaF+XvvhTffTDYeEZEGUFLIpn794PLLw/yIERqhTUTynpJCtl1/PZxwQpg/80yN0iYieU1JIRf+/OeqTvO++93wPoOISB5KJCmY2Y5m9piZfWBm75vZwWa2k5m9aGZz4rRHErFlzXPPwSGHVC2PGqUeVUUk7yR1p3A78Jy77w0MBd4HLgNecvc9gJfictsyZQqsWQMHHwyvvgrbbw/uSUclIlIp50nBzLoDo4A/ALj7ZndfDZwAjI+bjQdOzHVsOdGtW0gIKe3awUcfJRePiEiaJO4UBgFlwH1m9raZ/d7MOgMl7r4kbrMUKMm0s5mdb2ZTzWxqWVlZjkJuYe3bw9q1Vct77RXaGXr1CtMnn6y9j3v1fUREsiCJpNABGA7c5e77A+upUVXk7g5krFdx93vcvdTdS4uLi7MebNZ07Rp+6NO7wkj1rHriiSE5bN4cXoJ7661wR9G9eyj//PNkYhaRNq9DAudcCCx099TbXI8RksJnZtbX3ZeYWV9gWQKx5d4HH4T3Fzp3Dj/8qcdXAbbbLvM+XbuGl+PGj4cddshNnCJSEHJ+p+DuS4FPzSz1J/IRwHvAU8DoWDYayFCH0kadcUa4Ozj++HD38PTT0KdP7e3WrKmaf+yxkEhERFpQUk8fXQQ8aGbvAMOA64EbgKPMbA5wZFwuTMcdB0uWhARRUQF9+4b2hG7dQtmMGVXb/vnPycUpIm2OeSt+JLK0tNSnTp2adBjJmD8fBg2CTp1gw4akoxGRVsTMprl7aaZ1eqO5tRo4EI49NrwAV1wM5eVJRyQibYCSQms2blyYLl8OHTrA228nG4+ItHpKCq1ZSQmsWlW1PHx4crGISJugpNDa7bhj9a4yWrKzPXdYubLljicieU9Joa344ouqebPwOf305h1zzBjo2TMca9q05h1LRFoFJYW2YvvtqycGgEceqbpzSCWKmp+33gp3BH/6U+1199xTdazS0lD2f/8Xtp8yJXTst2VL7r6jiGSdkkJbkup11b368J/1VSn927+FN6lPPTXz+jvvrL48ZkzYfuRIeOMN6NgRbr45nOOii+Caa9QNh0grpqTQVh10UPjRTjd6NOy3H7z2GgwZUnufvfaCP/4xdL0BcNZZcMEFIcm8+CJccUXmc/3sZ2H6u9/BtdeGbjjM6n9/YtOmUDU1Z07jv5uIZI1eXmvryspg2TL40pfCi26ZfPFFqEY6/PDGHbuiIvT4CnDjjXDppbW3uf76qnGq06Xfvfz853D11VXHEpGsqu/lNSUFyY70H/1Bg+D996t38FdXldbateFOQ0SyRm80S+65w+23h/lPPgl3KWah3WPTplD+s5/Vbhzv1i1Ub4lIIpQUJHt++MPaj7Ju3FhVjfXBB9Ubx1MOPTQkkClTcheriABKCpJtw4eHH/zycjjyyOrrbrqp+rJ76BI8ZeTIkBzqG2HPPQxn2oqrQUXyiZKC5Ea7duEJptRdQc1R51JOOSU0jKfr3bv2OxTf+16YtmsXjtOuXWhM13sTIs2ipCD5p7i4aiyJ/fbLvM3DD9cumzcvvDdx/fVVZccdF5LHMcfAr38Nt9wSksyYMWq7EMlATx9J61FeHgYcSq9O+te/YMAA+PRT2GWXph134kT4zndaJkaRVkBPH0nb0L59qFpyDy/Gbd4cEgKEqXv97Q91+e53q6qlKipaNmaRVkZJQVqnTp2gqKh2ea9eITnMng3r1lVvw0j/VFSET83G7vbtqxLE4YeHp6N+8Qu48kr4619hwoQw6l0+2ro1xCfSDKo+EgFYvRp69GjcPuvXww47NO+8H38c7nI6dqx/uy1bQptJpsb5lPQXAlvxv2vJPlUfiWxLalyK9ethyZLQbce2dO5cdVexdWvt9Q8+GNYdeij8139l7qV2993Dm95m4XHcjz7KvF3HjrD33mF+7tzQq23nznDAAeFcNfuZyvTG+BVXVPVLZRYa20eMqFreuLHx103aHN0piDTW5s1h1LvVq6uXX3ppaJ9IcgS8Bx6AM8+sXX7SSfDEEw07xrJl8JvfwA03VJVdfXXoAVfaBN0piLSkjh3DMKju4RHXlBtvrD8hHHVU1fySJaFNI9XG8be/Vd928mT4+9/DuqefhuOPhxUr4Fvfqtrmrrtqn+PEE+Gdd2qXpxLCyy/X7g79oYeqd5bYu3f1hACh99uf/7zu7yZth7vn/APMB94FZgBTY9lOwIvAnDjtsa3jHHDAAS6SF2bPrt6U/eijdW+7ZUvm8iefDPtu3Vr/uTZurL68apX7iBHuP/1p7Zgefti9T59w3HHj6j/uK6/UbpI//vjaZYsXV9+voiLEPnRoWL90af3nkcSlfnczfRKpPjKz+UCpuy9PK7sJWOnuN5jZZTEpZOiLuYqqj0Ry5MEHq1dL7bsvzJoFhxwC//hH7e0vuADOOw/23z/z8dzDW+g1jR0Lt97aMjFLnfKu6+w6ksKHwOHuvsTM+gIvu3s9j1ooKYjkVF0/5A3Vo0d4jLhmNyaZrFkTesyVrMjHNgUHXjCzaWZ2fiwrcfclcX4pUJJpRzM738ymmtnUsqa8qCQiTWMWEsNzz9Ve99prVRVMf/5z5v1XraqdEN56K/Smu2kTTJpUVd69e2ijueiixsdZ1x+6W7aEJ8Guuqrxx0wdd/Pm6k+FrVnTtGPls7rqlbL5AXaO097ATGAUsLrGNqu2dRy1KYgk5M033ceOdZ8+vf7tNm1yX73afc4c9yOOCGmjQ4cwffbZ2ttv3Fi7DWOXXWpvd/jhYd3YsVVlzz5btc9xx7m/9171fW66qfaxwf3TT8P61PLs2bXPt3lz5n3rii/PkW9tCunM7Brgc+A8VH0kInVVU119dXhSqua43r//PTz+eHjjPJMpU+C66+DZZxsXx+bNobrrs8+gT5/q6x58EM44o2r5rrvCex+tRF5VH5lZZzPrmpoHvg7MAp4CRsfNRgNP5jo2EckDqWoq9+oDLV17bfWEkPpR/sEPqieE+++vfrxDD61KCJddFh4FXroUXn+9/jg6dgyxpCeE1GPE3/temF54YSi/4AJ49dXMfWdt2QI9e8L//E/omyv1OHImdZUvWgT33hvmZ88OHUBmS123ENn6ALsRqoxmArOBK2N5T+AlwiOpfwN22taxVH0kUgDWrXPfe++q6ppzznF/++2w7vrrq8prPpL7+efuu+5avapn9erM50h/DHjFitpVRH371h1fzWqpNWtCNdiKFdXjq/lZtqzqGAsW1F6/eLH7E0+4H3xw5v2//e0mXU73PK8+ag5VH4kIixeHMTgydZDYVBUVoUrqmWfgkktgn33q3/6oo2q/gNgQZ53V9E4M//jHzG+vN0DePZLaUpQURCSvnHlmaG9I1717eOqqY8eQbFIjBtbUsWN4Cgtq9101YQJ8//stFmZetSmIiLRZDzxQu6Jn9eqqXnDbtatqM/ngg6r9NmyoSghQ+xgtmBC2pUPOziQiIlX22isvuzjXnYKIiFRSUhARkUpKCiIiUklJQUREKikpiN3Ib84AAAdcSURBVIhIJSUFERGppKQgIiKVlBRERKRSq+7mwszKgAVN3L0XsHybWxUWXZPadE1q0zWprjVej13dvTjTiladFJrDzKbW1fdHodI1qU3XpDZdk+ra2vVQ9ZGIiFRSUhARkUqFnBTuSTqAPKRrUpuuSW26JtW1qetRsG0KIiJSWyHfKYiISA1KCiIiUqkgk4KZHWNmH5rZXDO7LOl4WpqZjTOzZWY2K61sJzN70czmxGmPWG5mdke8Fu+Y2fC0fUbH7eeY2ei08gPM7N24zx1mNccOzC9mNsDM/m5m75nZbDP7USwv5GvSyczeMrOZ8ZpcG8sHmdmb8XtMNLOOsXy7uDw3rh+YdqzLY/mHZnZ0Wnmr+3dmZu3N7G0z+0tcLrzr4e4F9QHaAx8DuwEdgZnAPknH1cLfcRQwHJiVVnYTcFmcvwy4Mc4fCzwLGDACeDOW7wTMi9Mecb5HXPdW3Nbivt9I+jtv43r0BYbH+a7AR8A+BX5NDOgS54uAN2P8k4DTYvndwAVx/j+Bu+P8acDEOL9P/De0HTAo/ttq31r/nQEXAw8Bf4nLBXc9CvFO4SBgrrvPc/fNwCPACQnH1KLcfTKwskbxCcD4OD8eODGtfIIHbwA7mllf4GjgRXdf6e6rgBeBY+K6bu7+hod/BRPSjpWX3H2Ju0+P8+uA94GdKexr4u7+eVwsih8HvgY8FstrXpPUtXoMOCLeDZ0APOLum9z9E2Au4d9Yq/t3Zmb9gW8Cv4/LRgFej0JMCjsDn6YtL4xlbV2Juy+J80uBkjhf1/Wor3xhhvJWId7m70/4y7igr0msKpkBLCMkuI+B1e6+NW6S/j0qv3tcvwboSeOvVT67DfgZUBGXe1KA16MQk0LBi3/NFtyzyGbWBfgTMNbd16avK8Rr4u7l7j4M6E/4S3bvhENKjJkdByxz92lJx5K0QkwKi4ABacv9Y1lb91ms5iBOl8Xyuq5HfeX9M5TnNTMrIiSEB9398Vhc0Nckxd1XA38HDiZUlXWIq9K/R+V3j+u7Ayto/LXKV4cAx5vZfELVzteA2ynE65F0o0auP0AHQgPhIKoafPZNOq4sfM+BVG9ovpnqjao3xflvUr1R9a1YvhPwCaFBtUec3ymuq9moemzS33cb18II9fy31Sgv5GtSDOwY57cHXgWOAx6lesPqf8b5C6nesDopzu9L9YbVeYRG1Vb77ww4nKqG5oK7HokHkNB/9GMJT6B8DFyZdDxZ+H4PA0uALYS6y3MJ9Z0vAXOAv6X9mBnwv/FavAuUph3n3wkNZXOBc9LKS4FZcZ/fEd+Mz9cPcCihaugdYEb8HFvg12QI8Ha8JrOA/47luxES3Nz4g7hdLO8Ul+fG9bulHevK+L0/JO2pq9b676xGUii466FuLkREpFIhtimIiEgdlBRERKSSkoKIiFRSUhARkUpKCiIiUklJQVo1M3MzuyVt+admdk0WzvNw7DH1xzXKrzGzn8b5s82sXwue83Az+0ra8hgzO6ulji+SSYdtbyKS1zYBJ5vZr9x9eTZOYGZ9gAPdffdtbHo24Zn/xY04dgev6lunpsOBz4F/ALj73Q09rkhT6U5BWruthDFyf1xzhZkNNLP/F//Cf8nMdqnvQHGMgfviuAhvm9lX46oXgJ3NbIaZjaxj31MJL7A9GLfbPo6x8IqZTTOz59O61HjZzG4zs6nAj8zsW7FP/rfN7G9mVhI77hsD/Dh13hp3JcPM7I343Z6wqrEgXjazGy2MlfBRKl4z2zeWzYj77NHoKy0FQUlB2oL/Bc4ws+41yn8LjHf3IcCDwB3bOM6FhL7x9gNOB8abWSfgeOBjdx/m7q9m2tHdHwOmAmd46GRuazz/qe5+ADAOuC5tl47uXurutwBTgBHuvj+h352fuft8QrcKt9Zx3gnApfG7vQtcnbaug7sfBIxNKx8D3B5jK6V6r64ilVR9JK2eu681swnAD4ENaasOBk6O838kDKpTn0MJP+S4+wdmtgDYE1hb716Z7QUMBl4M3ezTntD1SMrEtPn+wMR4J9GR0KdSnWLy29HdX4lF4wldLqSkOvybRugDC+B14Mo4ZsDj7j6nsV9ICoPuFKStuI3Qx1PnpAOJDJgd/8of5u77ufvX09avT5v/LfC7eIfyH4R+dZpjU5yWE//wc/eHCHc8G4C/mtnXmnkOaaOUFKRNcPeVhKETz00r/gehB0uAMwg9gdbn1bgdZrYnsAuhU7OGWkcY7pO4X7GZHRyPV2Rm+9axX3equlEenVaefrxK7r4GWJXWvvF94JWa26Uzs92Aee5+B/AkoUM8kVqUFKQtuQXolbZ8EXCOmb1D+OH8EVQ+2jkmw/53Au3M7F1C9c7Z7r4pw3Z1uR+4O45m1h44FbjRzGYSemb9Sh37XQM8ambTgPQnqJ4GTqqjgXs0cHP8bsOAX2wjtu8As2JsgwltEiK1qJdUERGppDsFERGppKQgIiKVlBRERKSSkoKIiFRSUhARkUpKCiIiUklJQUREKv1/x45gomK2iYkAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": "<a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=a01dcc76-30f1-4d9e-964c-6e3edcc6c5de' target=\"_blank\">\n<img style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ2h0PSI4MHB4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPgogICAgICAgIDxnIGlkPSJBcnRib2FyZCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTEyMzUuMDAwMDAwLCAtNzkuMDAwMDAwKSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD0iUGF0aC0yMCIgZmlsbD0iIzAyNjVCNCIgcG9pbnRzPSIyLjM3NjIzNzYyIDgwIDM4LjA0NzY2NjcgODAgNTcuODIxNzgyMiA3My44MDU3NTkyIDU3LjgyMTc4MjIgMzIuNzU5MjczOSAzOS4xNDAyMjc4IDMxLjY4MzE2ODMiPjwvcG9seWdvbj4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0zNS4wMDc3MTgsODAgQzQyLjkwNjIwMDcsNzYuNDU0OTM1OCA0Ny41NjQ5MTY3LDcxLjU0MjI2NzEgNDguOTgzODY2LDY1LjI2MTk5MzkgQzUxLjExMjI4OTksNTUuODQxNTg0MiA0MS42NzcxNzk1LDQ5LjIxMjIyODQgMjUuNjIzOTg0Niw0OS4yMTIyMjg0IEMyNS40ODQ5Mjg5LDQ5LjEyNjg0NDggMjkuODI2MTI5Niw0My4yODM4MjQ4IDM4LjY0NzU4NjksMzEuNjgzMTY4MyBMNzIuODcxMjg3MSwzMi41NTQ0MjUgTDY1LjI4MDk3Myw2Ny42NzYzNDIxIEw1MS4xMTIyODk5LDc3LjM3NjE0NCBMMzUuMDA3NzE4LDgwIFoiIGlkPSJQYXRoLTIyIiBmaWxsPSIjMDAyODY4Ij48L3BhdGg+CiAgICAgICAgICAgICAgICA8cGF0aCBkPSJNMCwzNy43MzA0NDA1IEwyNy4xMTQ1MzcsMC4yNTcxMTE0MzYgQzYyLjM3MTUxMjMsLTEuOTkwNzE3MDEgODAsMTAuNTAwMzkyNyA4MCwzNy43MzA0NDA1IEM4MCw2NC45NjA0ODgyIDY0Ljc3NjUwMzgsNzkuMDUwMzQxNCAzNC4zMjk1MTEzLDgwIEM0Ny4wNTUzNDg5LDc3LjU2NzA4MDggNTMuNDE4MjY3Nyw3MC4zMTM2MTAzIDUzLjQxODI2NzcsNTguMjM5NTg4NSBDNTMuNDE4MjY3Nyw0MC4xMjg1NTU3IDM2LjMwMzk1NDQsMzcuNzMwNDQwNSAyNS4yMjc0MTcsMzcuNzMwNDQwNSBDMTcuODQzMDU4NiwzNy43MzA0NDA1IDkuNDMzOTE5NjYsMzcuNzMwNDQwNSAwLDM3LjczMDQ0MDUgWiIgaWQ9IlBhdGgtMTkiIGZpbGw9IiMzNzkzRUYiPjwvcGF0aD4KICAgICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+' > </img>\nCreated in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>",
      "metadata": {
        "tags": [],
        "created_in_deepnote_cell": true,
        "deepnote_cell_type": "markdown"
      }
    }
  ],
  "nbformat": 4,
  "nbformat_minor": 2,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.3"
    },
    "deepnote_execution_queue": [],
    "deepnote_notebook_id": "eb967f6a-6f88-47a7-9352-c060ee6680bd",
    "deepnote": {}
  }
}