Attention Only Classification Analysis Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attention Only Classification Analysis Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "\n",
    "import Bio\n",
    "from Bio import SeqIO\n",
    "import Bio.motifs\n",
    "\n",
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn import model_selection\n",
    "import sklearn\n",
    "\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "from signal_models_scratch import *\n",
    "from utility_functions import *\n",
    "\n",
    "sns.set_context('notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "working_directory = '/home/jtao/analysis/genomic_grammar_analysis/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** kla-1h ***\n",
      "*** veh ***\n",
      "*** il4-24h ***\n"
     ]
    }
   ],
   "source": [
    "treatment_positive_fasta_dict = {}\n",
    "treatment_negative_fasta_dict = {}\n",
    "treatment_positive_seqArray_dict = {}\n",
    "treatment_positive_rc_seqArray_dict = {}\n",
    "treatment_negative_seqArray_dict = {}\n",
    "treatment_negative_rc_seqArray_dict = {}\n",
    "\n",
    "for ps in ['c57bl6_kla-1h_peaks.fasta', 'c57bl6_veh_peaks.fasta', 'c57bl6_il4-24h_peaks.fasta']:\n",
    "    treatment = ps.split('_')[1]\n",
    "    print('***',treatment, '***')\n",
    "    positive_seqRecords = list(SeqIO.parse(working_directory + '/peak_sequences/' + ps, 'fasta'))\n",
    "    negative_seqRecords = list(SeqIO.parse(working_directory + '/background_files/' + ps.replace('_peaks', '_background'), 'fasta'))[:len(positive_seqRecords)]\n",
    "    \n",
    "    positive_fasta_seq = [str(x.seq[:200]) for x in positive_seqRecords]\n",
    "    negative_fasta_seq = [str(x.seq[:200]) for x in negative_seqRecords]\n",
    "    \n",
    "    positive_rc_fasta_seq = [str(x[:200].reverse_complement().seq) for x in positive_seqRecords]\n",
    "    negative_rc_fasta_seq = [str(x[:200].reverse_complement().seq) for x in negative_seqRecords]\n",
    "    \n",
    "    positive_sequence_arrays = convert_sequences_to_array(positive_fasta_seq)\n",
    "    positive_rc_sequence_arrays = convert_sequences_to_array(positive_rc_fasta_seq)\n",
    "    \n",
    "    negative_sequence_arrays = convert_sequences_to_array(negative_fasta_seq)\n",
    "    negative_rc_sequence_arrays = convert_sequences_to_array(negative_rc_fasta_seq)\n",
    "    \n",
    "    treatment_positive_fasta_dict[treatment] = positive_fasta_seq\n",
    "    treatment_negative_fasta_dict[treatment] = negative_fasta_seq\n",
    "    treatment_positive_seqArray_dict[treatment] = positive_sequence_arrays\n",
    "    treatment_positive_rc_seqArray_dict[treatment] = positive_rc_sequence_arrays\n",
    "    treatment_negative_seqArray_dict[treatment] = negative_sequence_arrays\n",
    "    treatment_negative_rc_seqArray_dict[treatment] = negative_rc_sequence_arrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Model Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### settings for attention model ###\n",
    "attention_num_motifs=96\n",
    "attention_motif_size=14\n",
    "adjacent_bp_pool_size=1\n",
    "attention_num_dense_neurons = 8\n",
    "attention_dropout_rate=0.1\n",
    "\n",
    "### settings for convolution model ###\n",
    "convolution_num_motifs=16\n",
    "convolution_motif_size=24\n",
    "num_dense_neurons=32\n",
    "convolution_dropout_rate=0.5\n",
    "\n",
    "largeConvolution_num_motifs = 49\n",
    "large_num_dense_neurons = 98"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dotProductAttention_model 9738\n",
      "convolution_model 2162\n",
      "large_convolution_model 9851\n"
     ]
    }
   ],
   "source": [
    "dotProductAttention_model = get_dotProductAttention_model(200,\n",
    "    mode='classification',\n",
    "    num_motifs=attention_num_motifs, \n",
    "    motif_size=attention_motif_size,\n",
    "    adjacent_bp_pool_size=adjacent_bp_pool_size,\n",
    "    num_dense_neurons = attention_num_dense_neurons,\n",
    "    dropout_rate=attention_dropout_rate,\n",
    "    num_classes=2)\n",
    "print('dotProductAttention_model', dotProductAttention_model.count_params())\n",
    "\n",
    "convolution_model = get_convolution_model(200,\n",
    "    mode='classification',\n",
    "    num_motifs=convolution_num_motifs, \n",
    "    motif_size=convolution_motif_size,\n",
    "    num_dense_neurons=num_dense_neurons,\n",
    "    dropout_rate=convolution_dropout_rate,\n",
    "    num_classes=2)\n",
    "print('convolution_model', convolution_model.count_params())\n",
    "\n",
    "large_convolution_model = get_convolution_model(200,\n",
    "    mode='classification',\n",
    "    num_motifs=largeConvolution_num_motifs, \n",
    "    motif_size=convolution_motif_size,\n",
    "    num_dense_neurons=large_num_dense_neurons,\n",
    "    dropout_rate=convolution_dropout_rate,\n",
    "    num_classes=2)\n",
    "print('large_convolution_model', large_convolution_model.count_params())\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train All Models for all Data Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "condition_model_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 1\n",
      "Training DotAttention il4-24h\n",
      "Train on 53628 samples, validate on 13408 samples\n",
      "Epoch 1/20\n",
      "53628/53628 [==============================] - 51s 958us/step - loss: 0.6473 - categorical_accuracy: 0.6044 - val_loss: 0.5695 - val_categorical_accuracy: 0.7098\n",
      "Epoch 2/20\n",
      "53628/53628 [==============================] - 50s 928us/step - loss: 0.5588 - categorical_accuracy: 0.7133 - val_loss: 0.5207 - val_categorical_accuracy: 0.7452\n",
      "Epoch 3/20\n",
      "53628/53628 [==============================] - 50s 933us/step - loss: 0.5115 - categorical_accuracy: 0.7497 - val_loss: 0.5215 - val_categorical_accuracy: 0.7413\n",
      "Epoch 4/20\n",
      "53628/53628 [==============================] - 50s 928us/step - loss: 0.4741 - categorical_accuracy: 0.7728 - val_loss: 0.4450 - val_categorical_accuracy: 0.7982\n",
      "Epoch 5/20\n",
      "53628/53628 [==============================] - 50s 927us/step - loss: 0.4293 - categorical_accuracy: 0.8015 - val_loss: 0.4145 - val_categorical_accuracy: 0.8154\n",
      "Epoch 6/20\n",
      "53628/53628 [==============================] - 50s 927us/step - loss: 0.4045 - categorical_accuracy: 0.8169 - val_loss: 0.4380 - val_categorical_accuracy: 0.7984\n",
      "Epoch 7/20\n",
      "53628/53628 [==============================] - 50s 925us/step - loss: 0.3873 - categorical_accuracy: 0.8245 - val_loss: 0.3808 - val_categorical_accuracy: 0.8315\n",
      "Epoch 8/20\n",
      "53628/53628 [==============================] - 50s 927us/step - loss: 0.3672 - categorical_accuracy: 0.8367 - val_loss: 0.3701 - val_categorical_accuracy: 0.8368\n",
      "Epoch 9/20\n",
      "53628/53628 [==============================] - 49s 921us/step - loss: 0.3513 - categorical_accuracy: 0.8458 - val_loss: 0.3553 - val_categorical_accuracy: 0.8441\n",
      "Epoch 10/20\n",
      "53628/53628 [==============================] - 49s 922us/step - loss: 0.3414 - categorical_accuracy: 0.8491 - val_loss: 0.3492 - val_categorical_accuracy: 0.8452\n",
      "Epoch 11/20\n",
      "53628/53628 [==============================] - 49s 923us/step - loss: 0.3342 - categorical_accuracy: 0.8531 - val_loss: 0.3423 - val_categorical_accuracy: 0.8520\n",
      "Epoch 12/20\n",
      "53628/53628 [==============================] - 50s 930us/step - loss: 0.3209 - categorical_accuracy: 0.8607 - val_loss: 0.3374 - val_categorical_accuracy: 0.8523\n",
      "Epoch 13/20\n",
      "53628/53628 [==============================] - 50s 924us/step - loss: 0.3152 - categorical_accuracy: 0.8647 - val_loss: 0.3308 - val_categorical_accuracy: 0.8572\n",
      "Epoch 14/20\n",
      "53628/53628 [==============================] - 50s 926us/step - loss: 0.3076 - categorical_accuracy: 0.8681 - val_loss: 0.3281 - val_categorical_accuracy: 0.8570\n",
      "Epoch 15/20\n",
      "53628/53628 [==============================] - 50s 930us/step - loss: 0.3037 - categorical_accuracy: 0.8703 - val_loss: 0.3210 - val_categorical_accuracy: 0.8624\n",
      "Epoch 16/20\n",
      "53628/53628 [==============================] - 50s 925us/step - loss: 0.2938 - categorical_accuracy: 0.8763 - val_loss: 0.3480 - val_categorical_accuracy: 0.8490\n",
      "Epoch 17/20\n",
      "53628/53628 [==============================] - 50s 929us/step - loss: 0.2912 - categorical_accuracy: 0.8768 - val_loss: 0.3320 - val_categorical_accuracy: 0.8593\n",
      "Epoch 18/20\n",
      "53628/53628 [==============================] - 50s 928us/step - loss: 0.2839 - categorical_accuracy: 0.8797 - val_loss: 0.3177 - val_categorical_accuracy: 0.8635\n",
      "Epoch 19/20\n",
      "53628/53628 [==============================] - 50s 929us/step - loss: 0.2779 - categorical_accuracy: 0.8833 - val_loss: 0.3102 - val_categorical_accuracy: 0.8669\n",
      "Epoch 20/20\n",
      "53628/53628 [==============================] - 50s 929us/step - loss: 0.2757 - categorical_accuracy: 0.8844 - val_loss: 0.3282 - val_categorical_accuracy: 0.8602\n",
      "Training Large Convolution il4-24h\n",
      "Train on 53628 samples, validate on 13408 samples\n",
      "Epoch 1/20\n",
      "53628/53628 [==============================] - 5s 97us/step - loss: 0.5912 - categorical_accuracy: 0.6669 - val_loss: 0.4542 - val_categorical_accuracy: 0.7877\n",
      "Epoch 2/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.4396 - categorical_accuracy: 0.7988 - val_loss: 0.4003 - val_categorical_accuracy: 0.8191\n",
      "Epoch 3/20\n",
      "53628/53628 [==============================] - 4s 82us/step - loss: 0.3962 - categorical_accuracy: 0.8250 - val_loss: 0.4050 - val_categorical_accuracy: 0.8198\n",
      "Epoch 4/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3710 - categorical_accuracy: 0.8395 - val_loss: 0.3711 - val_categorical_accuracy: 0.8324\n",
      "Epoch 5/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3510 - categorical_accuracy: 0.8497 - val_loss: 0.3637 - val_categorical_accuracy: 0.8385\n",
      "Epoch 6/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3393 - categorical_accuracy: 0.8555 - val_loss: 0.3605 - val_categorical_accuracy: 0.8376\n",
      "Epoch 7/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3289 - categorical_accuracy: 0.8610 - val_loss: 0.3822 - val_categorical_accuracy: 0.8325\n",
      "Epoch 8/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3209 - categorical_accuracy: 0.8660 - val_loss: 0.3641 - val_categorical_accuracy: 0.8396\n",
      "Epoch 9/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3160 - categorical_accuracy: 0.8670 - val_loss: 0.3624 - val_categorical_accuracy: 0.8390\n",
      "Epoch 10/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3112 - categorical_accuracy: 0.8705 - val_loss: 0.3620 - val_categorical_accuracy: 0.8462\n",
      "Epoch 11/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3069 - categorical_accuracy: 0.8724 - val_loss: 0.3491 - val_categorical_accuracy: 0.8487\n",
      "Epoch 12/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.3004 - categorical_accuracy: 0.8751 - val_loss: 0.3637 - val_categorical_accuracy: 0.8462\n",
      "Epoch 13/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2969 - categorical_accuracy: 0.8761 - val_loss: 0.3580 - val_categorical_accuracy: 0.8437\n",
      "Epoch 14/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2952 - categorical_accuracy: 0.8780 - val_loss: 0.3601 - val_categorical_accuracy: 0.8411\n",
      "Epoch 15/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2904 - categorical_accuracy: 0.8786 - val_loss: 0.3972 - val_categorical_accuracy: 0.8350\n",
      "Epoch 16/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2878 - categorical_accuracy: 0.8806 - val_loss: 0.3605 - val_categorical_accuracy: 0.8439\n",
      "Epoch 17/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2858 - categorical_accuracy: 0.8823 - val_loss: 0.3639 - val_categorical_accuracy: 0.8440\n",
      "Epoch 18/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2818 - categorical_accuracy: 0.8845 - val_loss: 0.3655 - val_categorical_accuracy: 0.8482\n",
      "Epoch 19/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2789 - categorical_accuracy: 0.8843 - val_loss: 0.3619 - val_categorical_accuracy: 0.8469\n",
      "Epoch 20/20\n",
      "53628/53628 [==============================] - 4s 83us/step - loss: 0.2785 - categorical_accuracy: 0.8854 - val_loss: 0.3582 - val_categorical_accuracy: 0.8461\n",
      "Training DotAttention kla-1h\n",
      "Train on 34841 samples, validate on 8711 samples\n",
      "Epoch 1/20\n",
      "34841/34841 [==============================] - 34s 969us/step - loss: 0.6803 - categorical_accuracy: 0.5475 - val_loss: 0.6259 - val_categorical_accuracy: 0.6548\n",
      "Epoch 2/20\n",
      "34841/34841 [==============================] - 33s 950us/step - loss: 0.5982 - categorical_accuracy: 0.6767 - val_loss: 0.5819 - val_categorical_accuracy: 0.6879\n",
      "Epoch 3/20\n",
      "34841/34841 [==============================] - 33s 948us/step - loss: 0.5460 - categorical_accuracy: 0.7247 - val_loss: 0.5216 - val_categorical_accuracy: 0.7393\n",
      "Epoch 4/20\n",
      "34841/34841 [==============================] - 33s 945us/step - loss: 0.5002 - categorical_accuracy: 0.7567 - val_loss: 0.4880 - val_categorical_accuracy: 0.7621\n",
      "Epoch 5/20\n",
      "34841/34841 [==============================] - 33s 942us/step - loss: 0.4556 - categorical_accuracy: 0.7866 - val_loss: 0.4931 - val_categorical_accuracy: 0.7584\n",
      "Epoch 6/20\n",
      "34841/34841 [==============================] - 33s 941us/step - loss: 0.4217 - categorical_accuracy: 0.8064 - val_loss: 0.4540 - val_categorical_accuracy: 0.7869\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/20\n",
      "34841/34841 [==============================] - 33s 941us/step - loss: 0.3875 - categorical_accuracy: 0.8247 - val_loss: 0.3928 - val_categorical_accuracy: 0.8237\n",
      "Epoch 8/20\n",
      "34841/34841 [==============================] - 33s 949us/step - loss: 0.3648 - categorical_accuracy: 0.8376 - val_loss: 0.4330 - val_categorical_accuracy: 0.7989\n",
      "Epoch 9/20\n",
      "34841/34841 [==============================] - 33s 944us/step - loss: 0.3507 - categorical_accuracy: 0.8449 - val_loss: 0.3714 - val_categorical_accuracy: 0.8335\n",
      "Epoch 10/20\n",
      "34841/34841 [==============================] - 33s 940us/step - loss: 0.3390 - categorical_accuracy: 0.8524 - val_loss: 0.4191 - val_categorical_accuracy: 0.8110\n",
      "Epoch 11/20\n",
      "34841/34841 [==============================] - 33s 947us/step - loss: 0.3279 - categorical_accuracy: 0.8574 - val_loss: 0.3525 - val_categorical_accuracy: 0.8409\n",
      "Epoch 12/20\n",
      "34841/34841 [==============================] - 33s 945us/step - loss: 0.3163 - categorical_accuracy: 0.8641 - val_loss: 0.3616 - val_categorical_accuracy: 0.8425\n",
      "Epoch 13/20\n",
      "34841/34841 [==============================] - 33s 950us/step - loss: 0.3106 - categorical_accuracy: 0.8657 - val_loss: 0.3886 - val_categorical_accuracy: 0.8290\n",
      "Epoch 14/20\n",
      "34841/34841 [==============================] - 33s 942us/step - loss: 0.3025 - categorical_accuracy: 0.8706 - val_loss: 0.4030 - val_categorical_accuracy: 0.8147\n",
      "Epoch 15/20\n",
      "34841/34841 [==============================] - 33s 946us/step - loss: 0.2930 - categorical_accuracy: 0.8762 - val_loss: 0.3942 - val_categorical_accuracy: 0.8270\n",
      "Epoch 16/20\n",
      "34841/34841 [==============================] - 33s 948us/step - loss: 0.2854 - categorical_accuracy: 0.8816 - val_loss: 0.3270 - val_categorical_accuracy: 0.8624\n",
      "Epoch 17/20\n",
      "34841/34841 [==============================] - 33s 947us/step - loss: 0.2861 - categorical_accuracy: 0.8796 - val_loss: 0.3357 - val_categorical_accuracy: 0.8533\n",
      "Epoch 18/20\n",
      "34841/34841 [==============================] - 33s 954us/step - loss: 0.2781 - categorical_accuracy: 0.8833 - val_loss: 0.3420 - val_categorical_accuracy: 0.8531\n",
      "Epoch 19/20\n",
      "34841/34841 [==============================] - 33s 951us/step - loss: 0.2708 - categorical_accuracy: 0.8886 - val_loss: 0.3459 - val_categorical_accuracy: 0.8575\n",
      "Epoch 20/20\n",
      "34841/34841 [==============================] - 33s 947us/step - loss: 0.2751 - categorical_accuracy: 0.8849 - val_loss: 0.3200 - val_categorical_accuracy: 0.8657\n",
      "Training Large Convolution kla-1h\n",
      "Train on 34841 samples, validate on 8711 samples\n",
      "Epoch 1/20\n",
      "34841/34841 [==============================] - 4s 109us/step - loss: 0.6390 - categorical_accuracy: 0.6073 - val_loss: 0.5167 - val_categorical_accuracy: 0.7404\n",
      "Epoch 2/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.4603 - categorical_accuracy: 0.7871 - val_loss: 0.4261 - val_categorical_accuracy: 0.8007\n",
      "Epoch 3/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.4072 - categorical_accuracy: 0.8181 - val_loss: 0.4097 - val_categorical_accuracy: 0.8140\n",
      "Epoch 4/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.3788 - categorical_accuracy: 0.8369 - val_loss: 0.3858 - val_categorical_accuracy: 0.8253\n",
      "Epoch 5/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.3558 - categorical_accuracy: 0.8478 - val_loss: 0.3909 - val_categorical_accuracy: 0.8254\n",
      "Epoch 6/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.3412 - categorical_accuracy: 0.8552 - val_loss: 0.3790 - val_categorical_accuracy: 0.8263\n",
      "Epoch 7/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.3272 - categorical_accuracy: 0.8636 - val_loss: 0.3715 - val_categorical_accuracy: 0.8342\n",
      "Epoch 8/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.3195 - categorical_accuracy: 0.8674 - val_loss: 0.4169 - val_categorical_accuracy: 0.8129\n",
      "Epoch 9/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.3118 - categorical_accuracy: 0.8690 - val_loss: 0.3754 - val_categorical_accuracy: 0.8347\n",
      "Epoch 10/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.3087 - categorical_accuracy: 0.8691 - val_loss: 0.3797 - val_categorical_accuracy: 0.8363\n",
      "Epoch 11/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2963 - categorical_accuracy: 0.8768 - val_loss: 0.3818 - val_categorical_accuracy: 0.8377\n",
      "Epoch 12/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2921 - categorical_accuracy: 0.8789 - val_loss: 0.3977 - val_categorical_accuracy: 0.8260\n",
      "Epoch 13/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.2941 - categorical_accuracy: 0.8762 - val_loss: 0.3964 - val_categorical_accuracy: 0.8408\n",
      "Epoch 14/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2815 - categorical_accuracy: 0.8834 - val_loss: 0.3827 - val_categorical_accuracy: 0.8380\n",
      "Epoch 15/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.2783 - categorical_accuracy: 0.8837 - val_loss: 0.3727 - val_categorical_accuracy: 0.8386\n",
      "Epoch 16/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2822 - categorical_accuracy: 0.8830 - val_loss: 0.3998 - val_categorical_accuracy: 0.8315\n",
      "Epoch 17/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2751 - categorical_accuracy: 0.8873 - val_loss: 0.3804 - val_categorical_accuracy: 0.8381\n",
      "Epoch 18/20\n",
      "34841/34841 [==============================] - 3s 84us/step - loss: 0.2693 - categorical_accuracy: 0.8894 - val_loss: 0.3921 - val_categorical_accuracy: 0.8365\n",
      "Epoch 19/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2629 - categorical_accuracy: 0.8920 - val_loss: 0.3810 - val_categorical_accuracy: 0.8317\n",
      "Epoch 20/20\n",
      "34841/34841 [==============================] - 3s 85us/step - loss: 0.2620 - categorical_accuracy: 0.8926 - val_loss: 0.3883 - val_categorical_accuracy: 0.8348\n",
      "Training DotAttention veh\n",
      "Train on 50480 samples, validate on 12620 samples\n",
      "Epoch 1/20\n",
      "50480/50480 [==============================] - 48s 946us/step - loss: 0.6576 - categorical_accuracy: 0.5824 - val_loss: 0.5733 - val_categorical_accuracy: 0.7008\n",
      "Epoch 2/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.5427 - categorical_accuracy: 0.7241 - val_loss: 0.4892 - val_categorical_accuracy: 0.7631\n",
      "Epoch 3/20\n",
      "50480/50480 [==============================] - 47s 938us/step - loss: 0.4608 - categorical_accuracy: 0.7818 - val_loss: 0.4708 - val_categorical_accuracy: 0.7746\n",
      "Epoch 4/20\n",
      "50480/50480 [==============================] - 47s 927us/step - loss: 0.4142 - categorical_accuracy: 0.8105 - val_loss: 0.3964 - val_categorical_accuracy: 0.8195\n",
      "Epoch 5/20\n",
      "50480/50480 [==============================] - 47s 935us/step - loss: 0.3919 - categorical_accuracy: 0.8240 - val_loss: 0.3786 - val_categorical_accuracy: 0.8315\n",
      "Epoch 6/20\n",
      "50480/50480 [==============================] - 47s 926us/step - loss: 0.3735 - categorical_accuracy: 0.8359 - val_loss: 0.3832 - val_categorical_accuracy: 0.8303\n",
      "Epoch 7/20\n",
      "50480/50480 [==============================] - 47s 930us/step - loss: 0.3579 - categorical_accuracy: 0.8439 - val_loss: 0.3723 - val_categorical_accuracy: 0.8361\n",
      "Epoch 8/20\n",
      "50480/50480 [==============================] - 47s 928us/step - loss: 0.3468 - categorical_accuracy: 0.8501 - val_loss: 0.3651 - val_categorical_accuracy: 0.8395\n",
      "Epoch 9/20\n",
      "50480/50480 [==============================] - 47s 930us/step - loss: 0.3399 - categorical_accuracy: 0.8535 - val_loss: 0.3541 - val_categorical_accuracy: 0.8444\n",
      "Epoch 10/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.3293 - categorical_accuracy: 0.8581 - val_loss: 0.3434 - val_categorical_accuracy: 0.8490\n",
      "Epoch 11/20\n",
      "50480/50480 [==============================] - 47s 927us/step - loss: 0.3233 - categorical_accuracy: 0.8623 - val_loss: 0.3460 - val_categorical_accuracy: 0.8494\n",
      "Epoch 12/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.3162 - categorical_accuracy: 0.8655 - val_loss: 0.3354 - val_categorical_accuracy: 0.8560\n",
      "Epoch 13/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50480/50480 [==============================] - 47s 934us/step - loss: 0.3068 - categorical_accuracy: 0.8698 - val_loss: 0.3335 - val_categorical_accuracy: 0.8552\n",
      "Epoch 14/20\n",
      "50480/50480 [==============================] - 47s 929us/step - loss: 0.3035 - categorical_accuracy: 0.8717 - val_loss: 0.3574 - val_categorical_accuracy: 0.8471\n",
      "Epoch 15/20\n",
      "50480/50480 [==============================] - 47s 932us/step - loss: 0.2977 - categorical_accuracy: 0.8743 - val_loss: 0.3318 - val_categorical_accuracy: 0.8574\n",
      "Epoch 16/20\n",
      "50480/50480 [==============================] - 47s 934us/step - loss: 0.2943 - categorical_accuracy: 0.8755 - val_loss: 0.3315 - val_categorical_accuracy: 0.8579\n",
      "Epoch 17/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.2878 - categorical_accuracy: 0.8795 - val_loss: 0.3365 - val_categorical_accuracy: 0.8583\n",
      "Epoch 18/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.2849 - categorical_accuracy: 0.8805 - val_loss: 0.3591 - val_categorical_accuracy: 0.8408\n",
      "Epoch 19/20\n",
      "50480/50480 [==============================] - 47s 936us/step - loss: 0.2806 - categorical_accuracy: 0.8822 - val_loss: 0.3223 - val_categorical_accuracy: 0.8656\n",
      "Epoch 20/20\n",
      "50480/50480 [==============================] - 47s 931us/step - loss: 0.2800 - categorical_accuracy: 0.8820 - val_loss: 0.3253 - val_categorical_accuracy: 0.8630\n",
      "Training Large Convolution veh\n",
      "Train on 50480 samples, validate on 12620 samples\n",
      "Epoch 1/20\n",
      "50480/50480 [==============================] - 5s 104us/step - loss: 0.5941 - categorical_accuracy: 0.6607 - val_loss: 0.4568 - val_categorical_accuracy: 0.7857\n",
      "Epoch 2/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.4456 - categorical_accuracy: 0.7960 - val_loss: 0.4222 - val_categorical_accuracy: 0.8026\n",
      "Epoch 3/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.4038 - categorical_accuracy: 0.8194 - val_loss: 0.3994 - val_categorical_accuracy: 0.8158\n",
      "Epoch 4/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3791 - categorical_accuracy: 0.8326 - val_loss: 0.4045 - val_categorical_accuracy: 0.8171\n",
      "Epoch 5/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3632 - categorical_accuracy: 0.8423 - val_loss: 0.3958 - val_categorical_accuracy: 0.8199\n",
      "Epoch 6/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3522 - categorical_accuracy: 0.8476 - val_loss: 0.3744 - val_categorical_accuracy: 0.8336\n",
      "Epoch 7/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3411 - categorical_accuracy: 0.8541 - val_loss: 0.3780 - val_categorical_accuracy: 0.8345\n",
      "Epoch 8/20\n",
      "50480/50480 [==============================] - 4s 87us/step - loss: 0.3327 - categorical_accuracy: 0.8578 - val_loss: 0.3743 - val_categorical_accuracy: 0.8376\n",
      "Epoch 9/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3252 - categorical_accuracy: 0.8620 - val_loss: 0.3695 - val_categorical_accuracy: 0.8410\n",
      "Epoch 10/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3215 - categorical_accuracy: 0.8641 - val_loss: 0.3658 - val_categorical_accuracy: 0.8414\n",
      "Epoch 11/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3137 - categorical_accuracy: 0.8668 - val_loss: 0.3699 - val_categorical_accuracy: 0.8399\n",
      "Epoch 12/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3097 - categorical_accuracy: 0.8709 - val_loss: 0.3763 - val_categorical_accuracy: 0.8372\n",
      "Epoch 13/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3043 - categorical_accuracy: 0.8723 - val_loss: 0.3768 - val_categorical_accuracy: 0.8430\n",
      "Epoch 14/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.3017 - categorical_accuracy: 0.8723 - val_loss: 0.3686 - val_categorical_accuracy: 0.8471\n",
      "Epoch 15/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2969 - categorical_accuracy: 0.8747 - val_loss: 0.3665 - val_categorical_accuracy: 0.8465\n",
      "Epoch 16/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2932 - categorical_accuracy: 0.8777 - val_loss: 0.3741 - val_categorical_accuracy: 0.8446\n",
      "Epoch 17/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2908 - categorical_accuracy: 0.8779 - val_loss: 0.3696 - val_categorical_accuracy: 0.8441\n",
      "Epoch 18/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2877 - categorical_accuracy: 0.8808 - val_loss: 0.3741 - val_categorical_accuracy: 0.8476\n",
      "Epoch 19/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2852 - categorical_accuracy: 0.8813 - val_loss: 0.3581 - val_categorical_accuracy: 0.8468\n",
      "Epoch 20/20\n",
      "50480/50480 [==============================] - 4s 86us/step - loss: 0.2830 - categorical_accuracy: 0.8828 - val_loss: 0.3747 - val_categorical_accuracy: 0.8448\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "num_iterations = 1\n",
    "all_rocs = []\n",
    "all_accuracies = []\n",
    "all_precisions = []\n",
    "all_treatments = []\n",
    "model_types = []\n",
    "parameter_counts = []\n",
    "iteration_list = []\n",
    "\n",
    "train_dotAttention = True\n",
    "train_convolution = False\n",
    "train_large_convolution = True\n",
    "for iteration in range(1,num_iterations + 1):\n",
    "    print('iteration', iteration)\n",
    "    for treatment in sorted(treatment_positive_fasta_dict.keys()):\n",
    "        positive_sequence_arrays = treatment_positive_seqArray_dict[treatment] \n",
    "        positive_rc_sequence_arrays = treatment_positive_rc_seqArray_dict[treatment]\n",
    "        negative_sequence_arrays = treatment_negative_seqArray_dict[treatment]\n",
    "        negative_rc_sequence_arrays = treatment_negative_rc_seqArray_dict[treatment]\n",
    "\n",
    "        sequence_arrays = np.concatenate([positive_sequence_arrays, negative_sequence_arrays])\n",
    "        sequence_rc_arrays = np.concatenate([positive_rc_sequence_arrays, negative_rc_sequence_arrays])\n",
    "        labels = [1 for x in positive_sequence_arrays] + [0 for x in negative_sequence_arrays]\n",
    "        labels = np.array(labels)\n",
    "\n",
    "        x_train, x_test, x_rc_train, x_rc_test, y_train, y_test = model_selection.train_test_split(\n",
    "            sequence_arrays, \n",
    "            sequence_rc_arrays, \n",
    "            labels, \n",
    "            test_size=0.2)\n",
    "\n",
    "        num_classes = 2\n",
    "        y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "        y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "        ### dot product attention model ###\n",
    "        if train_dotAttention:\n",
    "            print('Training DotAttention', treatment)\n",
    "            dotProductAttention_model = get_dotProductAttention_model(200,\n",
    "                mode='classification',\n",
    "                num_motifs=attention_num_motifs, \n",
    "                motif_size=attention_motif_size,\n",
    "                adjacent_bp_pool_size=adjacent_bp_pool_size,\n",
    "                num_dense_neurons = attention_num_dense_neurons,  \n",
    "                dropout_rate=attention_dropout_rate,\n",
    "                num_classes=2)\n",
    "\n",
    "            dotProductAttention_model.fit([x_train], y_train,\n",
    "                      batch_size=64,\n",
    "                      epochs=num_epochs,\n",
    "                      verbose=1,\n",
    "                      validation_data=([x_test], y_test))\n",
    "\n",
    "            probs = dotProductAttention_model.predict([x_test])\n",
    "            test_predictions = [0 if x[0] > x[1] else 1 for x in probs]\n",
    "            roc = sklearn.metrics.roc_auc_score([y[1] for y in y_test], probs[:,1], )\n",
    "            precision = sklearn.metrics.precision_score([y[1] for y in y_test], [0 if x[0] > x[1] else 1 for x in probs])\n",
    "            acc = sklearn.metrics.accuracy_score([y[1] for y in y_test], test_predictions)\n",
    "            param_count = dotProductAttention_model.count_params()\n",
    "\n",
    "            all_rocs.append(roc)\n",
    "            all_accuracies.append(acc)\n",
    "            all_precisions.append(precision)\n",
    "            all_treatments.append(treatment)\n",
    "            model_types.append('DotAttention')\n",
    "            parameter_counts.append(param_count)\n",
    "            iteration_list.append(iteration)\n",
    "            condition_model_dict['DotAttention_'+treatment] = dotProductAttention_model\n",
    "        ### ###\n",
    "\n",
    "        ### large convolution model ###\n",
    "        if train_large_convolution:\n",
    "            print('Training Large Convolution', treatment)\n",
    "            large_convolution_model = get_convolution_model(200,\n",
    "                mode='classification',\n",
    "                num_motifs=largeConvolution_num_motifs, \n",
    "                motif_size=convolution_motif_size,\n",
    "                num_dense_neurons=large_num_dense_neurons,\n",
    "                dropout_rate=convolution_dropout_rate,\n",
    "                num_classes=2)\n",
    "            large_convolution_model.fit([x_train, x_rc_train], y_train,\n",
    "                      batch_size=64,\n",
    "                      epochs=num_epochs,\n",
    "                      verbose=1,\n",
    "                      validation_data=([x_test, x_rc_test], y_test))\n",
    "\n",
    "            probs = large_convolution_model.predict([x_test, x_rc_test])\n",
    "            test_predictions = [0 if x[0] > x[1] else 1 for x in probs]\n",
    "            roc = sklearn.metrics.roc_auc_score([y[1] for y in y_test], probs[:,1], )\n",
    "            precision = sklearn.metrics.precision_score([y[1] for y in y_test], [0 if x[0] > x[1] else 1 for x in probs])\n",
    "            acc = sklearn.metrics.accuracy_score([y[1] for y in y_test], test_predictions)\n",
    "            param_count = convolution_model.count_params()\n",
    "\n",
    "            all_rocs.append(roc)\n",
    "            all_accuracies.append(acc)\n",
    "            all_precisions.append(precision)\n",
    "            all_treatments.append(treatment)\n",
    "            model_types.append('LargeConvolution')\n",
    "            parameter_counts.append(param_count)\n",
    "            iteration_list.append(iteration)\n",
    "            condition_model_dict['LargeConvolution_'+treatment] = large_convolution_model\n",
    "\n",
    "        ### convolution model ###\n",
    "        if train_convolution:\n",
    "            print('Training Convolution', treatment)\n",
    "            convolution_model = get_convolution_model(200,\n",
    "                mode='classification',\n",
    "                num_motifs=convolution_num_motifs, \n",
    "                motif_size=convolution_motif_size,\n",
    "                num_dense_neurons=num_dense_neurons,\n",
    "                dropout_rate=convolution_dropout_rate,\n",
    "                num_classes=2)\n",
    "            convolution_model.fit([x_train, x_rc_train], y_train,\n",
    "                      batch_size=64,\n",
    "                      epochs=num_epochs,\n",
    "                      verbose=1,\n",
    "                      validation_data=([x_test, x_rc_test], y_test))\n",
    "\n",
    "            probs = convolution_model.predict([x_test, x_rc_test])\n",
    "            test_predictions = [0 if x[0] > x[1] else 1 for x in probs]\n",
    "            roc = sklearn.metrics.roc_auc_score([y[1] for y in y_test], probs[:,1], )\n",
    "            precision = sklearn.metrics.precision_score([y[1] for y in y_test], [0 if x[0] > x[1] else 1 for x in probs])\n",
    "            acc = sklearn.metrics.accuracy_score([y[1] for y in y_test], test_predictions)\n",
    "            param_count = convolution_model.count_params()\n",
    "\n",
    "            all_rocs.append(roc)\n",
    "            all_accuracies.append(acc)\n",
    "            all_precisions.append(precision)\n",
    "            all_treatments.append(treatment)\n",
    "            model_types.append('Convolution')\n",
    "            parameter_counts.append(param_count)\n",
    "            iteration_list.append(iteration)\n",
    "            condition_model_dict['Convolution_'+treatment] = convolution_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "performance_frame = pd.DataFrame({'Model Type': model_types, \n",
    "                                  'Treatment':all_treatments,\n",
    "                                  'Accuracy':all_accuracies, \n",
    "                                  'aucROC':all_rocs,\n",
    "                                  'Precision':all_precisions})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Model Type</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Treatment</th>\n",
       "      <th>aucROC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.860233</td>\n",
       "      <td>DotAttention</td>\n",
       "      <td>0.814514</td>\n",
       "      <td>il4-24h</td>\n",
       "      <td>0.926833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.846137</td>\n",
       "      <td>LargeConvolution</td>\n",
       "      <td>0.835491</td>\n",
       "      <td>il4-24h</td>\n",
       "      <td>0.925178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.865687</td>\n",
       "      <td>DotAttention</td>\n",
       "      <td>0.862601</td>\n",
       "      <td>kla-1h</td>\n",
       "      <td>0.920483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.834807</td>\n",
       "      <td>LargeConvolution</td>\n",
       "      <td>0.807774</td>\n",
       "      <td>kla-1h</td>\n",
       "      <td>0.903245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.862995</td>\n",
       "      <td>DotAttention</td>\n",
       "      <td>0.866582</td>\n",
       "      <td>veh</td>\n",
       "      <td>0.930970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.844849</td>\n",
       "      <td>LargeConvolution</td>\n",
       "      <td>0.860847</td>\n",
       "      <td>veh</td>\n",
       "      <td>0.921088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy        Model Type  Precision Treatment    aucROC\n",
       "0  0.860233      DotAttention   0.814514   il4-24h  0.926833\n",
       "1  0.846137  LargeConvolution   0.835491   il4-24h  0.925178\n",
       "2  0.865687      DotAttention   0.862601    kla-1h  0.920483\n",
       "3  0.834807  LargeConvolution   0.807774    kla-1h  0.903245\n",
       "4  0.862995      DotAttention   0.866582       veh  0.930970\n",
       "5  0.844849  LargeConvolution   0.860847       veh  0.921088"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEYCAYAAAC9Xlb/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcV3W+x/HXTxYbA01mQMuVizo6Go6gTkaZGS5XrVHT\nERds0nTIuozbqKCGtwSXGjd07pQ5amrGlJi7ZFxLJxcMTQMzzTEEJQEXBHRYz/3D628k2VQOGOf9\n/Ot3zvmdcz6/7wPeHL6/8/0em2EYBiIiUqPVqu4CRETEfAp7ERELUNiLiFiAwl5ExAIU9iIiFqCw\nFxGxAFPD/uTJk/j7+7N27drbtu3bt49BgwYxZMgQli1bZmYZIiKWZ1rYX7t2jTfeeIMuXbqUuH32\n7NlERkayfv16vvjiC7777juzShERsTzTwt7Z2Znly5fj4eFx27bk5GTq1avHww8/TK1atXjqqafY\nv3+/WaWIiFieaWHv6OjIAw88UOK29PR03Nzc7Mtubm6kp6ebVYqIiOU5VncBFRUfH1/dJYjUaL6+\nvtVdgpioWsLew8ODjIwM+/KFCxdK7O75Mf0wiojcnWq59bJx48ZkZ2eTkpJCQUEBu3fvxs/PrzpK\nERGxBNOu7BMSEpg3bx7nzp3D0dGRmJgYunfvTuPGjenRowezZs1i0qRJAPTp0wdPT0+zShERsTzb\nT2WK4/j4eHXjiIjcJY2gFRGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhag\nsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVE\nLEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFOFZ3AVI1hk1ZV+XnfH/+8Co/\n591S+0hNVyPCXr+oIiJlUzeOiIgFKOxFRCygRnTjVIffr/xjlZ9z1YuLq/ycIlIzKOxFqokuGKQq\nqRtHRMQCdGUvptGVq8j9Q1f2IiIWoLAXEbEAhb2IiAWY2mcfERHB0aNHsdlshIaG4u3tbd+2bt06\nNm/eTK1atWjXrh3Tp083sxQREUsz7co+Li6OpKQkoqKiCA8PJzw83L4tOzubFStWsG7dOtavX8/p\n06f56quvzCpFRMTyTAv7/fv34+/vD4CXlxeZmZlkZ2cD4OTkhJOTE9euXaOgoIDr169Tr149s0oR\nEbE808I+IyOD+vXr25fd3NxIT08HoHbt2rzyyiv4+/vz9NNP0759ezw9Pc0qRUTE8qrsPnvDMOyv\ns7Ozefvtt9m5cycuLi688MILnDhxgtatW5d5jPj4eLPLvK9Z/fNXhNqobGW1j6+vbxVWIlXNtLD3\n8PAgIyPDvpyWloa7uzsAp0+fpkmTJri5uQHQsWNHEhISyg37Un8Yo05UTtH3uXv6ZVQblU3tIzWc\nad04fn5+xMTEAJCYmIiHhwcuLi4ANGrUiNOnT/Ovf/0LgISEBJo3b25WKSIilmfalb2Pjw9t27Yl\nICAAm81GWFgY0dHRuLq60qNHD0aPHs3IkSNxcHCgQ4cOdOzY0axSREQsz9Q++8mTJxdbvrWbJiAg\ngICAADNPLyIi/08jaEVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxF\nRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ\n2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIi\nFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhbgaObBIyIiOHr0KDab\njdDQULy9ve3bUlNTmThxIvn5+fzqV7/i9ddfN7MUERFLM+3KPi4ujqSkJKKioggPDyc8PLzY9rlz\n5zJq1Cg++ugjHBwcOH/+vFmliIhYnmlhv3//fvz9/QHw8vIiMzOT7OxsAIqKioiPj6d79+4AhIWF\n8cgjj5hVioiI5ZnWjZORkUHbtm3ty25ubqSnp+Pi4sKlS5d48MEHmTNnDomJiXTs2JFJkyaVe8z4\n+Hizyv1JsPrnrwi1UdnKah9fX98qrESqmql99rcyDKPY6wsXLjBy5EgaNWrE2LFj+eyzz+jWrVuZ\nxyj1hzHqRCVWev+6p19GtVHZ1D5Sw5XbjXP69Om7OrCHhwcZGRn25bS0NNzd3QGoX78+jzzyCE2b\nNsXBwYEuXbpw6tSpuzqPiIiUr9ywDw4OZujQoWzYsIHr169X+MB+fn7ExMQAkJiYiIeHBy4uLgA4\nOjrSpEkTvv/+e/t2T0/PuyhfREQqotxunG3btnHy5El27NhBYGAgbdq0YfDgwcVuoyyJj48Pbdu2\nJSAgAJvNRlhYGNHR0bi6utKjRw9CQ0OZNm0ahmHQqlUr+5e1IiJS+SrUZ9+qVStatWqFn58fCxYs\nYNy4cTRr1ozw8HCaN29e6n6TJ08utty6dWv762bNmrF+/fq7q1pERO5IuWF/7tw5Nm7cyNatW2nR\nogVBQUE8+eSTfP311/zpT3/iww8/rIo6RUTkHpQb9oGBgQwaNIjVq1fToEED+3pvb+9yu3JEROT+\nUO4XtJs3b6Z58+b2oF+/fj05OTkAzJw509zqRESkUpQb9iEhIcVuobx+/TpTpkwxtSgREalc5Yb9\nlStXGDlypH151KhRXL161dSiRESkcpUb9vn5+cUGViUkJJCfn29qUSIiUrnK/YI2JCSEcePGkZWV\nRWFhIW5ubsyfP78qahMRkUpSbti3b9+emJgYLl++jM1m46GHHuLw4cNVUZuIiFSScsM+OzubTZs2\ncfnyZeBGt86GDRv4xz/+YXpxIiJSOcrtsx8/fjzffvst0dHR5OTksHv3bmbNmlUFpYmISGUpN+xz\nc3N5/fXXadSoEVOnTuW9995jx44dVVGbiIhUkgrdjXPt2jWKioq4fPkyDz30EMnJyVVRm4iIVJJy\nw/63v/0tf//73xk8eDB9+vShb9++/OIXv6iK2kRE7lh0dDR/+ctfSt3eo0ePKqzm/lHuF7Q3pygG\n6NKlCxcvXqRNmzamFyYi1nTw4EFeeuklYmNj8fDwAGD37t288sorHD9+vNLPt2TJEhITE/n+++/5\n2c9+RoMGDejWrRtDhw6t9HNVp3LDfuTIkaxZswaABg0aFJsMTUTEDD4+Pnz88ceMHTsWgJiYGPtF\nZmJiIpGRkbi5uZGbm8ucOXP4/vvvmT17Nq1ateLChQu0adOGwsJCZs6ciWEY5OTk8Morr/DLX/7y\ntnMFBwcDEBkZSdOmTfntb39LVlYW/fv35+OPPwZg2rRpDB48mBkzZtC3b1+ysrK4ePEi8+fP59Sp\nUyxcuBB3d3dsNhszZ87Eycmpilqq4soN+zZt2rB48WI6dOhQ7AN06dLF1MJExLp8fX3t43kuX75M\nnTp1qFOnDgALFy5k+vTpeHp6snDhQmJiYti3bx/BwcF07NiRN998E4Bdu3bx0EMPMWXKFM6fP09Y\nWBjLly+v0PldXV1p2bIlX375JY8++iinTp3C19eX3NxcnnvuOZo2bUpISAiHDx9m+fLl/OlPf6JF\nixYsXbqUXbt20adPH3Ma5h6UG/bffPMNAF9++aV9nc1mU9iLiGlsNhuPPfYYhw4d4ptvvmHAgAH2\nkfvnzp2jadOmADRu3JjU1FTS0tJo2LAhAE2aNOHSpUucPXuWI0eOMG3aNABq1Sr3K8piBg8ezIYN\nG7h06RI9e/a0r2/UqBFw4znb6enpJCcn8+677wJw9erV+/Y7zXLD/mYXjohIVRo4cCCLFi0iNze3\n2GSMjRs35uzZs3h6epKcnEzr1q355z//yfnz52ncuDFnzpyhXr16NG3aFD8/P1599VXy8vJITU29\no/N37tyZefPmceXKlWJji5KTk2nevDkpKSl069aNJk2a8F//9V80atSI1NRUXF1dK6sJKlW5YT9s\n2DD7F7S3WrdunSkFiYgA1K1bF0dHx9v62SdOnMibb75J/fr1KSwsJDg4mObNmxMeHs5//Md/kJWV\nRb169ejRowd79uwhJCSEixcvMnjwYJo1a3ZHNfTq1Yu4uDj7d5UODg5s2rSJH374AcMw+PWvf82E\nCROIiIjAxcWFrKws3njjDVxcXCqtHSqLzTAMo6w3xMXF2V/n5+dz4MAB6tSpw8svv2x6cbeKj4/H\n19e3xG3DplT9Hx7nNnHlv6mSrXpx8V3vqzYqm9pHSrJkyRJ+9atf4e/vD9y4bXPXrl3VXNXdKffK\nvnPnzsWW/fz8GDNmjGkFiYjcD0JCQoB/363zU1du2P94tGxqaipnzpwxrSARkfvBnDlzblv3U72q\nhwqE/QsvvGB/bbPZcHFx4dVXXzW1KBERqVzlhv3//u//UlRUZL9tKT8//74cMCAiIqUr98bTmJgY\nxo0bZ18ePnw4O3fuNLUoERGpXOWG/cqVK+0j0gBWrFjB3/72N1OLEhGRylVuN45hGMUGCbi6ut7x\nSDQR+Wmq7FtS358/vNz3pKSk8Oyzz9KuXTsMw8DBwYGgoKBSR+2fP3+ejIwMvL297etGjx5N7dq1\ni81+uXPnTnr37n3b6ztxc789e/aQkpLCsGHD7vgY1aXcsG/Xrh3jx4+nc+fOGIbB3r17adeuXVXU\nJiIW5enpaR+9f/bsWYKCgliwYAGtW7e+7b0HDhzg2rVr9rC/ePEip0+f5l//+hdZWVn2i9V33nmH\n3r17k5eXx6pVq+447G/dr2vXrvf4CateuWE/Y8YMNm/ezLFjx7DZbDz33HN39RdRRORuNG3alKCg\nIN5//328vLzYvn07AM888wyDBg1i6dKlODo68vDDD/PMM8+wfft2nn76aa5evconn3zC888/z7vv\nvsu3337Lq6++iru7O99++y2zZs1i5syZzJw5k+TkZAoKCggODqZLly4EBgbSpUsXDh48yOXLl/nr\nX//K8uXL7ft5e3tz6tQppk6dyurVq4vVNHbsWKZNm4a7uzvHjx/n/PnzvPXWW7Rt27Y6m7H8Pvvr\n16/j5OTEzJkzmTFjBpmZmVy/fr0qahMRAW70MHz++eds3LiRdevWsW7dOnbs2EF2djYDBgxg5MiR\nPPPMMwBs3bqVvn370q9fP3sIv/TSS7i4uLB06VJGjx6Np6cns2bNYsuWLbi7u7NmzRqWLVtGRESE\n/Zyurq6sXr2arl278sknnxTb76bk5OTbajp79ixw487FFStWMHLkSPtUydWp3LCfOnUqGRkZ9uXr\n168zZcoUU4sSEblVTk4OdevWpX379jg6OuLo6IiPjw8nTpwo9r7k5GQuXLiAr68vTzzxBCdOnODS\npUulHvfIkSPExsYSGBjIH//4R3Jzc8nLywOgY8eOADRs2JDs7OwS9//mm29Kraki+1elcrtxrly5\nUmzGuVGjRrF7925TixIRuVVCQgI5OTncOpVXfn7+bTeLbN26ldzcXPr37w9AQUEBO3bsYPjwkr8Y\ndnJyIigoiH79+t22zcHBwf66tCnEbDZbqTVVZP+qVKEHjp8+fdq+/PXXX5Ofn29qUSIiN509e5ZV\nq1Yxb948vvrqKwoKCigoKODo0aO0adMGm81GQUEBANu2bWPVqlVs2rSJTZs2sXTpUrZt2wb8O3Br\n1apFYWEhAO3btyc2Nha48cXuggULSq3j1v1uatOmTYk13Y/KvbIPCQlh3LhxZGVlUVRURP369e0P\nERCRmq0it0qa4cyZMwQGBpKXl0dhYSGvvfYanTp1YsiQIYwYMQLDMBg8eDCNGjWiQ4cOTJ06lezs\nbJydnYtNidyxY0cuXrxIamoqbdq0YdCgQaxfv578/HyCg4NZsGABBw4cICAggMLCwjKngnF3d7fv\n161bN+DG3Pol1XQ/KneK45tSU1M5ePAgGzdu5PTp0/zjH/8wu7ZiNMWxpjiuCE1xXDZNcWxd5V7Z\nf/XVV0RHR7N9+3aKiop44403ij2iS0RE7n+l9tkvX76cPn36MGHCBNzc3NiwYQNNmzalb9++mghN\nROQnptQr+0WLFtGiRQtee+01HnvsMYASH09YloiICI4ePYrNZiM0NLTYcOab/vznP/PVV1/pWbci\nIiYqNew/++wzNm7cSFhYGEVFRQwYMOCO7sKJi4sjKSmJqKgoTp8+TWhoKFFRUcXe891333Ho0CH9\npyAiYrJSu3Hc3d0ZO3YsMTExREREcPbsWc6dO0dQUBCff/55uQfev3+//bmNXl5eZGZm3jawYO7c\nuUyYMOEeP4KIiJSn3C9oATp16kSnTp2YMWMGW7duZdmyZTz11FNl7pORkVFsLgg3NzfS09PtT12P\njo6mc+fOd3SbUnx8fIXfWxNZ/fNXhNqobGW1T2l3u0nNUKGwv8nFxYWAgAACAgLu+ES33uF55coV\noqOjWblyJRcuXKjwMUr9YYw6UfL6GuaefhnVRmVT+5To9yv/WKnnr8itnykpKQQHBxMdHV2p5/6x\nFStWsHXrVh544AEMw2DChAn85je/MeVcFflMMTEx9OrVi+joaFxdXenRo0el1nBHYX8nPDw8is2p\nk5aWhru7O3BjStJLly4xfPhw8vLyOHv2LBEREYSGhppVjoiI3ZYtWzh06BBRUVE4Oztz5swZfv/7\n37N582bq1atX5fWkpKSwbds2evXqxcCBA005h2lh7+fnR2RkJAEBASQmJuLh4WHvwundu7d9muSU\nlBRCQkIU9CJSqn379rF48WKcnJyoW7cuixYt4siRI/ztb3/j2rVrTJ06lX379rFt2zaaNGlCQUEB\nL774Im3btiU0NJTMzEwKCwuZMWMGrVu3Zs2aNURERODs7AzcmD9/y5Yt1K1blx9++IHQ0FDy8/Ox\n2WyEh4djs9mYNm0aTZo04dtvv6VNmzZMnjyZgIAAYmJiANi4cSMnTpzgxRdfLHH/m7p3786WLVt4\n8MEHmTdvHi1btmTnzp0cO3aMpUuXYhgG9evXZ8SIEcyfP5/Dhw9TWFjI8OHD6d+/f4nTLz/yyCPl\ntqFpj5zy8fGhbdu2BAQEMHv2bMLCwoiOjmbXrl1mnVJEaqjMzEzeeust1q5di4uLi30E/8mTJ1mx\nYgWNGzdm3bp1REVFMWvWLOLiboxOXr16NU8++SSrV69m1qxZzJs3D4Bz587h5eVV7Bx169YFYPHi\nxQwaNIg1a9YwbNgwli5dCkBiYiITJ07ko48+4vPPP8fBwYGGDRty6tQpAGJjY+nVq1ep+5dl9OjR\ndO7cudh0DYcOHeLUqVN88MEHrF69mqVLl9pvcvnx9MsVYdqVPcDkyZOLLZf0lJnGjRvrHnsRKZOb\nmxszZsygsLCQ5ORkHnvsMR588EF++ctf4uzszIkTJ2jVqhUPPPAADzzwgH1Mz5EjR7h06RKbN28G\nsD+LwzAMDMMocexQQkICkyZNAuA3v/kNy5YtA248ROVmV7SHhwdZWVn07NmT3bt307RpU06dOkWH\nDh0ICwsrcf87lZCQQKdOnQCoU6cOLVq0ICkpCSg+ffKVK1cqdDxTw15EpDKEhobyzjvv4OXlxeuv\nv25ff7MbxjCMYtMd3wzxmw9e6tChQ7HjNWnShOPHjxd7xOqJEyfw8vIqNm1xaVMW3zynv78/48eP\np2XLljz55JPYbLZS9y9JWWOXfvyH6F6nT9aTw0Xkvpednc3DDz/M1atXOXjw4G0h2ahRI06dOkV+\nfj6XLl0iISEBuDGF8aeffgrcGMS5cuVKAF544QXmzZvHtWvXAPjnP//J+PHjuXr1Ko8++igHDx4E\nbnSllPXM7QYNGmCz2di6dSu9evUCKHd/FxcX0tPTKSws5OjRo8CN6ZNvTtN8U7t27ezHycnJ4ezZ\nszRr1uwOW+7fdGUvIqWqrlkyb05xfJO3tzdDhw6lefPmvPTSS0RGRjJx4kT79l/84hf069ePwYMH\n4+Xlhbe3Nw4ODowYMYKQkBCGDRtGUVER06dPB6BPnz7k5OQwZMgQ6tatS+3atVm0aBE///nPCQ4O\nZvr06fz973/HycmJiIiIMq/Au3fvznvvvcebb74JUO7+I0aMICgoCE9PT1q0aAHcGHh6/PhxIiIi\n7A9I79ixI+3atWP48OEUFBQwadIk6tSpc9dtWuEpjqubpjjWFMcVoSmOy1aTpziOjo6mX79+ODo6\n8uyzz7JixQoaNmxY3WXdN3RlLyI1QkZGBr/73e9wdnbm2WefVdD/iMJeRGqEsWPHMnbs2Oou476l\nL2hFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsR\nEQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU\n9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQBHMw8e\nERHB0aNHsdlshIaG4u3tbd924MABFixYQK1atfD09CQ8PJxatfS3R0TEDKala1xcHElJSURFRREe\nHk54eHix7a+99hpLlizhgw8+ICcnh71795pVioiI5ZkW9vv378ff3x8ALy8vMjMzyc7Otm+Pjo6m\nYcOGALi5uXH58mWzShERsTzTwj4jI4P69evbl93c3EhPT7cvu7i4AJCWlsYXX3zBU089ZVYpIiKW\nZ2qf/a0Mw7ht3cWLFwkKCiIsLKzYH4bSxMfHm1HaT4bVP39FqI3KVlb7+Pr6VmElUtVMC3sPDw8y\nMjLsy2lpabi7u9uXs7OzGTNmDOPHj+eJJ56o0DFL/WGMOnFPtf5U3NMvo9qobGofqeFM68bx8/Mj\nJiYGgMTERDw8POxdNwBz587lhRdeoGvXrmaVICIi/8+0K3sfHx/atm1LQEAANpuNsLAwoqOjcXV1\n5YknnuDjjz8mKSmJjz76CIB+/foxZMgQs8oREbE0U/vsJ0+eXGy5devW9tcJCQlmnlpERG6hUUwi\nIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo\n7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexER\nC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2\nIiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAaaGfUREBEOGDCEgIIBjx44V27Zv3z4GDRrEkCFDWLZs\nmZlliIhYnmlhHxcXR1JSElFRUYSHhxMeHl5s++zZs4mMjGT9+vV88cUXfPfdd2aVIiJieaaF/f79\n+/H39wfAy8uLzMxMsrOzAUhOTqZevXo8/PDD1KpVi6eeeor9+/ebVYqIiOWZFvYZGRnUr1/fvuzm\n5kZ6ejoA6enpuLm5lbhNREQqn2NVncgwjHs+Rnx8fInrJw1pfc/HvnNVf87SPn9FqI3Kpva5wdfX\nt4oqkapmWth7eHiQkZFhX05LS8Pd3b3EbRcuXMDDw6PM4+mHUETk7pnWjePn50dMTAwAiYmJeHh4\n4OLiAkDjxo3Jzs4mJSWFgoICdu/ejZ+fn1mliIhYns2ojP6VUrz11lt8+eWX2Gw2wsLCOH78OK6u\nrvTo0YNDhw7x1ltvAdCzZ09Gjx5tVhkiIpZnatiLiMj9QSNoRUQsQGEvImIBlg/7PXv28P777zNw\n4MBi6z/44AO6d+9e4j4nTpxg2LBhjBgxgnHjxnH9+nX7NsMwCAgIIDIyEoDIyEjWrl1r3geoItHR\n0cybN6/U5fKkpqYycODAYvvUlLa5V9OmTWP37t3VXYbUcJYP+65du9K1a9di6y5evMiuXbtK3Wf2\n7NlMmzaNtWvX0qxZM6Kjo+3bPvzwQ/Lz802r96cqNDSULl26VHcZIpZl+bCPjo5m/vz5xda9+eab\nBAcHl7rPX//6V7y9vYEbo3+vXLkCwKVLl9iyZQsBAQHF3n/y5En+8Ic/0KtXL/bs2VPJn6Dq/fnP\nf+aHH36wL8+ZM4ehQ4cycOBAPvzwwxL3iYyMxMvL67b1Na1tbjVgwADOnz8PwLlz5+jfvz+hoaEE\nBgYydOjQYlOEHDx4kNGjR9OnTx+OHz9eXSVLDWb5sP+xgwcPUrt2bdq3b1/qe26OF7h27RqbNm2i\nd+/ewI0/EhMmTMDBwaHY+69cucLbb7/NjBkz+OCDD8wrvgrs2LGD1NRUGjZsCEBubi6NGjVi/fr1\nvP/++yxevLjE/W622Y/VpLb5MX9/f3v3TGxsLP7+/ri7u7NmzRqWLVtGRESE/b02m40VK1YwcuRI\nNm7cWF0lSw1WZdMl/BTk5eWxZMkS/vKXv5T73mvXrvHyyy8zatQovLy8OHToEA4ODvj4+PD9998X\ne6+Pjw8ADRo0ICsry4zSq8SpU6f45JNP2L59O9u2bQOgdu3aZGZmEhAQgJOTE5cvX76jY9aUtilJ\nz549mTt3LsOHDyc2NhYnJyd++OEHDh8+DNz4Q5mXlwf8e4R4gwYNOHr0aLXVLDWXwv4W33zzDRkZ\nGYwZMwa4McXDhAkT6NOnD++99x4Aq1atwjAMxo0bR79+/exf7MbGxpKQkMDvfvc7Ll26RF5eHk2a\nNAHA0bFmNPO5c+do2bIlO3futK+Li4vjwIEDrFmzBicnJzp06ADAa6+9xpkzZ3j88cd5+eWXSz1m\nTWmbkrRs2ZK0tDRSU1PJysrCx8eH/v37069fv9vee+t/gxr6Imaoub9pd6F9+/b2KR4AunfvzsKF\nCwHo0aOHff3//M//0LlzZwYPHmxfN23aNPvr6Ohoex/tzbtyaoJu3boxZswYhg4dyvPPPw/A5cuX\nadiwIU5OTsTGxlJYWEheXh6vv/56NVd7f+jWrRsLFy6ke/fuNGvWjNjYWPr168fFixdZvXo1EydO\nrO4SxSJSSoNxAAAD1ElEQVTUZ38X1q1bx549ewgMDCQwMJClS5dWd0lVxs3NjeDgYN59910AHn/8\ncZKSkhgxYgTJycl069aNWbNmFdvnwoULBAYG8s4777B9+3YCAwMt87CaHj16sHXrVnr37s1//ud/\nUqdOHQICAggKCtLkflKlNF2CiIgF6MpeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQPfZW9T8+fP5\n+uuvyc3N5fjx4/bBUM8//zz9+/e/6+Nev36dvXv30rNnz8oq9TaHDx/G3d3dPmhNRMqnWy8tLiUl\nhWHDhlXaJGTx8fGsX7/e/shJM8yYMYM+ffrw+OOPm3YOkZpGV/ZSTGRkJCkpKZw/f56pU6fi5ubG\nf//3f3P9+nWuXbvGxIkTefzxxzl9+jRhYWE4ODiQnZ3N+PHj6dSpE9OnT+fq1avMnz+fFi1asHfv\nXgzD4Pjx4zz33HPk5+dz8OBBDMNg5cqV1KlTh+3bt7N27VoMw8DNzY3Zs2dTv359fH19CQoKYu/e\nvaSnp7No0SLOnj3Lzp07OXbsGCEhIZo2WaSiDLG05ORk48knn7QvL1myxBg2bJhRVFRkGIZhjBkz\nxti/f79hGIaRlpZmPP3000Z+fr5x4MABIy4uzjAMwzh8+LAxYMAAwzAMY8OGDcakSZPsr/39/Y3c\n3FwjOTnZaN26tXHgwAHDMAxjxIgRxq5du4zz588bzz77rJGbm2sYhmGsWrXKmDNnjmEYhtGqVSvj\ns88+MwzDMCIjI4033njDvu8XX3xharuI1DS6spfbtG/fHpvNBtyY8jknJ4dly5YBNyYuu3jxIu7u\n7syfP5+FCxeSn59vn9P/x9q1a4ezszMNGzakqKio2OyOWVlZHDlyhPT0dEaPHg3cmHm0cePG9v0f\ne+wxAB555BGSkpJM+8wiNZ3CXm7j5ORkf+3s7ExkZCRubm7F3vPiiy/St29fBg0axMmTJwkKCirx\nWD+e2//WWS4Nw8DZ2Rlvb2/efvvtcvc39PWSyF3TrZdSJl9fX3bs2AHceBJXeHg4ABkZGbRs2RKA\n7du32+dlr1WrFgUFBRU+/qOPPsqxY8dIT08Hbjwc5dNPPy1zH5vNpkc/itwhhb2Uafr06Xz66acM\nGzaMsWPH2rtVRo0axZQpUxg9ejS+vr7Uq1ePuXPn8uijj/Lll18SEhJSoeM3aNCA6dOn84c//IHh\nw4fz0Ucf8etf/7rMffz8/AgLC+OTTz65588nYhW69VJExAJ0ZS8iYgEKexERC1DYi4hYgMJeRMQC\nFPYiIhagsBcRsQCFvYiIBSjsRUQs4P8Aj76Z+wPoP7EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7146b67a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with sns.axes_style('whitegrid'):\n",
    "    sns.factorplot(data = performance_frame,\n",
    "        x = 'Treatment', \n",
    "        y = 'Accuracy',\n",
    "        hue = 'Model Type',\n",
    "        kind = 'bar'\n",
    "        )\n",
    "    plt.ylim(0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEYCAYAAAC9Xlb/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtYVXW+x/H3losdAz0yA2riLdLRMEwwJ2MyMlDHaEZN\nR/LWhcnochhvz4io4UnBS443tDNdPGlqxozRxfvtWHYKxZDRATPREkFRQBRBDYHW+YPjHkmgrbJA\nWZ/X8/g8e62111rf/Xvgw/K3f+u3bIZhGIiISIPWqL4LEBER8ynsRUQsQGEvImIBCnsREQtQ2IuI\nWIDCXkTEAkwN+8OHDxMcHMyqVauu2fbVV18xZMgQhg0bxtKlS80sQ0TE8kwL+4sXLzJjxgx69epV\n5faZM2cSHx/PmjVr+PLLLzly5IhZpYiIWJ5pYe/q6srbb7+Nl5fXNduysrJo1qwZrVq1olGjRjzy\nyCMkJSWZVYqIiOWZFvbOzs7ccccdVW7Ly8vDw8PDvuzh4UFeXp5ZpYiIWJ5zfRfgqJSUlPouQaRB\nCwgIqO8SxET1EvZeXl7k5+fbl0+fPl1ld89P6YdRROTG1MvQS29vb4qLi8nOzqasrIydO3cSGBhY\nH6WIiFiCaVf2aWlpzJkzhxMnTuDs7MyWLVvo06cP3t7ehISEMH36dCZMmADAgAED6NChg1mliIhY\nnu12meI4JSVF3TgiIjdId9CKiFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexER\nC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2\nIiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAKc67sAqRvD/7y6zs/5/twR\ndX7OG6X2kYZOYS9ST5559091fs7lzy6q83PKrUHdOCIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEa\njXODNJJCRG4nurIXEbEAhb2IiAUo7EVELMDUPvu4uDj279+PzWYjOjoaPz8/+7bVq1fz6aef0qhR\nI7p27cqUKVPMLEVExNJMC/vk5GQyMzNJSEjg6NGjREdHk5CQAEBxcTHLli1j69atODs789xzz/GP\nf/yD+++//4bOVR/zmrh2qfNTiojcMNPCPikpieDgYAB8fHwoLCykuLgYNzc3XFxccHFx4eLFizRp\n0oRLly7RrFkzs0qReqIRSyK3DtP67PPz82nevLl92cPDg7y8PAAaN27Myy+/THBwMI8++ijdunWj\nQ4cOZpUiImJ5dTbO3jAM++vi4mLefPNNNm/ejJubG08//TSHDh2ic+fONR4jJSXF7DJvaVb//I5Q\nG9WspvYJCAiow0qkrpkW9l5eXuTn59uXc3Nz8fT0BODo0aO0adMGDw8PAHr06EFaWtrPhn21P4wJ\nh2qn6FvcTf0yqo1qpvaRBs60bpzAwEC2bNkCQHp6Ol5eXri5uQHQunVrjh49yg8//ABAWloa7du3\nN6sUERHLM+3K3t/fH19fX8LCwrDZbMTExJCYmIi7uzshISGEh4czevRonJyc6N69Oz169DCrFBER\nyzO1z37ixImVlq/upgkLCyMsLMzM04uIyP/THbQiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsR\nEQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU\n9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiI\nBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7\nERELcDbz4HFxcezfvx+bzUZ0dDR+fn72bTk5OYwfP57S0lLuvfdeXnvtNTNLERGxNNOu7JOTk8nM\nzCQhIYHY2FhiY2MrbZ89ezbPPfcca9euxcnJiZMnT5pVioiI5ZkW9klJSQQHBwPg4+NDYWEhxcXF\nAPz444+kpKTQp08fAGJiYrjrrrvMKkVExPJM68bJz8/H19fXvuzh4UFeXh5ubm4UFBRw5513MmvW\nLNLT0+nRowcTJkz42WOmpKSYVe5tweqf3xFqo5rV1D4BAQF1WInUNVP77K9mGEal16dPn2b06NG0\nbt2aMWPG8NlnnxEUFFTjMar9YUw4VIuV3rpu6pdRbVQztY80cKZ143h5eZGfn29fzs3NxdPTE4Dm\nzZtz11130bZtW5ycnOjVqxcZGRlmlSIiYnkOXdnv3r2blStXUlhYWOkKffXq1dXuExgYSHx8PGFh\nYaSnp+Pl5YWbm1vFSZ2dadOmDceOHaN9+/akp6fz+OOP3+RHERGR6jgU9jExMbz44ovX9SWqv78/\nvr6+hIWFYbPZiImJITExEXd3d0JCQoiOjiYqKgrDMOjUqZP9y1oREal9DoW9t7c3AwcOvO6DT5w4\nsdJy586d7a/btWvHmjVrrvuYIiJy/RwK+4cffpiEhAR69uyJs/O/dmnTpo1phYmISO1xKOzfe+89\nAN588037OpvNxo4dO8ypSkREapVDYf8///M/ZtchIiImcijsc3NzWbhwIf/85z+x2Wzcf//9jB07\nFg8PD7PrExGRWuDQOPtXX30VX19f5s+fz7x587j77ruJjo42uzYREaklDl3ZX7p0iREjRtiXO3Xq\npK4dEZHbiENX9pcuXSI3N9e+fOrUKS5fvmxaUSIiUrscurJ/6aWXGDx4MJ6enhiGQUFBwTVTFouI\nyK3LobAPCgpi+/btHDt2DIAOHTrQuHFjM+sSEZFaVGPYf/jhhzz55JMsWrSoyu1/+tOfTClKRERq\nV41h36hRRZe+k5NTnRQjIiLmqDHsBw0aBMArr7xCcXExbm5u5Ofnc+zYMfz9/eukQBERuXkOjcaZ\nMWMGmzZt4ty5c4SFhbFq1SqmT59ucmkiItcvMTGRN954o9rtISEhdVjNrcOhsD948CBDhw5l06ZN\nDBo0iIULF5KZmWl2bSJiQXv27OG+++6rNNx7586d3Hvvvaacb/Hixbzwwgv069ePgQMH8sILLzTI\nGXkdGo1z5YEln332GWPHjgXQOHsRMY2/vz8ff/wxY8aMAWDLli106dIFgPT0dOLj4/Hw8KCkpIRZ\ns2Zx7NgxZs6cSadOnTh9+jRdunShvLycadOmYRgGFy5c4OWXX+ZXv/rVNeeKjIwEID4+nrZt2/L7\n3/+eoqIiBg4cyMcffwxAVFQUQ4cOZerUqTz++OMUFRVx5swZ5s6dS0ZGBgsWLMDT0xObzca0adNw\ncXGpo5ZynENX9u3bt2fAgAFcuHCBLl268PHHH9OsWTOzaxMRiwoICGDfvn0AnD17liZNmtCkSRMA\nFixYwKRJk4iLi8Pb25stW7bw7rvvEhkZydSpU2nbti0A27Zt49///d+ZNWsWUVFRzJs3z+Hzu7u7\n07FjR77++mtKSkrIyMggICCAkpISfve73zF58mRcXFzYt28f8+fPZ+LEicyYMQMvLy+2bdtW+w1S\nCxy6so+NjeXw4cP4+PgAcM899/D666+bWpiIWJfNZuPBBx9k7969fPPNNwwaNIi5c+cCcOLECXug\ne3t7k5OTQ25uLi1btgQqnrNRUFDA8ePHSU1NJSoqCvjX6EJHDR06lA8//JCCggL69u1rX9+6dWug\n4jnbeXl5ZGVl8c477wBw/vx5fvnLX97chzeJQ+PslyxZUuV2jbMXEbMMHjyYhQsXUlJSwujRo+3r\nvb29OX78OB06dCArK4vOnTvz3XffcfLkSby9vfn+++9p1qwZbdu2JTAwkFdeeYXLly+Tk5NzXefv\n2bMnc+bM4dy5c5UGpGRlZdG+fXuys7MJCgqiTZs2/Md//AetW7cmJycHd3f32mqCWqVx9iJyS2ra\ntCnOzs7X9LOPHz+e119/nebNm1NeXk5kZCTt27cnNjaWu+++m6KiIpo1a0ZISAi7du1i8uTJnDlz\nhqFDh9KuXbvrqqFfv34kJyfTokULoCILP/nkE06dOoVhGNx///2MGzeOuLg43NzcKCoqYsaMGbi5\nudVaO9QWm3Hl29calJeXk5qaSo8ePYCKh5kEBQVd93+LbkZKSgoBAQFVbhv+59V1VscVrl2S6/yc\ny5+t+k5mR6iNaqb2kaosXryYe++9l+DgYKBi2Oat2if/cxxK65iYGD7//HP78u7du5kyZYppRYmI\n1LfJkyeTk5NjD/rbnUNf0F4Z1nRFdHQ0I0eONK0oEZH6NmvWrGvW3a5X9eDglf0PP/zAuXPn7Mun\nT5/WOHsRkduIQ1f2L7/8MqGhobRq1Yry8nJyc3M1n72IyG3EobB/9NFH2b59O0eOHMFms3H33Xfz\nb//2b2bXJiIitcShbpzCwkIWLVrE8uXL8fX1JSkpiYKCArNrExGRWuLQlf3UqVN54IEHSE1NBSrm\nxZk0aRJvv/22qcWJSP2q7SGp788d8bPvyc7O5oknnqBr164YhoGTkxMRERH06tWryvefPHmS/Px8\n/Pz87OvCw8Np3LhxpdkvN2/eTP/+/a95fT2u7Ldr1y6ys7MZPnz4dR+jvjh0ZV9QUMDo0aPtk/v0\n79+fH374wdTCRMS6OnTowMqVK1m1ahUzZsxgxowZHDp0qMr37t69mwMHDtiXz5w5w9GjR9m3bx9F\nRUX29W+99RZQcbG6fPny667p6v169+59WwU9OHhlD1BaWorNZgMgPz+fixcvmlaUiMgVbdu2JSIi\ngvfffx8fHx82btwIwGOPPcaQIUNYsmQJzs7OtGrViscee4yNGzfy6KOPcv78ebZu3cqTTz7JO++8\nw7fffssrr7yCp6cn3377LdOnT2fatGlMmzaNrKwsysrKiIyMpFevXowaNYpevXqxZ88ezp49y1//\n+lfefvtt+35+fn5kZGQwadIkVqxYUammMWPGEBUVhaenJwcPHuTkyZPMmzcPX1/f+mxGx67sR4wY\nwZAhQzhy5AgRERH8/ve/Jzw83OzaREQA6Nq1K59//jkfffQRq1evZvXq1WzatIni4mIGDRrE6NGj\neeyxxwBYv349jz/+OKGhofYQ/uMf/4ibmxtLliwhPDycDh06MH36dNatW4enpycrV65k6dKlxMXF\n2c/p7u7OihUr6N27N1u3bq203xVZWVnX1HT8+HGg4gJ52bJljB492j5Vcn1y6Mp+wIAB+Pv7k5qa\niqurK6+99hpeXl5m1yYiAsCFCxdo2rQp3bp1w9m5Irb8/f2v6drJysri9OnTBAQEUFZWxtSpUyko\nKMDDw6PK46amppKSkmKfTrmkpMR+D9GV6WFatmxZ6T6jq33zzTfV1nT1/ld3M9UXh8J+7NixLFy4\nkN/+9rdm1yMico20tDQuXLjA1VN5lZaWXjM/1/r16ykpKWHgwIEAlJWVsWnTJkaMqPqLYRcXFyIi\nIggNDb1m29UTQFY3hZjNZqu2Jkf2r0sOdeN4e3uzdu1ajh49SlZWlv2fiIjZjh8/zvLly5kzZw7/\n+Mc/KCsro6ysjP3799OlSxdsNhtlZWUAbNiwgeXLl/PJJ5/wySefsGTJEjZs2AD8K3AbNWpEeXk5\nAN26dWPHjh1AxRe78+fPr7aOq/e7okuXLlXWdCty6Mp+48aN1/wFs9ls9kYSkYbJkaGSZvj+++8Z\nNWoUly9fpry8nFdffZUHHniAYcOGMXLkSAzDYOjQobRu3Zru3bszadIkiouLcXV1rTQlco8ePThz\n5gw5OTl06dKFIUOGsGbNGkpLS4mMjGT+/Pns3r2bsLAwysvLeeWVV6qtydPT075fUFAQUHEhXFVN\nt6IapzguLi7mjTfe4LvvvqNHjx48/fTT9fZsRU1xrCmOHaEpjmumKY6tq8ZunCvfOg8bNoyjR49W\nukFBRERuHzV245w4ccL+kN7evXvzzDPP1EVNIiJSy2q8sr8ynAhu7NGEcXFxDBs2jLCwsGqHHv3l\nL39h1KhR131sERFxXI1hf+WO2eqWa5KcnExmZiYJCQnExsZWOSXykSNH2Lt3r8PHFBGRG1NjN05q\naqr9W2eoGJoUFBSEYRjYbDY+++yzavdNSkqyP87Lx8eHwsJCiouLKz2Id/bs2YwbN44lS5bc3KcQ\nEZEa1Rj2mzdvvuED5+fnV5oLwsPDg7y8PHvYJyYm0rNnz+sappSSknLD9TQEVv/8jlAb1aym9qlu\ntJs0DDWGfW2OF716hOe5c+dITEzk3Xff5fTp0w4fo9ofxoSqZ8NraG7ql1FtVDO1T5WeefdPtXp+\nR4Z+ZmdnExkZSWJiYq2e+6eWLVvG+vXrueOOOzAMg3HjxvHrX//alHM58pm2bNlCv379SExMxN3d\nnZCQkFqtweFZL6+Xl5cX+fn59uXc3Fw8PT2BiilJCwoKGDFiBJcvX+b48ePExcURHR1tVjkiInbr\n1q1j7969JCQk4Orqyvfff88zzzzDp59+SrNmzeq8nuzsbDZs2EC/fv0YPHiwKecwLewDAwOJj48n\nLCyM9PR0vLy87F04/fv3tz84IDs7m8mTJyvoRaRaX331FYsWLcLFxYWmTZuycOFCUlNT+e///m8u\nXrzIpEmT+Oqrr9iwYQNt2rShrKyMZ599Fl9fX6KjoyksLKS8vJypU6fSuXNnVq5cSVxcHK6urkDF\n/Pnr1q2jadOmnDp1iujoaPu07rGxsdhsNqKiomjTpg3ffvstXbp0YeLEiYSFhbFlyxYAPvroIw4d\nOsSzzz5b5f5X9OnTh3Xr1nHnnXcyZ84cOnbsyObNmzlw4ABLlizBMAyaN2/OyJEjmTt3Lvv27aO8\nvJwRI0YwcODAKqdfvuuuu362DR2aG+dG+Pv74+vrS1hYGDNnziQmJobExES2bdtm1ilFpIEqLCxk\n3rx5rFq1Cjc3N/73f/8XgMOHD7Ns2TK8vb1ZvXo1CQkJTJ8+neTkiruTV6xYwcMPP8yKFSuYPn06\nc+bMASruIfLx8al0jqZNmwKwaNEihgwZwsqVKxk+fLh9AEl6ejrjx49n7dq1fP755zg5OdGyZUsy\nMjIA2LFjB/369at2/5qEh4fTs2fPStM17N27l4yMDD744ANWrFjBkiVLKC4uBq6dftkRpl3ZA0yc\nOLHScufOna95j7e3NytXrjSzDBG5zXl4eDB16lTKy8vJysriwQcf5M477+RXv/oVrq6uHDp0iE6d\nOnHHHXdwxx132B9RmJqaSkFBAZ9++ikAly5dAiq+Q7wyqvCn0tLSmDBhAgC//vWvWbp0KVDxEJUr\nXdFeXl4UFRXRt29fdu7cSdu2bcnIyKB79+7ExMRUuf/1SktL44EHHgCgSZMm3HPPPWRmZgKOTb/8\nU6aGvYhIbYiOjuatt97Cx8eH1157zb7+SjeMYRiVpju+EuIuLi5MmzaN7t27VzpemzZtOHjwIF27\ndrWvO3ToED4+PpUmfaxuyuIr5wwODmbs2LF07NiRhx9+GJvNVu3+VSktLa1220//EN3s9MmmdeOI\niNSW4uJiWrVqxfnz59mzZ881Idm6dWsyMjIoLS2loKCAtLQ0oGIK4+3btwMVN3G+++67ADz99NPM\nmTPH/njV7777jrFjx3L+/Hnuu+8+9uzZA1R0pVz9B+GnWrRogc1mY/369fTr1w/gZ/d3c3MjLy+P\n8vJy9u/fD1RMn3xlmuYrunbtaj/OhQsXOH78OO3atbvOlvsXXdmLSLXqa5bMK1McX+Hn58dTTz1F\n+/bt+eMf/0h8fDzjx4+3b//lL39JaGgoQ4cOxcfHBz8/P5ycnBg5ciSTJ09m+PDh/Pjjj0yZMgWo\nePrehQsXGDZsGE2bNqVx48YsXLiQX/ziF0RGRjJlyhT+9re/4eLiQlxcXI1X4H369OG9997j9ddf\nB/jZ/UeOHElERAQdOnTgnnvuASpuPD148CBxcXG4u7sDFV01Xbt2ZcSIEZSVlTFhwgSaNGlyw21a\n4xTHtxJNcawpjh2hKY5r1pCnOE5MTCQ0NBRnZ2eeeOIJli1bRsuWLeu7rFuGruxFpEHIz8/nD3/4\nA66urjzxxBMK+p9Q2ItIgzBmzBjGjBlT32XcsvQFrYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDC\nXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGx\nAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEv\nImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIW4GzmwePi4ti/fz82m43o6Gj8/Pzs23bv3s38+fNp1KgR\nHTp0IDY2lkaN9LdHRMQMpqVrcnIymZmZJCQkEBsbS2xsbKXtr776KosXL+aDDz7gwoULfPHFF2aV\nIiJieaaFfVJSEsHBwQD4+PhQWFhIcXGxfXtiYiItW7YEwMPDg7Nnz5pVioiI5ZkW9vn5+TRv3ty+\n7OHhQV5enn3Zzc0NgNzcXL788kseeeQRs0oREbE8U/vsr2YYxjXrzpw5Q0REBDExMZX+MFQnJSXF\njNJuG1b//I5QG9WspvYJCAiow0qkrpkW9l5eXuTn59uXc3Nz8fT0tC8XFxfz/PPPM3bsWH7zm984\ndMxqfxgTDt1UrbeLm/plVBvVTO0jDZxp3TiBgYFs2bIFgPT0dLy8vOxdNwCzZ8/m6aefpnfv3maV\nICIi/8+0K3t/f398fX0JCwvDZrMRExNDYmIi7u7u/OY3v+Hjjz8mMzOTtWvXAhAaGsqwYcPMKkdE\nxNJM7bOfOHFipeXOnTvbX6elpZl5ahERuYruYhIRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuI\nWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCw\nFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQs\nQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELMDXs\n4+LiGDZsGGFhYRw4cKDStq+++oohQ4YwbNgwli5damYZIiKWZ1rYJycnk5mZSUJCArGxscTGxlba\nPnPmTOLj41mzZg1ffvklR44cMasUERHLMy3sk5KSCA4OBsDHx4fCwkKKi4sByMrKolmzZrRq1YpG\njRrxyCOPkJSUZFYpIiKWZ1rY5+fn07x5c/uyh4cHeXl5AOTl5eHh4VHlNhERqX3OdXUiwzBu+hgp\nKSlVrp8wrPNNH/v61f05q/v8jlAb1UztUyEgIKCOKpG6ZlrYe3l5kZ+fb1/Ozc3F09Ozym2nT5/G\ny8urxuPph1BE5MaZ1o0TGBjIli1bAEhPT8fLyws3NzcAvL29KS4uJjs7m7KyMnbu3ElgYKBZpYiI\nWJ7NqI3+lWrMmzePr7/+GpvNRkxMDAcPHsTd3Z2QkBD27t3LvHnzAOjbty/h4eFmlSEiYnmmhr2I\niNwadAetiIgFKOxFRCzA8mG/a9cu3n//fQYPHlxp/QcffECfPn2q3OfQoUMMHz6ckSNH8tJLL3Hp\n0iX7NsMwCAsLIz4+HoD4+HhWrVpl3geoI4mJicyZM6fa5Z+Tk5PD4MGDK+3TUNrmZkVFRbFz5876\nLkMaOMuHfe/evendu3eldWfOnGHbtm3V7jNz5kyioqJYtWoV7dq1IzEx0b7t73//O6WlpabVe7uK\njo6mV69e9V2GiGVZPuwTExOZO3dupXWvv/46kZGR1e7z17/+FT8/P6Di7t9z584BUFBQwLp16wgL\nC6v0/sOHD/PCCy/Qr18/du3aVcufoO795S9/4dSpU/blWbNm8dRTTzF48GD+/ve/V7lPfHw8Pj4+\n16xvaG1ztUGDBnHy5EkATpw4wcCBA4mOjmbUqFE89dRTlaYI2bNnD+Hh4QwYMICDBw/WV8nSgFk+\n7H9qz549NG7cmG7dulX7niv3C1y8eJFPPvmE/v37AxV/JMaNG4eTk1Ol9587d44333yTqVOn8sEH\nH5hXfB3YtGkTOTk5tGzZEoCSkhJat27NmjVreP/991m0aFGV+11ps59qSG3zU8HBwfbumR07dhAc\nHIynpycrV65k6dKlxMXF2d9rs9lYtmwZo0eP5qOPPqqvkqUBq7PpEm4Hly9fZvHixbzxxhs/+96L\nFy/y4osv8txzz+Hj48PevXtxcnLC39+fY8eOVXqvv78/AC1atKCoqMiM0utERkYGW7duZePGjWzY\nsAGAxo0bU1hYSFhYGC4uLpw9e/a6jtlQ2qYqffv2Zfbs2YwYMYIdO3bg4uLCqVOn2LdvH1Dxh/Ly\n5cvAv+4Qb9GiBfv376+3mqXhUthf5ZtvviE/P5/nn38eqJjiYdy4cQwYMID33nsPgOXLl2MYBi+9\n9BKhoaH2L3Z37NhBWloaf/jDHygoKODy5cu0adMGAGfnhtHMJ06coGPHjmzevNm+Ljk5md27d7Ny\n5UpcXFzo3r07AK+++irff/89Dz30EC+++GK1x2wobVOVjh07kpubS05ODkVFRfj7+zNw4EBCQ0Ov\nee/V/xvUrS9ihob7m3YDunXrZp/iAaBPnz4sWLAAgJCQEPv6//qv/6Jnz54MHTrUvi4qKsr+OjEx\n0d5He2VUTkMQFBTE888/z1NPPcWTTz4JwNmzZ2nZsiUuLi7s2LGD8vJyLl++zGuvvVbP1d4agoKC\nWLBgAX369KFdu3bs2LGD0NBQzpw5w4oVKxg/fnx9lygWoT77G7B69Wp27drFqFGjGDVqFEuWLKnv\nkuqMh4cHkZGRvPPOOwA89NBDZGZmMnLkSLKysggKCmL69OmV9jl9+jSjRo3irbfeYuPGjYwaNcoy\nD6sJCQlh/fr19O/fn9/+9rc0adKEsLAwIiIiNLmf1ClNlyAiYgG6shcRsQCFvYiIBSjsRUQsQGEv\nImIBCnsE9bl7AAADSElEQVQREQvQOHuLmjt3Lv/85z8pKSnh4MGD9puhnnzySQYOHHjDx7106RJf\nfPEFffv2ra1Sr7Fv3z48PT3tN62JyM/T0EuLy87OZvjw4bU2CVlKSgpr1qyxP3LSDFOnTmXAgAE8\n9NBDpp1DpKHRlb1UEh8fT3Z2NidPnmTSpEl4eHjwn//5n1y6dImLFy8yfvx4HnroIY4ePUpMTAxO\nTk4UFxczduxYHnjgAaZMmcL58+eZO3cu99xzD1988QWGYXDw4EF+97vfUVpayp49ezAMg3fffZcm\nTZqwceNGVq1ahWEYeHh4MHPmTJo3b05AQAARERF88cUX5OXlsXDhQo4fP87mzZs5cOAAkydP1rTJ\nIo4yxNKysrKMhx9+2L68ePFiY/jw4caPP/5oGIZhPP/880ZSUpJhGIaRm5trPProo0Zpaamxe/du\nIzk52TAMw9i3b58xaNAgwzAM48MPPzQmTJhgfx0cHGyUlJQYWVlZRufOnY3du3cbhmEYI0eONLZt\n22acPHnSeOKJJ4ySkhLDMAxj+fLlxqxZswzDMIxOnToZn332mWEYhhEfH2/MmDHDvu+XX35paruI\nNDS6spdrdOvWDZvNBlRM+XzhwgWWLl0KVExcdubMGTw9PZk7dy4LFiygtLTUPqf/T3Xt2hVXV1da\ntmzJjz/+WGl2x6KiIlJTU8nLyyM8PByomHnU29vbvv+DDz4IwF133UVmZqZpn1mkoVPYyzVcXFzs\nr11dXYmPj8fDw6PSe5599lkef/xxhgwZwuHDh4mIiKjyWD+d2//qWS4Nw8DV1RU/Pz/efPPNn93f\n0NdLIjdMQy+lRgEBAWzatAmoeBJXbGwsAPn5+XTs2BGAjRs32udlb9SoEWVlZQ4f/7777uPAgQPk\n5eUBFQ9H2b59e4372Gw2PfpR5Dop7KVGU6ZMYfv27QwfPpwxY8bYu1Wee+45/vznPxMeHk5AQADN\nmjVj9uzZ3HfffXz99ddMnjzZoeO3aNGCKVOm8MILLzBixAjWrl3L/fffX+M+gYGBxMTEsHXr1pv+\nfCJWoaGXIiIWoCt7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgH/B/N1\nVPwCPRwtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7139d456a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with sns.axes_style('whitegrid'):\n",
    "    sns.factorplot(data = performance_frame,\n",
    "        x = 'Treatment', \n",
    "        y = 'Precision',\n",
    "        hue = 'Model Type',\n",
    "        kind = 'bar'\n",
    "        )\n",
    "    plt.ylim(0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEYCAYAAAC9Xlb/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8D3e+x/HXTy6UYOVsgopLmrJRGitJKdmimmBt7EFZ\ncWtXlabbrnXbVoLGthL3ahv2VLEoqjmrKXWNy7Gr6xYNy4lUXbYiISQRIkFJYs4fOX4rchFkQjPv\n51+Z7/xm5vP7PpL3b/L9zXzHZhiGgYiIVGnVHnYBIiJiPoW9iIgFKOxFRCxAYS8iYgEKexERC1DY\ni4hYgKlhf+zYMQIDA1mxYkWxdbt376Zfv34MGDCA+fPnm1mGiIjlmRb2V69e5b333qNDhw4lrp86\ndSrR0dGsWrWKXbt2ceLECbNKERGxPNPC3tnZmYULF+Lu7l5sXUpKCnXr1qVhw4ZUq1aNzp07s2fP\nHrNKERGxPNPC3tHRkRo1apS4LiMjA1dXV/uyq6srGRkZZpUiImJ5jg+7gPJKSEh42CWIVGl+fn4P\nuwQx0UMJe3d3dzIzM+3L58+fL3G45076ZRQRuT8P5dJLDw8PcnNzSU1NJT8/nx07dhAQEPAwShER\nsQTTzuwTExOZMWMGZ86cwdHRkbi4OLp27YqHhwdBQUFMmTKFcePGAdCzZ088PT3NKkVExPJsP5Yp\njhMSEjSMIyJyn3QHrYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2I\niAUo7EVELEBhLyJiAT+a+exFzDTorZWVfszPZg6u9GOKdVWJsH8Yf6jOLeMr/ZhLh31Y6ccUkaqh\nSoS93J3OXB89v13yh0o/pk4YrEtj9iIiFqCwFxGxAIW9iIgFaMxeTKMxaZFHh87sRUQsQGEvImIB\nCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5E\nxAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERCzD1GbRRUVEcOnQIm81GeHg4Pj4+9nUr\nV67kq6++olq1arRu3ZqJEyeaWYqIiKWZdmYfHx9PcnIyMTExREZGEhkZaV+Xm5vL4sWLWblyJatW\nreLkyZP885//NKsUERHLMy3s9+zZQ2BgIABeXl5kZ2eTm5sLgJOTE05OTly9epX8/HyuXbtG3bp1\nzSpFRMTyTAv7zMxM6tWrZ192dXUlIyMDgOrVq/PGG28QGBjI888/T5s2bfD09DSrFBERyzN1zP52\nhmHYf87NzWXBggVs3rwZFxcXXn75ZY4ePYq3t3eZ+0hISDC7zEea1d9/eaiPylZW//j5+VViJVLZ\nTAt7d3d3MjMz7cvp6em4ubkBcPLkSRo3boyrqysA/v7+JCYm3jXsS/1ljDlaMUU/4h7oj1F9VDb1\nj1Rxpg3jBAQEEBcXB8CRI0dwd3fHxcUFgEaNGnHy5El++OEHABITE2nWrJlZpYiIWJ5pZ/a+vr60\natWKkJAQbDYbERERxMbGUrt2bYKCghg+fDgvvfQSDg4OtG3bFn9/f7NKERGxPFPH7MePH19k+fZh\nmpCQEEJCQsw8vIiI/D/dQSsiYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQs\nQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiL\niFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhag\nsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEARzN3HhUVxaFD\nh7DZbISHh+Pj42Nfl5aWxtixY8nLy+Opp57i3XffNbMUERFLM+3MPj4+nuTkZGJiYoiMjCQyMrLI\n+unTp/PKK6+wevVqHBwcOHv2rFmliIhYnmlhv2fPHgIDAwHw8vIiOzub3NxcAG7evElCQgJdu3YF\nICIigscff9ysUkRELM+0YZzMzExatWplX3Z1dSUjIwMXFxeysrKoVasW06ZN48iRI/j7+zNu3Li7\n7jMhIcGscn8UrP7+y0N9VLay+sfPz68SK5HKZuqY/e0Mwyjy8/nz53nppZdo1KgRI0eO5G9/+xtd\nunQpcx+l/jLGHK3ASh9dD/THqD4qm/pHqjjThnHc3d3JzMy0L6enp+Pm5gZAvXr1ePzxx2nSpAkO\nDg506NCB48ePm1WKiIjlmRb2AQEBxMXFAXDkyBHc3d1xcXEBwNHRkcaNG3Pq1Cn7ek9PT7NKERGx\nPNOGcXx9fWnVqhUhISHYbDYiIiKIjY2ldu3aBAUFER4ezoQJEzAMgxYtWti/rBURkYp317D/5ptv\n8Pf3ty9funSJY8eO0a5du7vufPz48UWWvb297T83bdqUVatW3UutIiJyn8ocxtm4cSMTJ04kJyfH\n3nbp0iXeffdddu3aZXpxIiJSMcoM+2XLlrFs2TJq165tb2vWrBmLFi1iwYIFphcnIiIVo8ywd3R0\npEGDBsXaGzRoUORSShERebSVGfa3D9/c7ubNm1y8eNGUgkREpOKVGfZ+fn7MmTOHmzdv2tvy8vKI\njIy86w1QIiLy6Cjzapzx48cTHh7OCy+8gLe3NwUFBSQlJdGxY0fCwsIqq0YREXlAZYZ9rVq1+PDD\nD0lOTubEiRM4ODjQvHlzGjVqVFn1iYhIBbjrdfZ5eXns37+fpKQkqlWrRnZ2Ng0bNqRaNT33RETk\nx6LMxE5JSSE4OJidO3fSqFEjGjZsyI4dO+jduzfnzp2rrBpFROQBlXlmP2vWLEaPHs0vf/nLIu3b\ntm1jxowZzJ0719TiRESkYtz1zP7OoAcIDAwkOTnZtKJERKRilRn2Dg4O97VOREQeLWWGfY0aNTh5\n8mSx9mPHjlGjRg3TihIRuV+xsbH8+c9/LnV9UFBQJVbz6ChzzP71119nxIgRjBw5Eh8fH27evMnB\ngwdZsmQJ8+fPr6waRcRC9u3bx6uvvsr27dtxd3cHYMeOHbzxxhskJSVV+PE++ugjjhw5wqlTp3js\nsceoX78+Xbp0YeDAgRV+rIepzLAPCAhgwYIFfPzxx3z66ac89thjeHt7s2jRIp544onKqlFELMbX\n15c1a9YwcuRIAOLi4mjZsiVQ+LCj6OhoXF1duX79OtOmTePUqVNMnTqVFi1acP78eVq2bElBQQGT\nJ0/GMAyuXLnCG2+8wc9+9rNixxo1ahQA0dHRNGnShP/8z/8kJyeH3r17s2bNGgAmTJhA//79mTRp\nEr/61a/IycnhwoULzJw5k+PHjzN37lzc3Nyw2WxMnjwZJyenSuqp8rvrdfbNmzdnzpw5xdoLCgo0\nbi8ipvDz8+PAgQMAXLx4kZo1a1KzZk0A5s6dy8SJE/H09GTu3LnExcWxe/duRo0ahb+/P7NmzQJg\n69at/OQnP+Gtt97i7NmzREREsHDhwnIdv3bt2jRv3pxvvvmGp59+muPHj+Pn58f169f59a9/TZMm\nTQgLC+PAgQMsXLiQP/7xjzz55JPMmzePrVu30rNnT3M65gGUGfbnz59n+vTpnDhxgrZt2xIWFsZj\njz3Gt99+S1hYmP1TT0SkItlsNp599ln279/Pt99+S58+fZg5cyYAZ86coUmTJgB4eHiQlpZGenq6\nfYbexo0bk5WVxenTpzl48CATJkwAuOcbQfv3788XX3xBVlYW3bp1s7ffmkHA3d2djIwMUlJSWLRo\nEQCXL1/mpz/96YO9eZOUGfYRERF07tyZ1157jbVr1zJ9+nRcXV1Zt24dEydOrKwaRcSC+vbtywcf\nfMD169d56aWX7O0eHh6cPn0aT09PUlJS8Pb25l//+hdnz57Fw8OD77//nrp169KkSRMCAgJ48803\nuXHjBmlpafd0/Hbt2jFjxgwuXbrElClT7O0pKSk0a9aM1NRUunTpQuPGjfn9739Po0aNSEtLK/L8\nj0dJmWGfm5tr/5LC29ub9u3b06tXL9auXUutWrUqpUARsaY6derg6OhYbJx97NixzJo1i3r16lFQ\nUMCoUaNo1qwZkZGRPPHEE+Tk5FC3bl2CgoLYuXMnYWFhXLhwgf79+9O0adN7qqF79+7Ex8dTv359\noPCS87Vr13Lu3DkMw+DnP/85Y8aMISoqChcXF3JycnjvvfdwcXGpsH6oKGWG/Z3/9rRo0YJJkyaZ\nWpCIWFv79u1p3749AOHh4fb25cuXA9CyZctil1Y+9dRTrFy5sti+oqKiirVt3bq1xOP+/ve/L9b2\nww8/EBISUqTtD3/4Q5Flb2/vH8XVifc0iGWz2cyqQ0TkkRIWFkZaWhqBgYEPu5QKUeaZ/cmTJ3nr\nrbdKXb71hYmISFUzbdq0Ym2l/VfwY3DXh5fcrkOHDqYWIyIi5igz7Pv06QMUXlN/8OBB/P39Afif\n//kfPZZQRORHpFxj9hEREfz973+3L+/du1eXXoqI/IiUK+xPnTrFuHHj7Mvh4eGkpKSYVpSIiFSs\nu06XAIWXH126dImf/OQnQOGdtTdu3DC1MBF5+Aa9Vfxyxgfx2czBd31NamoqvXr1onXr1hiGgYOD\nA6GhoaV+Z3j27FkyMzPx8fGxtw0fPpzq1asXuURz8+bN9OjRo9jP9+LWdjt37iQ1NZVBgwbd8z4e\nlnKF/RtvvEFwcDANGzakoKCA9PR0IiMjza5NRCzK09PTfl396dOnCQ0N5f3338fb27vYa/fu3cvV\nq1ftYX/hwgVOnjzJDz/8QE5Ojv2O1k8++YQePXpw48YNli5des9hf/t2nTp1esB3WPnKFfbPP/88\n27Zt48SJE9hsNp544gkee+wxs2sTEaFJkyaEhoby2Wef4eXlxcaNGwF44YUX6NevH/PmzcPR0ZGG\nDRvywgsvsHHjRp5//nkuX77Mli1bePHFF1m0aBHfffcdb775Jm5ubnz33XdMmTKFyZMnM3nyZFJS\nUsjPz2fUqFF06NCBoUOH0qFDB/bt28fFixf5+OOPWbhwoX07Hx8fjh8/zttvv82yZcuK1DRy5Egm\nTJiAm5sbSUlJnD17ltmzZ9OqVauH2Y3lC/sPP/ywxPY77yQTETFD69atmTNnDocPH2b16tVA4URl\nPXr0oE+fPtSrV48XXngBgPXr1/PHP/6RnJwcVqxYwYsvvsirr77KwoULmTdvHqmpqRw6dIgpU6aw\nZs0a3NzciIqKIisri5dffpl169YBhTNfLlu2jNmzZ7NlyxaGDx9u3y42NhYonCfnyy+/LFYTQF5e\nHosXL2bVqlWsWbPmxxH2t09lnJeXx/79+3nqqadMK0pE5HZXrlyhTp06tGnTBkfHwtjy9fXl6NGj\nRV6XkpLC+fPn8fPzIz8/n0mTJpGVlYWrq2uJ+z148CAJCQn26ZSvX79u/z7y1qXmDRo04NKlSyVu\n/+2335Za0+3bHz58+EHefoUoV9i/+eabRZYLCgpKnEdCRMQMiYmJXLlyBcMw7G15eXnF5u9av349\n169fp3fv3gDk5+ezadMmBg8u+YthJycnQkNDCQ4OLrbu9pPc2497O5vNVmpN5dm+Mt3bBM//Lz8/\nn+Tk5IquRUSkmNOnT7N06VJmzJjBP//5T/Lz88nPz+fQoUO0bNkSm81Gfn4+ABs2bGDp0qWsXbuW\ntWvXMm/ePDZs2AD8O3CrVatGQUEBAG3atGH79u1A4Re777//fql13L7dLS1btiyxpkdRuc7sO3fu\nXGQStOzsbPsnp4hUXeW5VNIM33//PUOHDuXGjRsUFBTwzjvv8MwzzzBgwACGDBmCYRj079+fRo0a\n0bZtW95++21yc3NxdnYuMiWyv78/Fy5cIC0tjZYtW9KvXz9WrVpFXl4eo0aN4v3332fv3r2EhIRQ\nUFBQbBTjdm5ubvbtbs0g4OHhUWJNjyKbUY7/L86cOcPVq1dJTEzEZrPh7OxMdHQ0mzZtqowaAUhI\nSMDPz6/EdRV9LXB5OLeMr/RjLh1W8hfl5aE+Kpv6R6q6cp3ZL126lH/84x9kZmbSpEkTUlJSeOWV\nV8yuTUREKki5xuwPHz7Mpk2b8Pb25osvvuAvf/kL165dM7s2ERGpIOUKe2dnZ6Dwm2bDMGjdurX9\nUqWyREVFMWDAAEJCQkq99GjOnDkMHTr0HkoWEZF7Va5hHE9PT1auXIm/vz/Dhg3D09OTnJycMreJ\nj48nOTmZmJgYTp48SXh4ODExMUVec+LECfbv34+Tk9P9vwMREbmrcoX9n/70J7Kzs6lTpw4bNmzg\nwoULvPbaa2Vus2fPHvvjvLy8vMjOziY3N7fIg3inT5/OmDFjmDdv3gO8BRERuZtyhb3NZrPPeNmr\nV69y7TgzM7PI7cGurq5kZGTYwz42NpZ27drd02VKCQkJ5X5tVWT1918e6qOyldU/pV3tJlVDucK+\nItx+heelS5eIjY1lyZIlnD9/vtz7KPWXMeZoye1VzAP9MaqPyqb+KdFvl1Ts/FflufQzNTWVUaNG\n2eefMcvixYtZv349NWrUwDAMxowZQ/v27U05VnneU1xcHN27dyc2NpbatWsTFBRUoTWYFvbu7u5k\nZmbal9PT03FzcwMKpyTNyspi8ODB3Lhxg9OnTxMVFUV4eLhZ5YiI2K1bt479+/cTExODs7Mz33//\nPb/97W/56quvqFu3bqXXk5qayoYNG+jevTt9+/Y15RimhX1AQADR0dGEhIRw5MgR3N3d7UM4PXr0\nsM8Ml5qaSlhYmIJeREq1e/duPvzwQ5ycnKhTpw4ffPABBw8e5C9/+QtXr17l7bffZvfu3WzYsIHG\njRuTn5/PsGHDaNWqFeHh4WRnZ1NQUMCkSZPw9vZm+fLlREVF2a809PT0ZN26ddSpU4dz584RHh5O\nXl4eNpuNyMhIbDYbEyZMoHHjxnz33Xe0bNmS8ePHExISQlxcHABffvklR48eZdiwYSVuf0vXrl1Z\nt24dtWrVYsaMGTRv3pzNmzdz+PBh5s2bh2EY1KtXjyFDhjBz5kwOHDhAQUEBgwcPpnfv3iVOv/z4\n44/ftQ/va26c8vD19aVVq1aEhIQwdepUIiIiiI2NZevWrWYdUkSqqOzsbGbPns2KFStwcXHhH//4\nBwDHjh1j8eLFeHh4sHLlSmJiYpgyZQrx8YV3Jy9btoznnnuOZcuWMWXKFGbMmAEUzgrg5eVV5Bh1\n6tQBCqd079evH8uXL2fQoEH2C0iOHDnC2LFjWb16NX//+99xcHCgQYMGHD9+HIDt27fTvXv3Urcv\ny/Dhw2nXrl2R6Rr279/P8ePH+fzzz1m2bBnz5s0jNzcX+Pf0y506dWLLli3l6kNTx+zHjx9fZLmk\np8x4eHjYn0gjIlISV1dXJk2aREFBASkpKTz77LPUqlWLn/3sZzg7O3P06FFatGhBjRo1qFGjhv2p\nVQcPHiQrK4uvvvoKwH4zqGEYGIZR5Iz7lsTERPszt9u3b8/8+fOBwoeo3BqKdnd3Jycnh27durFj\nxw6aNGnC8ePHadu2LRERESVuf68SExN55plnAKhZsyZPPvmkfQLK8ky/fKdK+4JWROR+hYeH88kn\nn+Dl5cW7775rb781DGMYRpHpjm+FuJOTE5MnT6Zt27ZF9te4cWOSkpJo3bq1ve3o0aN4eXkVmba4\ntCmLbx0zMDCQ0aNH07x5c5577jlsNlup25ckLy+v1HV3fhA96PTJpg3jiIhUlNzcXBo2bMjly5fZ\nt29fsZBs1KgRx48fJy8vj6ysLBITE4HCKYy3bdsGFN7EuWTJEgBefvllZsyYwdWrVwH417/+xejR\no7l8+TJPP/00+/btAwqHUm7/QLhT/fr1sdlsrF+/nu7duwPcdXsXFxcyMjIoKCjg0KFDQOH0ybem\nab6ldevW9v1cuXKF06dP07Rp03vsuX/Tmb2IlOphzZJ5a4rjW3x8fBg4cCDNmjXj1VdfJTo6mrFj\nx9rX//SnPyU4OJj+/fvj5eWFj48PDg4ODBkyhLCwMAYNGsTNmzeZOHEiAD179uTKlSsMGDCAOnXq\nUL16dT744AP+4z/+g1GjRjFx4kT++7//GycnJ6Kioso8A+/atSuffvops2bNArjr9kOGDCE0NBRP\nT0+efPJJoPDG06SkJKKiouwPSPf396d169YMHjyY/Px8xo0bR82aNe+7T8s1xfGjQFMca4rj8tAU\nx2WrylMcx8bGEhwcjKOjI7169WLx4sU0aNDgYZf1yNCZvYhUCZmZmfzmN7/B2dmZXr16KejvoLAX\nkSph5MiRjBw58mGX8cjSF7QiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIBCnsREQtQ2IuIWIDCXkTE\nAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCwFxGxAIW9\niIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQsQGEvImIB\nCnsREQtQ2IuIWICjmTuPiori0KFD2Gw2wsPD8fHxsa/bu3cv77//PtWqVcPT05PIyEiqVdNnj4iI\nGUxL1/j4eJKTk4mJiSEyMpLIyMgi69955x0++ugjPv/8c65cucLXX39tVikiIpZnWtjv2bOHwMBA\nALy8vMjOziY3N9e+PjY2lgYNGgDg6urKxYsXzSpFRMTyTAv7zMxM6tWrZ192dXUlIyPDvuzi4gJA\neno6u3btonPnzmaVIiJieaaO2d/OMIxibRcuXCA0NJSIiIgiHwylSUhIMKO0Hw2rv//yUB+Vraz+\n8fPzq8RKpLKZFvbu7u5kZmbal9PT03Fzc7Mv5+bmMmLECEaPHs0vfvGLcu2z1F/GmKMPVOuPxQP9\nMaqPyqb+kSrOtGGcgIAA4uLiADhy5Aju7u72oRuA6dOn8/LLL9OpUyezShARkf9n2pm9r68vrVq1\nIiQkBJvNRkREBLGxsdSuXZtf/OIXrFmzhuTkZFavXg1AcHAwAwYMMKscERFLM3XMfvz48UWWvb29\n7T8nJiaaeWgREbmN7mISEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiLiFiAwl5ExAIU9iIiFqCw\nFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhagsBcRsQCFvYiIBSjsRUQs\nQGEvImIBCnsREQtQ2IuIWIDCXkTEAhT2IiIWoLAXEbEAhb2IiAUo7EVELEBhLyJiAQp7ERELUNiL\niFiAwl5ExAIU9iIiFqCwFxGxAIW9iIgFKOxFRCxAYS8iYgEKexERCzA17KOiohgwYAAhISEcPny4\nyLrdu3fTr18/BgwYwPz5880sQ0TE8kwL+/j4eJKTk4mJiSEyMpLIyMgi66dOnUp0dDSrVq1i165d\nnDhxwqxSREQsz7Sw37NnD4GBgQB4eXmRnZ1Nbm4uACkpKdStW5eGDRtSrVo1OnfuzJ49e8wqRUTE\n8kwL+8zMTOrVq2dfdnV1JSMjA4CMjAxcXV1LXCciIhXPsbIOZBjGA+8jISGhxPZxA7wfeN/3rvKP\nWdr7Lw/1UdnUP4X8/PwqqRKpbKaFvbu7O5mZmfbl9PR03NzcSlx3/vx53N3dy9yffglFRO6facM4\nAQEBxMXFAXDkyBHc3d1xcXEBwMPDg9zcXFJTU8nPz2fHjh0EBASYVYqIiOXZjIoYXynF7Nmz+eab\nb7DZbERERJCUlETt2rUJCgpi//79zJ49G4Bu3boxfPhws8oQEbE8U8NeREQeDbqDVkTEAhT2IiIW\nYPmw37lzJ5999hl9+/Yt0v7555/TtWvXErc5evQogwYNYsiQIfzud7/j2rVr9nWGYRASEkJ0dDQA\n0dHRrFixwrw3UEliY2OZMWNGqct3k5aWRt++fYtsU1X65kFNmDCBHTt2POwypIqzfNh36tSJTp06\nFWm7cOECW7duLXWbqVOnMmHCBFasWEHTpk2JjY21r/vrX/9KXl6eafX+WIWHh9OhQ4eHXYaIZVk+\n7GNjY5k5c2aRtlmzZjFq1KhSt/n444/x8fEBCu/+vXTpEgBZWVmsW7eOkJCQIq8/duwYr732Gt27\nd2fnzp0V/A4q35w5czh37px9edq0aQwcOJC+ffvy17/+tcRtoqOj8fLyKtZe1frmdn369OHs2bMA\nnDlzht69exMeHs7QoUMZOHBgkSlC9u3bx/Dhw+nZsydJSUkPq2Spwiwf9nfat28f1atXp02bNqW+\n5tb9AlevXmXt2rX06NEDKPyQGDNmDA4ODkVef+nSJRYsWMCkSZP4/PPPzSu+EmzatIm0tDQaNGgA\nwPXr12nUqBGrVq3is88+48MPPyxxu1t9dqeq1Dd3CgwMtA/PbN++ncDAQNzc3Fi+fDnz588nKirK\n/lqbzcbixYt56aWX+PLLLx9WyVKFVdp0CT8GN27c4KOPPuLPf/7zXV979epVXn/9dV555RW8vLzY\nv38/Dg4O+Pr6curUqSKv9fX1BaB+/frk5OSYUXqlOH78OFu2bGHjxo1s2LABgOrVq5OdnU1ISAhO\nTk5cvHjxnvZZVfqmJN26dWP69OkMHjyY7du34+TkxLlz5zhw4ABQ+EF548YN4N93iNevX59Dhw49\ntJql6lLY3+bbb78lMzOTESNGAIVTPIwZM4aePXvy6aefArB06VIMw+B3v/sdwcHB9i92t2/fTmJi\nIr/5zW/Iysrixo0bNG7cGABHx6rRzWfOnKF58+Zs3rzZ3hYfH8/evXtZvnw5Tk5OtG3bFoB33nmH\n77//no4dO/L666+Xus+q0jclad68Oenp6aSlpZGTk4Ovry+9e/cmODi42Gtv/29Qt76IGaruX9p9\naNOmjX2KB4CuXbsyd+5cAIKCguzt//Vf/0W7du3o37+/vW3ChAn2n2NjY+1jtLeuyqkKunTpwogR\nIxg4cCAvvvgiABcvXqRBgwY4OTmxfft2CgoKuHHjBu++++5DrvbR0KVLF+bOnUvXrl1p2rQp27dv\nJzg4mAtSnFY0AAAD5ElEQVQXLrBs2TLGjh37sEsUi9CY/X1YuXIlO3fuZOjQoQwdOpR58+Y97JIq\njaurK6NGjWLRokUAdOzYkeTkZIYMGUJKSgpdunRhypQpRbY5f/48Q4cO5ZNPPmHjxo0MHTrUMg+r\nCQoKYv369fTo0YNf/vKX1KxZk5CQEEJDQzW5n1QqTZcgImIBOrMXEbEAhb2IiAUo7EVELEBhLyJi\nAQp7EREL0HX2FjVz5kz+93//l+vXr5OUlGS/GerFF1+kd+/e973fa9eu8fXXX9OtW7eKKrWYAwcO\n4ObmZr9pTUTuTpdeWlxqaiqDBg2qsEnIEhISWLVqlf2Rk2aYNGkSPXv2pGPHjqYdQ6Sq0Zm9FBEd\nHU1qaipnz57l7bffxtXVlT/96U9cu3aNq1evMnbsWDp27MjJkyeJiIjAwcGB3NxcRo8ezTPPPMPE\niRO5fPkyM2fO5Mknn+Trr7/GMAySkpL49a9/TV5eHvv27cMwDJYsWULNmjXZuHEjK1aswDAMXF1d\nmTp1KvXq1cPPz4/Q0FC+/vprMjIy+OCDDzh9+jSbN2/m8OHDhIWFadpkkfIyxNJSUlKM5557zr78\n0UcfGYMGDTJu3rxpGIZhjBgxwtizZ49hGIaRnp5uPP/880ZeXp6xd+9eIz4+3jAMwzhw4IDRp08f\nwzAM44svvjDGjRtn/zkwMNC4fv26kZKSYnh7ext79+41DMMwhgwZYmzdutU4e/as0atXL+P69euG\nYRjG0qVLjWnTphmGYRgtWrQw/va3vxmGYRjR0dHGe++9Z992165dpvaLSFWjM3sppk2bNthsNqBw\nyucrV64wf/58oHDisgsXLuDm5sbMmTOZO3cueXl59jn979S6dWucnZ1p0KABN2/eLDK7Y05ODgcP\nHiQjI4Phw4cDhTOPenh42Ld/9tlnAXj88cdJTk427T2LVHUKeynGycnJ/rOzszPR0dG4uroWec2w\nYcP41a9+Rb9+/Th27BihoaEl7uvOuf1vn+XSMAycnZ3x8fFhwYIFd93e0NdLIvdNl15Kmfz8/Ni0\naRNQ+CSuyMhIADIzM2nevDkAGzdutM/LXq1aNfLz88u9/6effprDhw+TkZEBFD4cZdu2bWVuY7PZ\n9OhHkXuksJcyTZw4kW3btjFo0CBGjhxpH1Z55ZVXeOuttxg+fDh+fn7UrVuX6dOn8/TTT/PNN98Q\nFhZWrv3Xr1+fiRMn8tprrzF48GBWr17Nz3/+8zK3CQgIICIigi1btjzw+xOxCl16KSJiATqzFxGx\nAIW9iIgFKOxFRCxAYS8iYgEKexERC1DYi4hYgMJeRMQCFPYiIhbwf82DeuPQ1zwtAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f714a62f208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with sns.axes_style('whitegrid'):\n",
    "    sns.factorplot(data = performance_frame,\n",
    "        x = 'Treatment', \n",
    "        y = 'aucROC',\n",
    "        hue = 'Model Type',\n",
    "        kind = 'bar'\n",
    "        )\n",
    "    plt.ylim(0,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}