run_stopping_point_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook runs the experiments in \"Stopping Criteria for Technology-Assisted Reviewsbased on Counting Processes\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# IMPORT LIBRARIES\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from scipy.optimize import curve_fit\n",
    "import random\n",
    "import glob\n",
    "from scipy.integrate import simps\n",
    "from scipy.stats import norm\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# IMPORT EXPERIEMENTAL FUNCTIONS\n",
    "from utils.read_data_fns import *\n",
    "from utils.target_method_fns import *  \n",
    "from utils.knee_method_fns import *  \n",
    "from utils.inhomogeneous_pp_fns import *   \n",
    "from utils.eval_fns import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# READ TOPIC RELEVANCE DATA\n",
    "with open('data/relevance/qrel_abs_test.txt', 'r') as infile:\n",
    "    qrels_data = infile.readlines()    \n",
    "query_rel_dic = make_rel_dic(qrels_data) # make dictionary of list of docids relevant to each queryid\n",
    "all_runs = glob.glob('data/runs2017_table3/Waterloo/B*')    \n",
    "\n",
    "\n",
    "# SET POISSON PROCESS PARAMETERS\n",
    "sample_props = [0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,\n",
    "                0.7,0.75,0.8,0.85,0.9,0.95,1]  # proportion of docs to sample\n",
    "min_rel_in_sample = 20 # min number rel docs must be initial sample to proceed with algorithm \n",
    "n_windows = 10  # number of windows to male from sample\n",
    "\n",
    "# SET KNEE METHOD PARAMETERS\n",
    "knee_rho = 6 # knee method rho \n",
    "\n",
    "# SET EXPERIMENTAL PARAMETERS (des_probs are specified below)\n",
    "des_recalls = [0.95, 0.9, 0.8, 0.7] # desired recalls to experiment over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fn to run stopping methods (oracle, knee method, target method, poisson process and cox process over all topics)\n",
    "def run_sp_algorithms(des_recall, des_prob):\n",
    "    \n",
    "    # PREPARE SCORING DICTIONARIES\n",
    "    run_score_dic = {}\n",
    "    oracle_dic = {}\n",
    "    \n",
    "    # LOOP OVER RUNS\n",
    "    for run in sorted(all_runs):\n",
    "\n",
    "\n",
    "        # MAKE DATA DICTIONARIES\n",
    "        with open(run, 'r') as infile:\n",
    "            run_data = infile.readlines()\n",
    "        doc_rank_dic = make_rank_dic(run_data)  # make dictionary of ranked docids for each queryid\n",
    "        rank_rel_dic = make_rank_rel_dic(query_rel_dic,doc_rank_dic) # make dic of list relevances of ranked docs for each queryid\n",
    "\n",
    "        # PREPARE SCORING DICTIONARIES\n",
    "        run_name = run[5:]\n",
    "        score_dic = {}\n",
    "        oracle_dic[run_name] = []\n",
    "\n",
    "        # LOOP OVER TOPICS\n",
    "        topics_list = make_topics_list(doc_rank_dic,1)  # sort topics by no docs\n",
    "        for query_id in topics_list:\n",
    "            #print(query_id)\n",
    "            score_dic[query_id] = []      \n",
    "\n",
    "            # EXTRACT COUNTS AND REL LISTS\n",
    "            n_docs = len(doc_rank_dic[query_id])  # total n. docs in topic\n",
    "            rel_list = rank_rel_dic[query_id]  # list binary rel of ranked docs \n",
    "\n",
    "\n",
    "            # ORACLE\n",
    "            rel_doc_idxs = np.where(np.array(rel_list) == 1)[0]\n",
    "            orcale_n_rel = math.ceil(len(rel_doc_idxs)*des_recall)\n",
    "            oracle_idx = rel_doc_idxs[orcale_n_rel-1]\n",
    "            oracle_eff = oracle_idx+1\n",
    "            oracle_dic[run_name].append(oracle_eff)\n",
    "            \n",
    "\n",
    "\n",
    "            # TARGET METHOD\n",
    "            random.seed(1)\n",
    "            target_size = get_target_size(des_recall, des_prob)\n",
    "            target_list, examined_list = make_target_set(rel_list, n_docs, target_size)  # get target sample and list all docs examined\n",
    "            tar_stop_n = get_stopping_target(target_list, n_docs, target_size)  # stopping point\n",
    "            all_examined_idxs = get_all_target_examined_idxs(examined_list, tar_stop_n)  # list of every doc examined during method\n",
    "            tar_recall = calc_recall(rel_list, tar_stop_n)\n",
    "            tar_effort = len(all_examined_idxs) # total effort (inc. sampling)\n",
    "            tar_accept = calc_accept(tar_recall, des_recall)\n",
    "            score_dic[query_id].append((tar_recall, tar_effort, tar_accept))\n",
    "            \n",
    "            \n",
    "            # KNEE METHOD\n",
    "            batches = get_batches(n_docs)\n",
    "            knee, knee_stop = get_knee_stopping_point_var_adjust(rel_list, batches, knee_rho, 150)[0:2]\n",
    "            knee_recall = calc_recall(rel_list, knee_stop)\n",
    "            knee_effort = knee_stop\n",
    "            knee_accept = calc_accept(knee_recall, des_recall)\n",
    "            score_dic[query_id].append((knee_recall, knee_effort, knee_accept))\n",
    "\n",
    "\n",
    "            \n",
    "            # INHOMOGENEOUS POISSON PROCESS EXPONENTIAL (IP-P)\n",
    "            # check topic meets initial relevance requirement\n",
    "            n_samp_docs = int(round(n_docs*sample_props[0]))\n",
    "            sample_rel_list = rel_list[0:n_samp_docs]  # chunck of rel list examined in sample\n",
    "\n",
    "            # if meet size requirement run algorithm; else return n_docs as stopping point\n",
    "            if (np.sum(sample_rel_list) >= min_rel_in_sample):\n",
    "\n",
    "                windows_end_point = 0\n",
    "                pred_stop_n = n_docs\n",
    "                i = 0\n",
    "\n",
    "                while (i < len(sample_props)) and (pred_stop_n > n_samp_docs):\n",
    "                    sample_prop = sample_props[i]\n",
    "\n",
    "                    n_samp_docs = int(round(n_docs*sample_props[i]))\n",
    "                    sample_rel_list = rel_list[0:n_samp_docs]  # chunck of rel list examined in sample\n",
    "\n",
    "                    # get points\n",
    "                    windows = make_windows(n_windows, sample_prop, n_docs)\n",
    "                    window_size = windows[0][1]\n",
    "\n",
    "                    x,y = get_points(windows, window_size, sample_rel_list)  # calculate points that will be used to fit curve\n",
    "\n",
    "                    good_curve_fit = 0\n",
    "                    # try to fit curve\n",
    "                    try: \n",
    "                        p0 = [0.1, 0.001]  # initialise curve parameters\n",
    "                        opt, pcov = curve_fit(model_func, x, y, p0)  # fit curve\n",
    "                        good_curve_fit = 1\n",
    "                    except: \n",
    "                        pass\n",
    "                    \n",
    "                    if(good_curve_fit == 1):\n",
    "                        a, k = opt\n",
    "                        y2 = model_func(x, a, k) # get y-values for fitted curve\n",
    "\n",
    "                        # check distance between \"curves\" at end sample\n",
    "                        n_rel_at_end_samp = np.sum(sample_rel_list)\n",
    "                        y3 =  model_func(np.array(range(1,len(sample_rel_list)+1)), a, k)\n",
    "                        est_by_curve_end_samp = np.sum(y3)\n",
    "                        est_by_curve_end_samp = int(round(est_by_curve_end_samp))\n",
    "\n",
    "\n",
    "                        if n_rel_at_end_samp >= des_recall*est_by_curve_end_samp:\n",
    "\n",
    "\n",
    "                            # using inhom Poisson process with fitted curve as rate fn, predict total number rel docs in topic \n",
    "                            mu = (a/-k)*(math.exp(-k*n_docs)-1)  # integral model_func\n",
    "                            pred_n_rel = predict_n_rel(des_prob, n_docs, mu) # predict max number rel docs (using poisson cdf)\n",
    "                            des_n_rel = des_recall*pred_n_rel\n",
    "                            if des_n_rel <= n_rel_at_end_samp:\n",
    "                                pred_stop_n = n_rel_at_end_samp             \n",
    "\n",
    "\n",
    "                    i += 1  # increase sample proportion size\n",
    "\n",
    "\n",
    "                # score result \n",
    "                inhom_recall = calc_recall(rel_list, n_samp_docs)\n",
    "                inhom_effort = n_samp_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "\n",
    "\n",
    "            else: # if not enough rel docs in min sample, stopping point is n_docs\n",
    "                inhom_recall = calc_recall(rel_list, n_docs)\n",
    "                inhom_effort = n_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "                \n",
    "                \n",
    "            # INHOMOGENEOUS POISSON PROCESS POWER (IP-P)\n",
    "            # check topic meets initial relevance requirement\n",
    "            n_samp_docs = int(round(n_docs*sample_props[0]))\n",
    "            sample_rel_list = rel_list[0:n_samp_docs]  # chunck of rel list examined in sample\n",
    "\n",
    "            # if meet size requirement run algorithm; else return n_docs as stopping point\n",
    "            if (np.sum(sample_rel_list) >= min_rel_in_sample):\n",
    "\n",
    "                windows_end_point = 0\n",
    "                pred_stop_n = n_docs\n",
    "                i = 0\n",
    "\n",
    "                while (i < len(sample_props)) and (pred_stop_n > n_samp_docs):\n",
    "                    sample_prop = sample_props[i]\n",
    "\n",
    "                    n_samp_docs = int(round(n_docs*sample_props[i]))\n",
    "                    sample_rel_list = rel_list[0:n_samp_docs]  # chunck of rel list examined in sample\n",
    "\n",
    "                    # get points\n",
    "                    windows = make_windows(n_windows, sample_prop, n_docs)\n",
    "                    window_size = windows[0][1]\n",
    "\n",
    "                    x,y = get_points_power(windows, window_size, sample_rel_list)  # calculate points that will be used to fit curve\n",
    "\n",
    "                    good_curve_fit = 0\n",
    "                    # try to fit curve\n",
    "                    try: \n",
    "                        p0 = [0.1, 0.001]  # initialise curve parameters\n",
    "                        opt, pcov = curve_fit(model_func_power, x, y, p0)  # fit curve\n",
    "                        good_curve_fit = 1\n",
    "                    except: \n",
    "                        pass\n",
    "                    \n",
    "                    if(good_curve_fit == 1):\n",
    "                        a, k = opt\n",
    "                        y2 = model_func_power(x, a, k) # get y-values for fitted curve\n",
    "\n",
    "                        # check distance between \"curves\" at end sample\n",
    "                        n_rel_at_end_samp = np.sum(sample_rel_list)\n",
    "                        y3 =  model_func_power(np.array(range(1,len(sample_rel_list)+1)), a, k)\n",
    "                        est_by_curve_end_samp = np.sum(y3)\n",
    "                        est_by_curve_end_samp = int(round(est_by_curve_end_samp))\n",
    "\n",
    "\n",
    "                        if n_rel_at_end_samp >= des_recall*est_by_curve_end_samp:\n",
    "\n",
    "\n",
    "                            # using inhom Poisson process with fitted curve as rate fn, predict total number rel docs in topic \n",
    "                            #mu = (a/-k)*(math.exp(-k*n_docs)-1)  # integral model_func\n",
    "                            mu = (a/(k+1))*(n_docs**(k+1)-1)  # update\n",
    "                            pred_n_rel = predict_n_rel(des_prob, n_docs, mu) # predict max number rel docs (using poisson cdf)\n",
    "                            des_n_rel = des_recall*pred_n_rel\n",
    "                            if des_n_rel <= n_rel_at_end_samp:\n",
    "                                pred_stop_n = n_rel_at_end_samp             \n",
    "\n",
    "\n",
    "                    i += 1  # increase sample proportion size\n",
    "\n",
    "\n",
    "                # score result \n",
    "                inhom_recall = calc_recall(rel_list, n_samp_docs)\n",
    "                inhom_effort = n_samp_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "\n",
    "\n",
    "            else: # if not enough rel docs in min sample, stopping point is n_docs\n",
    "                inhom_recall = calc_recall(rel_list, n_docs)\n",
    "                inhom_effort = n_docs\n",
    "                inhom_accept = calc_accept(inhom_recall, des_recall)\n",
    "                score_dic[query_id].append((inhom_recall, inhom_effort, inhom_accept))\n",
    "\n",
    "\n",
    "        \n",
    "            # COX PROCESS EXP CURVE (CX-E)\n",
    "            # check topic meets initial relevance requirement\n",
    "            n_samp_docs = int(round(n_docs*sample_props[0]))\n",
    "            sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "            \n",
    "            # if meet size requirement run algorithm; else return n_docs as stopping point\n",
    "            if (np.sum(sample_rel_list) >= min_rel_in_sample):\n",
    "\n",
    "                windows_end_point = 0\n",
    "                pred_stop_n = n_docs\n",
    "                i = 0\n",
    "\n",
    "                while (i < len(sample_props)) and (pred_stop_n > n_samp_docs):\n",
    "                    sample_prop = sample_props[i]\n",
    "\n",
    "                    n_samp_docs = int(round(n_docs*sample_props[i]))\n",
    "                    sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "\n",
    "                    # get points\n",
    "                    windows = make_windows(n_windows, sample_prop, n_docs)\n",
    "                    window_size = windows[0][1]\n",
    "\n",
    "                    x,y = get_points(windows, window_size, sample_rel_list)  # calculate points that will be used to fit curve\n",
    "\n",
    "                    good_curve_fit = 0\n",
    "                    # try to fit curve\n",
    "                    try: \n",
    "                        p0 = [0.1, 0.001]  # initialise curve parameters\n",
    "                        opt, pcov = curve_fit(model_func, x, y, p0)  # fit curve\n",
    "                        good_curve_fit = 1\n",
    "                    except: \n",
    "                        pass\n",
    "                    \n",
    "                    if(good_curve_fit == 1):\n",
    "                        # marks@18/12/2020\n",
    "                        # Standard deviation errors on curve fit parameters\n",
    "                        perr = np.sqrt(np.diag(pcov))\n",
    "                        #print(\"perr:\", perr)\n",
    "                        \n",
    "                        a, k = opt \n",
    "                        y2 = model_func(x, a, k) # get y-values for fitted curve \n",
    "                        \n",
    "                        # check distance between \"curves\" at end sample\n",
    "                        n_rel_at_end_samp = np.sum(sample_rel_list)\n",
    "                        y3 =  model_func(np.array(range(1,len(sample_rel_list)+1)), a, k) \n",
    "                        est_by_curve_end_samp = np.sum(y3)\n",
    "                        est_by_curve_end_samp = int(round(est_by_curve_end_samp))\n",
    "\n",
    "\n",
    "                        if n_rel_at_end_samp >= des_recall*est_by_curve_end_samp:\n",
    "\n",
    "                            # marks@16/12/2020\n",
    "                            # Cox process with fitted curve.\n",
    "                            # Sample points from normal distribution; generate probability + predicted\n",
    "                            # value from Poisson Process\n",
    "                            norm_samples = np.linspace(norm.ppf(0.01), norm.ppf(0.99), 100)\n",
    "                            #print(\"norm_samples:\",norm_samples)\n",
    "                            vals = []\n",
    "                            for sample in norm_samples:\n",
    "                                a_val = a + sample*perr[0]\n",
    "                                k_val = k - sample*perr[1]   # Note need to *subtract* from one\n",
    "                                mu = (a_val/-k_val)*(math.exp(-k_val*n_docs)-1)  # integral of model_func\n",
    "                                vals.append(norm.pdf(sample) * predict_n_rel(des_prob, n_docs, mu))   # predict max number rel docs (using poisson cdf)\n",
    "                            # Integrate over samples to produce final prediction\n",
    "                            cox_pred_n_rel = simps(vals, norm_samples)\n",
    "\n",
    "                            des_n_rel = des_recall*cox_pred_n_rel  # alison\n",
    "                            if des_n_rel <= n_rel_at_end_samp:\n",
    "                                pred_stop_n = n_rel_at_end_samp             \n",
    "\n",
    "\n",
    "                    i += 1  # increase sample proportion size\n",
    "\n",
    "\n",
    "                # score result \n",
    "                cox_recall = calc_recall(rel_list, n_samp_docs)\n",
    "                cox_effort = n_samp_docs\n",
    "                cox_accept = calc_accept(cox_recall, des_recall) # alison\n",
    "                score_dic[query_id].append((cox_recall, cox_effort, cox_accept))\n",
    "\n",
    "            \n",
    "            else: # if not enough rel docs in min sample, stopping point is n_docs\n",
    "                cox_recall = calc_recall(rel_list, n_docs)\n",
    "                cox_effort = n_docs\n",
    "                cox_accept = calc_accept(cox_recall, des_recall) # alison\n",
    "                score_dic[query_id].append((cox_recall, cox_effort, cox_accept))\n",
    "                \n",
    "        \n",
    "\n",
    "            # COX PROCESS POWER CURVE (CX-P)\n",
    "            # check topic meets initial relevance requirement\n",
    "            n_samp_docs = int(round(n_docs*sample_props[0]))\n",
    "            sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "            \n",
    "            # if meet size requirement run algorithm; else return n_docs as stopping point\n",
    "            if (np.sum(sample_rel_list) >= min_rel_in_sample):\n",
    "\n",
    "                windows_end_point = 0\n",
    "                pred_stop_n = n_docs\n",
    "                i = 0\n",
    "\n",
    "                while (i < len(sample_props)) and (pred_stop_n > n_samp_docs):\n",
    "                    sample_prop = sample_props[i]\n",
    "\n",
    "                    n_samp_docs = int(round(n_docs*sample_props[i]))\n",
    "                    sample_rel_list = rel_list[0:n_samp_docs]  # chuck of rel list examined in sample\n",
    "\n",
    "                    # get points\n",
    "                    windows = make_windows(n_windows, sample_prop, n_docs)\n",
    "                    window_size = windows[0][1]\n",
    "\n",
    "                    x,y = get_points_power(windows, window_size, sample_rel_list)  # update\n",
    "\n",
    "                    good_curve_fit = 0\n",
    "                    # try to fit curve\n",
    "                    try: \n",
    "                        p0 = [0.1, 0.001]  # initialise curve parameters\n",
    "                        opt, pcov = curve_fit(model_func_power, x, y, p0)  # update\n",
    "                        good_curve_fit = 1\n",
    "                    except: \n",
    "                        pass\n",
    "                    \n",
    "                    if(good_curve_fit == 1):\n",
    "                        # marks@18/12/2020\n",
    "                        # Standard deviation errors on curve fit parameters\n",
    "                        perr = np.sqrt(np.diag(pcov))\n",
    "                        #print(\"perr:\", perr)\n",
    "                        \n",
    "                        a, k = opt \n",
    "                        y2 = model_func_power(x, a, k) # updated\n",
    "                        \n",
    "                        # check distance between \"curves\" at end sample\n",
    "                        n_rel_at_end_samp = np.sum(sample_rel_list)\n",
    "                        y3 =  model_func_power(np.array(range(1,len(sample_rel_list)+1)), a, k) \n",
    "                        est_by_curve_end_samp = np.sum(y3)\n",
    "                        est_by_curve_end_samp = int(round(est_by_curve_end_samp))\n",
    "\n",
    "\n",
    "                        if n_rel_at_end_samp >= des_recall*est_by_curve_end_samp:\n",
    "\n",
    "                            # marks@16/12/2020\n",
    "                            # Cox process with fitted curve.\n",
    "                            # Sample points from normal distribution; generate probability + predicted\n",
    "                            # value from Poisson Process\n",
    "                            norm_samples = np.linspace(norm.ppf(0.01), norm.ppf(0.99), 100)\n",
    "                            vals = []\n",
    "                            for sample in norm_samples:\n",
    "                                a_val = a + sample*perr[0]\n",
    "                                k_val = k - sample*perr[1]   # Note need to *subtract* from one\n",
    "                                mu = (a_val/(k_val+1))*(n_docs**(k_val+1)-1)  # update\n",
    "                                vals.append(norm.pdf(sample) * predict_n_rel(des_prob, n_docs, mu))   # predict max number rel docs (using poisson cdf)\n",
    "                            # Integrate over samples to produce final prediction\n",
    "                            cox_pred_n_rel = simps(vals, norm_samples)\n",
    "              \n",
    "\n",
    "                           \n",
    "                            des_n_rel = des_recall*cox_pred_n_rel  # alison\n",
    "                            if des_n_rel <= n_rel_at_end_samp:\n",
    "                                pred_stop_n = n_rel_at_end_samp             \n",
    "\n",
    "\n",
    "                    i += 1  # increase sample proportion size\n",
    "\n",
    "\n",
    "                # score result \n",
    "                cox_power_recall = calc_recall(rel_list, n_samp_docs)\n",
    "                cox_power_effort = n_samp_docs\n",
    "                cox_power_accept = calc_accept(cox_power_recall, des_recall) # alison\n",
    "                score_dic[query_id].append((cox_power_recall, cox_power_effort, cox_power_accept))\n",
    "\n",
    "            \n",
    "            else: # if not enough rel docs in min sample, stopping point is n_docs\n",
    "                cox_power_recall = calc_recall(rel_list, n_docs)\n",
    "                cox_power_effort = n_docs\n",
    "                cox_power_accept = calc_accept(cox_power_recall, des_recall) # alison\n",
    "                score_dic[query_id].append((cox_power_recall, cox_power_effort, cox_power_accept))\n",
    "        \n",
    "        \n",
    "           \n",
    "        \n",
    "        # SCORE RESULTS\n",
    "        tar_recall_vec =        [val[0][0] for val in score_dic.values()]\n",
    "        km150_recall_vec =      [val[1][0] for val in score_dic.values()]\n",
    "        inhom_recall_vec =      [val[2][0] for val in score_dic.values()]\n",
    "        inhom_power_recall_vec= [val[3][0] for val in score_dic.values()]\n",
    "        cox_recall_vec =        [val[4][0] for val in score_dic.values()]\n",
    "        cox_power_recall_vec =  [val[5][0] for val in score_dic.values()]\n",
    "        \n",
    "        tar_eff_vec =        [val[0][1] for val in score_dic.values()]\n",
    "        km150_eff_vec =      [val[1][1] for val in score_dic.values()]\n",
    "        inhom_eff_vec =      [val[2][1] for val in score_dic.values()]\n",
    "        inhom_power_eff_vec= [val[3][1] for val in score_dic.values()]\n",
    "        cox_eff_vec =        [val[4][1] for val in score_dic.values()]\n",
    "        cox_power_eff_vec =  [val[5][1] for val in score_dic.values()]\n",
    "    \n",
    "        topic_size_vec = [len(doc_rank_dic[query_id]) for query_id in topics_list]\n",
    "\n",
    "        \n",
    "        # ADD SCORES TO DICTIONARIES\n",
    "        run_score_dic[run_name] = {}\n",
    "    \n",
    "        run_score_dic[run_name]['TM Rec'] = np.mean(tar_recall_vec)\n",
    "        run_score_dic[run_name]['KM Rec'] = np.mean(km150_recall_vec)\n",
    "        run_score_dic[run_name]['IP-E Rec'] = np.mean(inhom_recall_vec)\n",
    "        run_score_dic[run_name]['CX-E Rec'] = np.mean(cox_recall_vec)\n",
    "        run_score_dic[run_name]['IP-P Rec'] = np.mean(inhom_power_recall_vec)\n",
    "        run_score_dic[run_name]['CX-P Rec'] = np.mean(cox_power_recall_vec)\n",
    "     \n",
    "        run_score_dic[run_name]['TM Eff'] =  np.sum(tar_eff_vec)\n",
    "        run_score_dic[run_name]['KM Eff'] = np.sum(km150_eff_vec)\n",
    "        run_score_dic[run_name]['IP-E Eff'] = np.sum(inhom_eff_vec)\n",
    "        run_score_dic[run_name]['CX-E Eff'] = np.sum(cox_eff_vec)\n",
    "        run_score_dic[run_name]['IP-P Eff'] = np.sum(inhom_power_eff_vec)\n",
    "        run_score_dic[run_name]['CX-P Eff'] = np.sum(cox_power_eff_vec)\n",
    "        run_score_dic[run_name]['OR Eff'] = np.sum(oracle_dic[run_name])\n",
    " \n",
    "\n",
    "    # MAKE DATAFRAME OF ALL RESULTS\n",
    "    df = pd.DataFrame.from_dict(run_score_dic, orient='index')\n",
    "    \n",
    "    return  (df, df.mean().round(2).to_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# RUN EXPERIMENTS\n",
    "\n",
    "def pes(eff):  # fn to calculate % of effort saved\n",
    "    saving = 117562-eff\n",
    "    return round(100*saving/117562,1)\n",
    "\n",
    "\n",
    "\n",
    "def run_experiments(des_prob):  http://localhost:8888/notebooks/Documents/work_home_backups/cox_processes/SIGIR21_code/run_stopping_point_experiments.ipynb#\n",
    "    print(des_prob)\n",
    "    \n",
    "    # get results\n",
    "    results_dict_verbose = {}\n",
    "    results_dict = {}\n",
    "    for r in des_recalls:\n",
    "        print(\"recall\", r)\n",
    "        results_dict_verbose[r], results_dict[r] = run_sp_algorithms(r, des_prob)\n",
    "        \n",
    "    # make results into dataframe\n",
    "    df = pd.DataFrame.from_dict(results_dict, orient='index')    \n",
    "    \n",
    "    # add percentage effort saved cols\n",
    "    df['TM PES'] = [pes(eff) for eff in df['TM Eff'].tolist()]\n",
    "    df['KM PES'] = [pes(eff) for eff in df['KM Eff'].tolist()]\n",
    "    df['IP-E PES'] = [pes(eff) for eff in df['IP-E Eff'].tolist()]\n",
    "    df['CX-E PES'] = [pes(eff) for eff in df['CX-E Eff'].tolist()]\n",
    "    df['IP-P PES'] = [pes(eff) for eff in df['IP-P Eff'].tolist()]\n",
    "    df['CX-P PES'] = [pes(eff) for eff in df['CX-P Eff'].tolist()]\n",
    "    df['OR PES'] = [pes(eff) for eff in df['OR Eff'].tolist()]\n",
    "\n",
    "    # make effort cols integers\n",
    "    df['KM Eff'] = df['KM Eff'].round(0).astype(int)\n",
    "    df['TM Eff'] = df['TM Eff'].round(0).astype(int)\n",
    "    df['IP-E Eff'] = df['IP-E Eff'].round(0).astype(int)\n",
    "    df['CX-E Eff'] = df['CX-E Eff'].round(0).astype(int)\n",
    "    df['IP-P Eff'] = df['IP-P Eff'].round(0).astype(int)\n",
    "    df['CX-P Eff'] = df['CX-P Eff'].round(0).astype(int)\n",
    "    df['OR Eff'] = df['OR Eff'].round(0).astype(int)\n",
    "\n",
    "    df = df.sort_index()\n",
    "    \n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.95\n",
      "recall 0.95\n",
      "recall 0.9\n",
      "recall 0.8\n",
      "recall 0.7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TM Rec</th>\n",
       "      <th>KM Rec</th>\n",
       "      <th>IP-E Rec</th>\n",
       "      <th>CX-E Rec</th>\n",
       "      <th>IP-P Rec</th>\n",
       "      <th>CX-P Rec</th>\n",
       "      <th>TM Eff</th>\n",
       "      <th>KM Eff</th>\n",
       "      <th>IP-E Eff</th>\n",
       "      <th>CX-E Eff</th>\n",
       "      <th>IP-P Eff</th>\n",
       "      <th>CX-P Eff</th>\n",
       "      <th>OR Eff</th>\n",
       "      <th>TM PES</th>\n",
       "      <th>KM PES</th>\n",
       "      <th>IP-E PES</th>\n",
       "      <th>CX-E PES</th>\n",
       "      <th>IP-P PES</th>\n",
       "      <th>CX-P PES</th>\n",
       "      <th>OR PES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.70</th>\n",
       "      <td>0.92</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>55005</td>\n",
       "      <td>86243</td>\n",
       "      <td>54754</td>\n",
       "      <td>48290</td>\n",
       "      <td>53585</td>\n",
       "      <td>53263</td>\n",
       "      <td>7419</td>\n",
       "      <td>53.2</td>\n",
       "      <td>26.6</td>\n",
       "      <td>53.4</td>\n",
       "      <td>58.9</td>\n",
       "      <td>54.4</td>\n",
       "      <td>54.7</td>\n",
       "      <td>93.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.80</th>\n",
       "      <td>0.95</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>62867</td>\n",
       "      <td>86243</td>\n",
       "      <td>73728</td>\n",
       "      <td>80040</td>\n",
       "      <td>56840</td>\n",
       "      <td>57455</td>\n",
       "      <td>9675</td>\n",
       "      <td>46.5</td>\n",
       "      <td>26.6</td>\n",
       "      <td>37.3</td>\n",
       "      <td>31.9</td>\n",
       "      <td>51.7</td>\n",
       "      <td>51.1</td>\n",
       "      <td>91.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.90</th>\n",
       "      <td>0.98</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>83954</td>\n",
       "      <td>86243</td>\n",
       "      <td>103509</td>\n",
       "      <td>99314</td>\n",
       "      <td>61361</td>\n",
       "      <td>61711</td>\n",
       "      <td>16706</td>\n",
       "      <td>28.6</td>\n",
       "      <td>26.6</td>\n",
       "      <td>12.0</td>\n",
       "      <td>15.5</td>\n",
       "      <td>47.8</td>\n",
       "      <td>47.5</td>\n",
       "      <td>85.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.95</th>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>102105</td>\n",
       "      <td>86243</td>\n",
       "      <td>116521</td>\n",
       "      <td>117558</td>\n",
       "      <td>63605</td>\n",
       "      <td>63314</td>\n",
       "      <td>22637</td>\n",
       "      <td>13.1</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.9</td>\n",
       "      <td>46.1</td>\n",
       "      <td>80.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      TM Rec  KM Rec  IP-E Rec  CX-E Rec  IP-P Rec  CX-P Rec  TM Eff  KM Eff  \\\n",
       "0.70    0.92     1.0      0.97      0.98      0.99      0.99   55005   86243   \n",
       "0.80    0.95     1.0      0.99      0.99      1.00      1.00   62867   86243   \n",
       "0.90    0.98     1.0      1.00      1.00      1.00      1.00   83954   86243   \n",
       "0.95    1.00     1.0      1.00      1.00      1.00      1.00  102105   86243   \n",
       "\n",
       "      IP-E Eff  CX-E Eff  IP-P Eff  CX-P Eff  OR Eff  TM PES  KM PES  \\\n",
       "0.70     54754     48290     53585     53263    7419    53.2    26.6   \n",
       "0.80     73728     80040     56840     57455    9675    46.5    26.6   \n",
       "0.90    103509     99314     61361     61711   16706    28.6    26.6   \n",
       "0.95    116521    117558     63605     63314   22637    13.1    26.6   \n",
       "\n",
       "      IP-E PES  CX-E PES  IP-P PES  CX-P PES  OR PES  \n",
       "0.70      53.4      58.9      54.4      54.7    93.7  \n",
       "0.80      37.3      31.9      51.7      51.1    91.8  \n",
       "0.90      12.0      15.5      47.8      47.5    85.8  \n",
       "0.95       0.9       0.0      45.9      46.1    80.7  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_95 = run_experiments(0.95)\n",
    "df_95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrrrrrrrrrrrrrrr}\n",
      "\\toprule\n",
      "{} &  TM Rec &  KM Rec &  IP-E Rec &  CX-E Rec &  IP-P Rec &  CX-P Rec &  TM Eff &  KM Eff &  IP-E Eff &  CX-E Eff &  IP-P Eff &  CX-P Eff &  OR Eff &  TM PES &  KM PES &  IP-E PES &  CX-E PES &  IP-P PES &  CX-P PES &  OR PES \\\\\n",
      "\\midrule\n",
      "0.70 &    0.92 &     1.0 &      0.97 &      0.98 &      0.99 &      0.99 &   55005 &   86243 &     54754 &     48290 &     53585 &     53263 &    7419 &    53.2 &    26.6 &      53.4 &      58.9 &      54.4 &      54.7 &    93.7 \\\\\n",
      "0.80 &    0.95 &     1.0 &      0.99 &      0.99 &      1.00 &      1.00 &   62867 &   86243 &     73728 &     80040 &     56840 &     57455 &    9675 &    46.5 &    26.6 &      37.3 &      31.9 &      51.7 &      51.1 &    91.8 \\\\\n",
      "0.90 &    0.98 &     1.0 &      1.00 &      1.00 &      1.00 &      1.00 &   83954 &   86243 &    103509 &     99314 &     61361 &     61711 &   16706 &    28.6 &    26.6 &      12.0 &      15.5 &      47.8 &      47.5 &    85.8 \\\\\n",
      "0.95 &    1.00 &     1.0 &      1.00 &      1.00 &      1.00 &      1.00 &  102105 &   86243 &    116521 &    117558 &     63605 &     63314 &   22637 &    13.1 &    26.6 &       0.9 &       0.0 &      45.9 &      46.1 &    80.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df_95.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8\n",
      "recall 0.95\n",
      "recall 0.9\n",
      "recall 0.8\n",
      "recall 0.7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TM Rec</th>\n",
       "      <th>KM Rec</th>\n",
       "      <th>IP-E Rec</th>\n",
       "      <th>CX-E Rec</th>\n",
       "      <th>IP-P Rec</th>\n",
       "      <th>CX-P Rec</th>\n",
       "      <th>TM Eff</th>\n",
       "      <th>KM Eff</th>\n",
       "      <th>IP-E Eff</th>\n",
       "      <th>CX-E Eff</th>\n",
       "      <th>IP-P Eff</th>\n",
       "      <th>CX-P Eff</th>\n",
       "      <th>OR Eff</th>\n",
       "      <th>TM PES</th>\n",
       "      <th>KM PES</th>\n",
       "      <th>IP-E PES</th>\n",
       "      <th>CX-E PES</th>\n",
       "      <th>IP-P PES</th>\n",
       "      <th>CX-P PES</th>\n",
       "      <th>OR PES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.70</th>\n",
       "      <td>0.85</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>43396</td>\n",
       "      <td>86243</td>\n",
       "      <td>46783</td>\n",
       "      <td>47622</td>\n",
       "      <td>51479</td>\n",
       "      <td>51236</td>\n",
       "      <td>7419</td>\n",
       "      <td>63.1</td>\n",
       "      <td>26.6</td>\n",
       "      <td>60.2</td>\n",
       "      <td>59.5</td>\n",
       "      <td>56.2</td>\n",
       "      <td>56.4</td>\n",
       "      <td>93.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.80</th>\n",
       "      <td>0.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>52417</td>\n",
       "      <td>86243</td>\n",
       "      <td>56051</td>\n",
       "      <td>56050</td>\n",
       "      <td>54948</td>\n",
       "      <td>54758</td>\n",
       "      <td>9675</td>\n",
       "      <td>55.4</td>\n",
       "      <td>26.6</td>\n",
       "      <td>52.3</td>\n",
       "      <td>52.3</td>\n",
       "      <td>53.3</td>\n",
       "      <td>53.4</td>\n",
       "      <td>91.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.90</th>\n",
       "      <td>0.96</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>66476</td>\n",
       "      <td>86243</td>\n",
       "      <td>79563</td>\n",
       "      <td>83510</td>\n",
       "      <td>58619</td>\n",
       "      <td>59113</td>\n",
       "      <td>16706</td>\n",
       "      <td>43.5</td>\n",
       "      <td>26.6</td>\n",
       "      <td>32.3</td>\n",
       "      <td>29.0</td>\n",
       "      <td>50.1</td>\n",
       "      <td>49.7</td>\n",
       "      <td>85.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.95</th>\n",
       "      <td>0.99</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.99</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>86381</td>\n",
       "      <td>86243</td>\n",
       "      <td>95235</td>\n",
       "      <td>92819</td>\n",
       "      <td>59703</td>\n",
       "      <td>60043</td>\n",
       "      <td>22637</td>\n",
       "      <td>26.5</td>\n",
       "      <td>26.6</td>\n",
       "      <td>19.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>49.2</td>\n",
       "      <td>48.9</td>\n",
       "      <td>80.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      TM Rec  KM Rec  IP-E Rec  CX-E Rec  IP-P Rec  CX-P Rec  TM Eff  KM Eff  \\\n",
       "0.70    0.85     1.0      0.97      0.97      0.99      0.99   43396   86243   \n",
       "0.80    0.90     1.0      0.98      0.98      0.99      0.99   52417   86243   \n",
       "0.90    0.96     1.0      0.99      0.99      1.00      1.00   66476   86243   \n",
       "0.95    0.99     1.0      1.00      0.99      1.00      1.00   86381   86243   \n",
       "\n",
       "      IP-E Eff  CX-E Eff  IP-P Eff  CX-P Eff  OR Eff  TM PES  KM PES  \\\n",
       "0.70     46783     47622     51479     51236    7419    63.1    26.6   \n",
       "0.80     56051     56050     54948     54758    9675    55.4    26.6   \n",
       "0.90     79563     83510     58619     59113   16706    43.5    26.6   \n",
       "0.95     95235     92819     59703     60043   22637    26.5    26.6   \n",
       "\n",
       "      IP-E PES  CX-E PES  IP-P PES  CX-P PES  OR PES  \n",
       "0.70      60.2      59.5      56.2      56.4    93.7  \n",
       "0.80      52.3      52.3      53.3      53.4    91.8  \n",
       "0.90      32.3      29.0      50.1      49.7    85.8  \n",
       "0.95      19.0      21.0      49.2      48.9    80.7  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_80 = run_experiments(0.80)\n",
    "df_80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrrrrrrrrrrrrrrr}\n",
      "\\toprule\n",
      "{} &  TM Rec &  KM Rec &  IP-E Rec &  CX-E Rec &  IP-P Rec &  CX-P Rec &  TM Eff &  KM Eff &  IP-E Eff &  CX-E Eff &  IP-P Eff &  CX-P Eff &  OR Eff &  TM PES &  KM PES &  IP-E PES &  CX-E PES &  IP-P PES &  CX-P PES &  OR PES \\\\\n",
      "\\midrule\n",
      "0.70 &    0.85 &     1.0 &      0.97 &      0.97 &      0.99 &      0.99 &   43396 &   86243 &     46783 &     47622 &     51479 &     51236 &    7419 &    63.1 &    26.6 &      60.2 &      59.5 &      56.2 &      56.4 &    93.7 \\\\\n",
      "0.80 &    0.90 &     1.0 &      0.98 &      0.98 &      0.99 &      0.99 &   52417 &   86243 &     56051 &     56050 &     54948 &     54758 &    9675 &    55.4 &    26.6 &      52.3 &      52.3 &      53.3 &      53.4 &    91.8 \\\\\n",
      "0.90 &    0.96 &     1.0 &      0.99 &      0.99 &      1.00 &      1.00 &   66476 &   86243 &     79563 &     83510 &     58619 &     59113 &   16706 &    43.5 &    26.6 &      32.3 &      29.0 &      50.1 &      49.7 &    85.8 \\\\\n",
      "0.95 &    0.99 &     1.0 &      1.00 &      0.99 &      1.00 &      1.00 &   86381 &   86243 &     95235 &     92819 &     59703 &     60043 &   22637 &    26.5 &    26.6 &      19.0 &      21.0 &      49.2 &      48.9 &    80.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df_80.to_latex())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}