Figure-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "The jupyternotify extension is already loaded. To reload it, use:\n",
      "  %reload_ext jupyternotify\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%load_ext jupyternotify\n",
    "import numpy as np \n",
    "from sets import Set\n",
    "from conf_ope_rl import * \n",
    "from __future__ import division \n",
    "\n",
    "pu1 = 1/6\n",
    "pu2 = 1/2\n",
    "pis1u1 = 1.0/4\n",
    "pis2u1 = 1.0/6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stationary distribution \n",
    "$p^{\\infty}_b(s,u)  $ \n",
    "\n",
    "What is $p^{\\infty}_b(s,a,s')$? \n",
    "Note $\\mathbb{E}[\\pi_b(a \\mid s,u)] = \\frac{1}{2} \\pi_b(a=1\\mid s,u=1) +  \\frac{1}{2} \\pi_b(a=1\\mid s,u=2) $\n",
    "\n",
    "The true marginalized transition matrix is $p_b^\\infty(s,a,s') = \\mathbb{E}_{u \\sim p_b}[ P(s' \\mid a,s,u) \\pi_b(a \\mid s,u) p_b^\\infty(s,u)] $\n",
    "\n",
    "and \n",
    "$p_b^\\infty(s',a\\mid s) = \\mathbb{E}_{u \\sim p_b}[ P(s' \\mid a,s,u) \\pi_b(a \\mid s,u) p_b^\\infty(u \\mid s)] $\n",
    "\n",
    "The biased transitions are $\\tilde p(s'\\mid s,a) = p_b^\\infty(s',a\\mid s) /  \\mathbb{E}_{u\\sim p_b^\\infty} [\\pi_b(a \\mid s,u)\\mid s] $\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.75 0.75 0.75 0.75]]\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division\n",
    "nS = 4\n",
    "nA = 2\n",
    "P = np.zeros([nS,nA, nS])\n",
    "state_dist = np.ones(nS) * 1.0/nS\n",
    "n = 20000\n",
    "nU = 2\n",
    "nSmarg = nS / nU\n",
    "gamma = 0.99\n",
    "p0 = np.asarray([0.25, 0.25,0.25,0.25])\n",
    "p_e_su = np.asarray([ p0 ,1-p0 ])\n",
    "print p_e_su\n",
    "\n",
    "# very extreme\n",
    "\n",
    "\n",
    "def get_P(pu1, pu2, pis1u1, pis2u1): \n",
    "    Pi = np.array(# s=a,u1; s=a,u2; s=b,u1; s=b,u2\n",
    "        [[pis1u1, 1-pis1u1, pis2u1, 1-pis2u1],\n",
    "            [1-pis1u1, pis1u1, 1-pis2u1, pis2u1]]) \n",
    "    P = np.zeros([nS,nA, nS])\n",
    "    P[:,0,:] = np.asarray([np.asarray([pu1, \n",
    "         pu1, (1/2 - pu1), (1/2 - pu1)]), np.asarray([(1/2 - pu2), (1/2 - pu2), pu2, pu2]), \n",
    "    np.asarray([pu1, \n",
    "         pu1, (1/2 - pu1), (1/2 - pu1)]), np.asarray([(1/2 - pu2), (1/2 - pu2), pu2, pu2])]);\n",
    "    P[:,1,:] = np.asarray([np.asarray([(1/2 - pu1), (1/2 - pu1), pu1, pu1]), \n",
    "    np.asarray([pu2, \n",
    "         pu2, (1/2 - pu2), (1/2 - pu2)]), np.asarray([(1/2 - pu1), (1/2 - pu1), pu1, pu1]), \n",
    "    np.asarray([pu2, pu2, (1/2 - pu2), (1/2 - pu2)])]);\n",
    "    return [P,Pi]\n",
    "\n",
    "[P,Pi] = get_P(pu1, pu2, pis1u1, pis2u1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.25 0.25 0.25 0.25]\n",
      "[0.20318984 0.20183991 0.29733513 0.29763512] [0.40502975 0.59497025]\n",
      "('stationary distn on s,u: ', array([0.20318984, 0.20183991, 0.29733513, 0.29763512]))\n",
      "[0.25 0.25 0.25 0.25]\n",
      "\n",
      "--------------------------------------------\n",
      "Warning: your license will expire in 5 days\n",
      "--------------------------------------------\n",
      "\n",
      "Using license file /Users/az/gurobi.lic\n",
      "Academic license - for non-commercial use only\n",
      "LP solution norm [0.37013753 0.37580984 0.12711234 0.12694029]\n",
      "[0.16331629 0.16581909 0.05608595 0.05601004]\n",
      "oracle solution norm [[0.37163941 0.37001928 0.12854891 0.12979241]]\n",
      "[[1.63681102 1.6296755  0.56616782 0.57164455]]\n",
      "marginalized stationary distribution [0.74165869 0.25834131]\n"
     ]
    }
   ],
   "source": [
    "tight = False\n",
    "# def get_full_agg_w_densratio( gamma, nS, nA, nU, P, Pi, p_e_su, state, n ): \n",
    "nSmarg = int(nS / nU)\n",
    "[ stateChangeHist, stateHist, a_s, s_a_sprime, p_infty_b_su, distr_hist ] =  simulate_rollouts( \n",
    "    nS, nA, P, Pi, state_dist, n )\n",
    "p_infty_b_s = (reshape_byxrow(p_infty_b_su.T, nU).T).flatten()\n",
    "p_infty_b_su = p_infty_b_su.flatten()\n",
    "print p_infty_b_su, p_infty_b_s\n",
    "print('stationary distn on s,u: ', p_infty_b_su)\n",
    "\n",
    "# process history \n",
    "[ p_a1_su, joint_s_a_sprime, s_a_giv_sprime ] = get_auxiliary_info_from_traj(stateChangeHist, \n",
    "                                        stateHist, a_s, s_a_sprime, p_infty_b_su, distr_hist, nA,nS)\n",
    "# agg history and process\n",
    "[aggStateHist, p_a1_s, p_e_s, agg_s_a_sprime, joint_s_a_sprime_agg, s_a_giv_sprime_agg] = agg_history(\n",
    "            stateHist, s_a_sprime, p_infty_b_s, a_s, p_e_su, nA, nS, nSmarg, nU)\n",
    "# Simulate from evaluation policy: \n",
    "[ stateChangeHist_e, stateHist_e, a_s_e, s_a_sprime_e, p_infty_e, distr_hist_e ] =  simulate_rollouts( \n",
    "    nS, nA, P, p_e_su, state_dist, n )\n",
    "\n",
    "\n",
    "# Solve on the full information space \n",
    "w_su = get_w_lp(gamma, s_a_giv_sprime, p_infty_b_su, p_e_su, p_a1_su, nA, nS, tight = tight)\n",
    "print 'LP solution norm', w_su / sum(w_su)\n",
    "print w_su\n",
    "print 'oracle solution norm', (p_infty_e / p_infty_b_su) / np.sum((p_infty_e / p_infty_b_su))\n",
    "print (p_infty_e / p_infty_b_su)\n",
    "\n",
    "w_su_oracle_norm = (p_infty_e / p_infty_b_su)/ np.sum((p_infty_e / p_infty_b_su))\n",
    "w_s_oracle_norm = reshape_byxrow(w_su_oracle_norm.T,nU).T.flatten()\n",
    "print 'marginalized stationary distribution', w_s_oracle_norm\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.25 0.25 0.25 0.25]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[[0.20077992 0.20417958 0.29567043 0.29937006]]\n",
      "[[0.33166683 0.33446655 0.16578342 0.16808319]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "[ stateChangeHist, stateHist, a_s, s_a_sprime, p_infty_b_su_long, distr_hist ] =  simulate_rollouts( \n",
    "    nS, nA, P, Pi, state_dist, 10000 )\n",
    "p_infty_b_s = (reshape_byxrow(p_infty_b_su_long.T, nU).T).flatten()\n",
    "\n",
    "\n",
    "[ p_a1_su, joint_s_a_sprime, s_a_giv_sprime ] = get_auxiliary_info_from_traj(stateChangeHist, \n",
    "                                        stateHist, a_s, s_a_sprime, p_infty_b_su_long, distr_hist, nA,nS)\n",
    "# agg history and process\n",
    "[aggStateHist, p_a1_s, p_e_s, agg_s_a_sprime, joint_s_a_sprime_agg, s_a_giv_sprime_agg] = agg_history(\n",
    "            stateHist, s_a_sprime, p_infty_b_s, a_s, p_e_su, nA, nS, nSmarg, nU)\n",
    "\n",
    "[ stateChangeHist_e, stateHist_e, a_s, s_a_sprime_e, p_infty_e_long, distr_hist_e ] =  simulate_rollouts( \n",
    "    nS, nA, P, p_e_su, state_dist, 10000 )\n",
    "\n",
    "print p_infty_b_su_long\n",
    "print p_infty_e_long"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get biased transitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.0185, 0.1805],\n",
       "        [0.1574, 0.0486]],\n",
       "\n",
       "       [[0.0181, 0.2839],\n",
       "        [0.211 , 0.082 ]]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joint_s_a_sprime_agg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get marginalized transitions and solve for model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.09295116, 0.9069019 ],\n",
       "        [0.76433643, 0.23600223]],\n",
       "\n",
       "       [[0.05993977, 0.94016023],\n",
       "        [0.71996281, 0.27979597]]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_transitions(joint_s_a_sprime_agg, p_a1_s, p_b_infty_s,nS,nA): \n",
    "    P = np.zeros([nS,nA, nS])\n",
    "    for s in range(P.shape[0]): \n",
    "        for a in range(nA): \n",
    "            P[s,a,:] = joint_s_a_sprime_agg[s,a,:] / (p_a1_s[a,s]*p_b_infty_s[s])\n",
    "    return P\n",
    "P_marg = get_transitions(joint_s_a_sprime_agg, p_a1_s, p_infty_b_s,nSmarg,nA)\n",
    "P_marg_mdptoolbox = np.zeros([ nA, nSmarg, nSmarg ])\n",
    "for a_ind in range(nA): \n",
    "    for i_s in range(nSmarg): \n",
    "        for i_sprime in range(nSmarg): \n",
    "            P_marg_mdptoolbox[ a_ind, i_s, i_sprime] = P_marg[a_ind, i_s, i_sprime]\n",
    "P_marg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]] [[0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.6540346 0.3459654]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.5005749712514376\n",
      "1.36510737327\n",
      "[0.9707324812563278, 1.055329144538254]\n",
      "1.3301001029\n",
      "[1.0242575861376138, 0.9541419448190372]\n",
      "1.38362307549\n",
      "[0.9424225075664808, 1.1088480861293781]\n",
      "1.31631712557\n",
      "[1.0453313602125445, 0.9143027667195794]\n",
      "1.40393972897\n",
      "[0.9113589314491677, 1.167572609650576]\n",
      "1.30375013308\n",
      "[1.0645459286158887, 0.8779783470877084]\n",
      "1.42623560911\n",
      "[0.877269174936965, 1.2320180135078935]\n",
      "1.29229086002\n",
      "[1.0820668260038577, 0.8448557488753657]\n",
      "1.45070726866\n",
      "[0.8398527145952123, 1.3027524259632128]\n",
      "1.28184084579\n",
      "[1.098044595822, 0.8146503753549994]\n",
      "1.47757153914\n",
      "[0.7987780212603536, 1.3804025904439385]\n",
      "1.27231051896\n",
      "[1.1126161901669556, 0.7871033237335096]\n",
      "1.50369025968\n",
      "[0.7588432522488072, 1.4558977708209686]\n",
      "1.26361837185\n",
      "[1.1259062319387851, 0.761978999100695]\n",
      "1.5187944358\n",
      "[0.7357494033917624, 1.499555824397249]\n",
      "1.25569021547\n",
      "[1.1380281539679964, 0.7390629609524089]\n",
      "1.53252671847\n",
      "[0.7147531399762697, 1.539248471507289]\n",
      "1.24845850648\n",
      "[1.1490852280566521, 0.7181599778270049]\n",
      "1.54506023374\n",
      "[0.6955897572816008, 1.575476126480072]\n",
      "1.24186173871\n",
      "[1.159171495362305, 0.699092268449469]\n",
      "1.5564995926\n",
      "[0.678099308118208, 1.6085411634678617]\n",
      "1.23584389256\n",
      "[1.1683726082417525, 0.6816979102574268]\n",
      "1.56694028315\n",
      "[0.6621357939481172, 1.6387195872212037]\n",
      "1.23035393643\n",
      "[1.1767665925293775, 0.6658293983425841]\n",
      "1.57646946516\n",
      "[0.6475659499920307, 1.6662633297983025]\n",
      "1.22534537501\n",
      "[1.184424538227494, 0.6513523397265781]\n",
      "1.58516669564\n",
      "[0.634268135893408, 1.6914023477228952]\n",
      "1.22077583968\n",
      "[1.1914112257130276, 0.6381442695407761]\n",
      "1.59310459129\n",
      "[0.622131322811131, 1.7143465368475121]\n",
      "1.21660671701\n",
      "[1.1977856937985691, 0.6260935771281961]\n",
      "1.60034943338\n",
      "[0.6110541685945516, 1.7352874807002998]\n",
      "1.21280281157\n",
      "[1.2036017553116904, 0.6150985313601829]\n",
      "1.60696172\n",
      "[0.6009441734111285, 1.7544000467392509]\n",
      "1.20933203967\n",
      "[1.208908465261907, 0.6050663955843539]\n",
      "1.61299667025\n",
      "[0.5917169088527389, 1.7718438436977557]\n",
      "1.20616515126\n",
      "[1.2137505461392544, 0.595912623613623]\n",
      "1.61850468455\n",
      "[0.583295314147142, 1.7877645520703884]\n",
      "1.2032754771\n",
      "[1.2181687744230887, 0.5875601290458314]\n",
      "1.62353176483\n",
      "[0.5756090536504247, 1.8022951387492983]\n",
      "1.20063869899\n",
      "[1.2222003319667318, 0.5799386209842792]\n",
      "1.6281198982\n",
      "[0.5685939302988687, 1.8155569658714161]\n",
      "1.19823264071\n",
      "[1.225879125556349, 0.5729839999236764]\n",
      "1.63230740712\n",
      "[0.5621913501584072, 1.8276608030675892]\n",
      "1.19603707792\n",
      "[1.229236077615306, 0.566637808184475]\n",
      "1.6361292691\n",
      "[0.5563478336302091, 1.8387077515100583]\n",
      "1.19403356516\n",
      "[1.2322993907333615, 0.5608467298303704]\n",
      "1.63961740851\n",
      "[0.5510145692553626, 1.8487900874279402]\n",
      "1.19220527835\n",
      "[1.2350947884391048, 0.5555621354970566]\n",
      "1.64280096301\n",
      "[0.5461470064129772, 1.8579920320961607]\n",
      "1.19053687144\n",
      "[1.23764573440052, 0.5507396680017911]\n",
      "1.64570652667\n",
      "[0.5417044835272664, 1.866390454695997]\n",
      "1.18901434586\n",
      "[1.2399736320291554, 0.546338864999218]\n",
      "1.64835837197\n",
      "[0.5376498886926865, 1.8740555138905022]\n",
      "1.1876249316\n",
      "[1.2420980062754394, 0.5423228153041474]\n",
      "1.65077865237\n",
      "[0.5339493498944815, 1.8810512434509237]\n",
      "1.18635697896\n",
      "[1.2440366692337608, 0.5386578458213788]\n",
      "1.65298758728\n",
      "[0.5305719522470628, 1.8874360868069255]\n",
      "1.18519985978\n",
      "[1.2458058710238997, 0.5353132363100205]\n",
      "1.65500363084\n",
      "[0.5274894798966057, 1.8932633849700296]\n",
      "1.18414387759\n",
      "[1.2474204372784865, 0.5322609594686181]\n",
      "[[0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]] [[0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.60163984 0.39836016]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.5023748812559372\n",
      "1.41852071956\n",
      "[0.9664906570800407, 1.050608864545531]\n",
      "1.37896923589\n",
      "[1.0322301266202216, 0.9513231245296504]\n",
      "1.43938303245\n",
      "[0.9318149065088857, 1.1029793442610532]\n",
      "1.36335839916\n",
      "[1.0581772727202547, 0.9121353790266635]\n",
      "1.46222943798\n",
      "[0.8938413479792926, 1.1603304741989449]\n",
      "1.3491056802\n",
      "[1.081867058720391, 0.876356904538004]\n",
      "1.48724736727\n",
      "[0.8522584477122017, 1.2231327610732134]\n",
      "1.33609333517\n",
      "[1.1034951894535037, 0.8436921297837019]\n",
      "1.50925317116\n",
      "[0.8156820733257296, 1.2783737361443488]\n",
      "1.32421375977\n",
      "[1.1232405166230384, 0.8138709366162598]\n",
      "1.52571748673\n",
      "[0.7883163395700097, 1.319703962050013]\n",
      "1.31336862747\n",
      "[1.1412664711064286, 0.7866464968254565]\n",
      "1.54075320793\n",
      "[0.7633251067778558, 1.35744800012993]\n",
      "1.30346809912\n",
      "[1.1577223767172116, 0.7617932879750345]\n",
      "1.55448356017\n",
      "[0.7405035590349354, 1.391915182049898]\n",
      "1.29443009826\n",
      "[1.1727446546976485, 0.7391052742681345]\n",
      "1.56702126199\n",
      "[0.7196643441578745, 1.4233884641571457]\n",
      "1.28617964701\n",
      "[1.1864579275815168, 0.7183942393930755]\n",
      "1.57846940053\n",
      "[0.7006361185499943, 1.4521266251718585]\n",
      "1.27864825765\n",
      "[1.1989760304608228, 0.6994882592161722]\n",
      "1.58892223746\n",
      "[0.6832622076034451, 1.4783662893699976]\n",
      "1.2717733754\n",
      "[1.2104029371128953, 0.6822303030601679]\n",
      "1.5984659504\n",
      "[0.6673993734466787, 1.5023237876434075]\n",
      "1.26549786834\n",
      "[1.22083360789709, 0.6664769531333357]\n",
      "1.60717931436\n",
      "[0.6529166822453936, 1.5241968682051874]\n",
      "1.25976956045\n",
      "[1.2303547658138134, 0.6520972324543887]\n",
      "1.61513432777\n",
      "[0.6396944636878279, 1.5441662680698642]\n",
      "1.25454080441\n",
      "[1.2390456066329263, 0.638971532351828]\n",
      "1.62239678719\n",
      "[0.6276233557125276, 1.5623971557926009]\n",
      "1.2497680907\n",
      "[1.246978448543348, 0.6269906313038843]\n",
      "1.62902681471\n",
      "[0.6166034279616549, 1.5790404553099202]\n",
      "1.24541169017\n",
      "[1.2542193263502528, 0.6160547975277433]\n",
      "1.63507934161\n",
      "[0.6065433778600525, 1.5942340600943934]\n",
      "1.24143532705\n",
      "[1.2608285348495796, 0.6060729683258235]\n",
      "1.64060455172\n",
      "[0.5973597936257016, 1.6081039462234321]\n",
      "1.23780588011\n",
      "[1.2668611256405804, 0.5969619997541736]\n",
      "1.64564828781\n",
      "[0.588976478907985, 1.6207651923721522]\n",
      "1.2344931094\n",
      "[1.2723673612945143, 0.5886459806955089]\n",
      "1.65025242377\n",
      "[0.581323834124153, 1.6323229141754445]\n",
      "1.23146940645\n",
      "[1.2773931304800221, 0.5810556058990226]\n",
      "1.65445520543\n",
      "[0.574338289920341, 1.6428731198667943]\n",
      "1.22870956608\n",
      "[1.2819803273518067, 0.5741276029930167]\n",
      "1.65829156269\n",
      "[0.5679617885205506, 1.652503493592331]\n",
      "1.22619057776\n",
      "[1.286167198237594, 0.5678042088866458]\n",
      "1.66179339501\n",
      "[0.5621413090448804, 1.6612941123185128]\n",
      "1.22389143509\n",
      "[1.2899886584075904, 0.5620326913558055]\n",
      "1.66498983277\n",
      "[0.5568284331769614, 1.669318101800759]\n",
      "1.22179296163\n",
      "[1.2934765814794176, 0.556764911957416]\n",
      "1.66790747634\n",
      "[0.5519789478403212, 1.6766422366578284]\n",
      "1.21987765185\n",
      "[1.2966600637984842, 0.5519569267380827]\n",
      "1.67057061464\n",
      "[0.5475524818047242, 1.683327489202052]\n",
      "1.21812952585\n",
      "[1.2995656659376986, 0.5475686214992134]\n",
      "1.67300142513\n",
      "[0.5435121733870522, 1.6894295313077579]\n",
      "1.21653399666\n",
      "[1.3022176332800977, 0.5435633786530251]\n",
      "1.67522015658\n",
      "[0.5398243666377442, 1.6949991932582062]\n",
      "1.21507774897\n",
      "[1.3046380974822427, 0.5399077729541528]\n",
      "1.67724529613\n",
      "[0.5364583336139659, 1.7000828831939676]\n",
      "1.21374862852\n",
      "[1.3068472604640486, 0.5365712936214406]\n",
      "1.67909372207\n",
      "[0.5333860205354313, 1.7047229704915436]\n",
      "1.21253554093\n",
      "[1.3088635624309763, 0.5335260905755059]\n",
      "[[0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]] [[0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.55124488 0.44875512]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.4978751062446878\n",
      "1.47276032267\n",
      "[0.9564527503974531, 1.0534928669917205]\n",
      "1.42701955004\n",
      "[1.0394299800622653, 0.9515647326017675]\n",
      "1.49693791798\n",
      "[0.9125927593441429, 1.1073699014562703]\n",
      "1.40900030519\n",
      "[1.0721182564496934, 0.9114108850697061]\n",
      "1.51776375964\n",
      "[0.8748131035495161, 1.153777932293119]\n",
      "1.3925657091\n",
      "[1.101931859845743, 0.8747882479212163]\n",
      "1.53618633707\n",
      "[0.8413931512683309, 1.1948305608417318]\n",
      "1.37757552139\n",
      "[1.1291251969164542, 0.8413843113635446]\n",
      "1.55299135823\n",
      "[0.810907568721153, 1.2322786483155712]\n",
      "1.36390206071\n",
      "[1.1539298913159937, 0.8109145519551978]\n",
      "1.56832154713\n",
      "[0.7830974437110579, 1.2664402390420986]\n",
      "1.35142905534\n",
      "[1.176556868776067, 0.7831198713096128]\n",
      "1.58230691787\n",
      "[0.7577269207976182, 1.297605054733229]\n",
      "1.34005060306\n",
      "[1.197198243933524, 0.7577642783410167]\n",
      "1.595065926\n",
      "[0.7345811126625145, 1.3260370601362652]\n",
      "1.32967022931\n",
      "[1.2160290289661329, 0.7346327903987769]\n",
      "1.60670651227\n",
      "[0.7134642065575509, 1.351976789048178]\n",
      "1.32020003368\n",
      "[1.2332086818802743, 0.7135295313712229]\n",
      "1.61732704957\n",
      "[0.6941977466468696, 1.3756434542637719]\n",
      "1.31155991606\n",
      "[1.2488825103447838, 0.6942760072346719]\n",
      "1.62701720218\n",
      "[0.6766190750982978, 1.3972368625185128]\n",
      "1.30367687459\n",
      "[1.2631829452536762, 0.6767095416257762]\n",
      "1.63585870612\n",
      "[0.6605799165801582, 1.4169391532739724]\n",
      "1.29648436848\n",
      "[1.2762306966943366, 0.6606818558654461]\n",
      "1.64392607795\n",
      "[0.6459450924115626, 1.434916378238649]\n",
      "1.28992173926\n",
      "[1.2881358036691966, 0.6460577794945901]\n",
      "1.6512872589\n",
      "[0.6325913520242173, 1.4513199367848686]\n",
      "1.28393368521\n",
      "[1.2989985877444596, 0.6327140788245978]\n",
      "1.65800420052\n",
      "[0.6204063106438923, 1.4662878808868587]\n",
      "1.27846978348\n",
      "[1.3089105197589617, 0.6205383922836109]\n",
      "1.66413339717\n",
      "[0.6092874832125706, 1.4799461018380344]\n",
      "1.27348405579\n",
      "[1.317955007802647, 0.6094282624741553]\n",
      "1.66972637048\n",
      "[0.5991414055619646, 1.4924094097898597]\n",
      "1.26893457338\n",
      "[1.3262081138527013, 0.5992902558667689]\n",
      "1.6748301102\n",
      "[0.5898828347336759, 1.5037825160680136]\n",
      "1.26478309757\n",
      "[1.3337392057234772, 0.5900391619533135]\n",
      "1.6794874754\n",
      "[0.581434021130334, 1.5141609272523326]\n",
      "1.26099475283\n",
      "[1.340611550333108, 0.581597264486091]\n",
      "1.68373755975\n",
      "[0.573724045889409, 1.5236317591380768]\n",
      "1.25753772898\n",
      "[1.3468828537057396, 0.573893678146225]\n",
      "1.68761602409\n",
      "[0.5666882175049007, 1.5322744779178883]\n",
      "1.25438301029\n",
      "[1.352605752605457, 0.5668637446270314]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.69115539942\n",
      "[0.5602675222908698, 1.5401615752251414]\n",
      "1.25150412862\n",
      "[1.3578282622292082, 0.560448482693934]\n",
      "1.69438536282\n",
      "[0.5544081237919314, 1.547359183051489]\n",
      "1.24887693869\n",
      "[1.362594183965156, 0.5545940872994866]\n",
      "1.69733298883\n",
      "[0.5490609067057344, 1.553927633986472]\n",
      "1.24647941327\n",
      "[1.366943476844542, 0.5492514732968004]\n",
      "1.70002297861\n",
      "[0.5441810612967418, 1.5599219717181512]\n",
      "1.24429145669\n",
      "[1.3709125959746742, 0.5443758597129282]\n",
      "1.70247786864\n",
      "[0.5397277046541562, 1.5653924162748747]\n",
      "1.24229473477\n",
      "[1.3745348009338758, 0.5399263909205757]\n",
      "1.70471822107\n",
      "[0.535663535484011, 1.5703847880741195]\n",
      "1.24047251992\n",
      "[1.3778404368325592, 0.5358657913863862]\n",
      "1.70676279716\n",
      "[0.531954519430007, 1.5749408944702254]\n",
      "1.23880954993\n",
      "[1.3808571904948679, 0.5321600509807919]\n",
      "1.70862871544\n",
      "[0.5285696021930381, 1.579098882154586]\n",
      "1.23729189919\n",
      "[1.3836103239896365, 0.5287781381116612]\n",
      "1.71033159588\n",
      "[0.5254804479685066, 1.5828935584557986]\n",
      "1.23590686142\n",
      "[1.3861228875354135, 0.5256917381945779]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]] [[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.50564944 0.49435056]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.5045747712614372\n",
      "1.51735657709\n",
      "[0.9545020511138067, 1.0465378494169657]\n",
      "1.46919140551\n",
      "[1.0497561308108077, 0.9491066437074727]\n",
      "1.53757637576\n",
      "[0.914514270526961, 1.0874395901992633]\n",
      "1.44952125724\n",
      "[1.0886568926867612, 0.909316766521652]\n",
      "1.55601383328\n",
      "[0.8780513453275983, 1.1247359115449707]\n",
      "1.43156852464\n",
      "[1.1241611993430347, 0.8730009738920455]\n",
      "1.57282738536\n",
      "[0.8447999444304585, 1.1587473060305766]\n",
      "1.4151832867\n",
      "[1.1565655427646055, 0.8398559972167048]\n",
      "1.58816116201\n",
      "[0.8144750284205307, 1.1897653279282718]\n",
      "1.40022870503\n",
      "[1.186140542011031, 0.8096050321703512]\n",
      "1.60214631637\n",
      "[0.7868172216764827, 1.2180552811452319]\n",
      "1.38657988305\n",
      "[1.21313319944068, 0.7819954309119094]\n",
      "1.6149022239\n",
      "[0.7615904407360828, 1.2438586450642453]\n",
      "1.37412282442\n",
      "[1.2377689604399187, 0.7567965952781818]\n",
      "1.62653756569\n",
      "[0.7385797519377588, 1.2673952658678709]\n",
      "1.3627534823\n",
      "[1.2602535937270947, 0.7337980535036573]\n",
      "1.63715130783\n",
      "[0.7175894345229141, 1.28886533770583]\n",
      "1.35237689142\n",
      "[1.2807749078254977, 0.7128077045158692]\n",
      "1.64683358758\n",
      "[0.6984412281190029, 1.3084511952674356]\n",
      "1.34290637578\n",
      "[1.2995043179501093, 0.6936502152359012]\n",
      "1.65566651574\n",
      "[0.6809727459049675, 1.3263189368848258]\n",
      "1.33426282538\n",
      "[1.3165982763193886, 0.6761655575754151]\n",
      "1.66372490368\n",
      "[0.6650360368398952, 1.3426198951659891]\n",
      "1.32637403612\n",
      "[1.3321995777757476, 0.660207672974928]\n",
      "1.67107692242\n",
      "[0.6504962821558059, 1.3574919702949209]\n",
      "1.31917410706\n",
      "[1.3464385515678106, 0.6456432533821967]\n",
      "1.67778470058\n",
      "[0.6372306129126846, 1.3710608395025392]\n",
      "1.31260289046\n",
      "[1.359434149205769, 0.6323506285328528]\n",
      "1.68390486699\n",
      "[0.6251270368189334, 1.3834410547748084]\n",
      "1.30660548959\n",
      "[1.3712949374406207, 0.6202187502756432]\n",
      "1.68948904347\n",
      "[0.614083463759304, 1.39473703959733]\n",
      "1.30113180048\n",
      "[1.3821200046318707, 0.6091462654888002]\n",
      "1.69458429241\n",
      "[0.6040068205672542, 1.4050439944157354]\n",
      "1.29613609357\n",
      "[1.3919997880499906, 0.5990406698687697]\n",
      "1.69923352357\n",
      "[0.5948122465482425, 1.414448719499502]\n",
      "1.29157663191\n",
      "[1.401016829003853, 0.5898175355435912]\n",
      "1.70347586389\n",
      "[0.5864223621209385, 1.4230303630166694]\n",
      "1.28741532259\n",
      "[1.4092464620840357, 0.5813998060762603]\n",
      "1.70734699381\n",
      "[0.5787666037083401, 1.4308611013444426]\n",
      "1.2836173987\n",
      "[1.4167574442655284, 0.573717152983257]\n",
      "1.71087945318\n",
      "[0.5717806186922375, 1.4380067579436397]\n",
      "1.28015112891\n",
      "[1.423612529113489, 0.5667053884047504]\n",
      "1.71410291971\n",
      "[0.5654057148525024, 1.4445273665030127]\n",
      "1.27698755256\n",
      "[1.4298689908791575, 0.5603059290299554]\n",
      "1.71704446228\n",
      "[0.5595883592561208, 1.4504776835035997]\n",
      "1.27410023766\n",
      "[1.435579102856148, 0.5544653068075361]\n",
      "1.71972877168\n",
      "[0.5542797220473119, 1.4559076548557328]\n",
      "1.27146506023\n",
      "[1.4407905739866638, 0.5491347223603237]\n",
      "1.72217837066\n",
      "[0.54943526102611, 1.460862840815324]\n",
      "1.26906000272\n",
      "[1.4455469473595963, 0.5442696373791507]\n",
      "1.72441380522\n",
      "[0.5450143432941854, 1.4653848029857006]\n",
      "1.26686497015\n",
      "[1.449887963925074, 0.5398294025952468]\n",
      "1.72645381889\n",
      "[0.5409799005984504, 1.4695114568514636]\n",
      "1.26486162221\n",
      "[1.4538498944602676, 0.5357769182270281]\n",
      "1.72831551153\n",
      "[0.5372981153194978, 1.4732773929670913]\n",
      "1.2630332201\n",
      "[1.4574658425567042, 0.5320783240677076]\n",
      "1.73001448402\n",
      "[0.5339381343370716, 1.476714169631357]\n",
      "1.26136448665\n",
      "[1.4607660211577997, 0.5287027166272261]\n",
      "1.73156497006\n",
      "[0.5308718082618503, 1.47985057961566]\n",
      "1.25984147875\n",
      "[1.4637780049548181, 0.5256218909675333]\n",
      "[[0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]] [[0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]\n",
      " [0.2 0.2 0.3 0.3]\n",
      " [0.3 0.3 0.2 0.2]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.46335366 0.53664634]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.4973751312434378\n",
      "1.55903857546\n",
      "[0.9516735448373135, 1.0417262517652113]\n",
      "1.51220016242\n",
      "[1.0527592092365732, 0.9544463991797503]\n",
      "1.57750006712\n",
      "[0.9118303471569326, 1.0761278500605131]\n",
      "1.48974108119\n",
      "[1.101229919504985, 0.9125955940029623]\n",
      "1.59434287049\n",
      "[0.8754805680321732, 1.107513144724969]\n",
      "1.47023899153\n",
      "[1.143318913628691, 0.8762549197400121]\n",
      "1.60970942744\n",
      "[0.8423167924399799, 1.1361475654617352]\n",
      "1.45245125069\n",
      "[1.181708036651106, 0.8431088053211804]\n",
      "1.62372955881\n",
      "[0.8120588438364484, 1.16227302360013]\n",
      "1.43622625786\n",
      "[1.2167244702533, 0.8128747540238881]\n",
      "1.63652159709\n",
      "[0.7844513395509054, 1.1861100228286015]\n",
      "1.42142596889\n",
      "[1.248666138354229, 0.7852955309980441]\n",
      "1.64819341511\n",
      "[0.7592614707619597, 1.2078595760180881]\n",
      "1.4079246589\n",
      "[1.2778043777072523, 0.7601368574072279]\n",
      "1.65884336055\n",
      "[0.7362769855678918, 1.2277049466887255]\n",
      "1.39560780198\n",
      "[1.3043863557185014, 0.7371853228247558]\n",
      "1.66856110538\n",
      "[0.7153043558586373, 1.2458132317777297]\n",
      "1.3843710562\n",
      "[1.3286372608908508, 0.7162464939504001]\n",
      "1.67742841809\n",
      "[0.6961671106453862, 1.2623368006836744]\n",
      "1.37411934357\n",
      "[1.350762288507714, 0.6971432001220892]\n",
      "1.68551986602\n",
      "[0.6787043202373847, 1.2774146040655785]\n",
      "1.3647660155\n",
      "[1.3709484417239595, 0.6797139782096463]\n",
      "1.6929034544\n",
      "[0.662769217200386, 1.2911733645413475]\n",
      "1.35623209543\n",
      "[1.3893661660796262, 0.6638116613353853]\n",
      "1.69964120783\n",
      "[0.6482279414102922, 1.3037286602393712]\n",
      "1.34844559117\n",
      "[1.406170833555729, 0.6493020975037733]\n",
      "1.70578969951\n",
      "[0.6349583977485664, 1.3151859110924433]\n",
      "1.34134087032\n",
      "[1.4215040906161311, 0.6360629856688743]\n",
      "1.71140053321\n",
      "[0.6228492160901528, 1.3256412768098067]\n",
      "1.33485809264\n",
      "[1.435495083196478, 0.6239828180487276]\n",
      "1.71652078204\n",
      "[0.6117988042247691, 1.3351824746082392]\n",
      "1.32894269406\n",
      "[1.4482615702855057, 0.612959918631818]\n",
      "1.7211933882\n",
      "[0.6017144852414927, 1.3438895240154505]\n",
      "1.3235449175\n",
      "[1.4599109365747431, 0.6029015688303784]\n",
      "1.72545752704\n",
      "[0.5925117117058604, 1.3518354253689293]\n",
      "1.31861938601\n",
      "[1.4705411136113362, 0.5937232121343521]\n",
      "1.72934893887\n",
      "[0.5841133496781364, 1.3590867780122071]\n",
      "1.31412471445\n",
      "[1.480241417959942, 0.585347730421768]\n",
      "1.7329002312\n",
      "[0.5764490262696605, 1.3657043436307794]\n",
      "1.31002315595\n",
      "[1.4890933140497293, 0.5777047852978489]\n",
      "1.73614115436\n",
      "[0.5694545350188713, 1.3717435596650922]\n",
      "1.30628028023\n",
      "[1.4971711086399042, 0.5707302184763711]\n",
      "1.73909885252\n",
      "[0.5630712938964891, 1.377255007282219]\n",
      "1.30286468049\n",
      "[1.5045425831716284, 0.564365505791443]\n",
      "1.74179809273\n",
      "[0.5572458512263012, 1.382284837976024]\n",
      "1.29974770664\n",
      "[1.5112695696769576, 0.5585572599435398]\n",
      "1.74426147356\n",
      "[0.5519294352393507, 1.3868751624931712]\n",
      "1.29690322219\n",
      "[1.5174084753787382, 0.5532567775470332]\n",
      "1.74650961544\n",
      "[0.5470775433696758, 1.3910644054452994]\n",
      "1.29430738278\n",
      "[1.523010760632638, 0.5484196264632671]\n",
      "1.74856133417\n",
      "[0.5426495677532627, 1.394887628662452]\n",
      "1.29193843445\n",
      "[1.52812337442763, 0.5440052697787149]\n",
      "1.75043379917\n",
      "[0.5386084537122372, 1.3983768260662368]\n",
      "1.28977652965\n",
      "[1.5327891512692393, 0.5399767231262055]\n",
      "1.7521426778\n",
      "[0.5349203882967853, 1.4015611925904037]\n",
      "1.28780355971\n",
      "[1.5370471729160917, 0.5363002423526795]\n",
      "1.75370226696\n",
      "[0.5315545162208047, 1.4044673694490015]\n",
      "1.28600300203\n",
      "[1.5409330981207345, 0.5329450388128407]\n",
      "1.75512561317\n",
      "[0.5284826807664871, 1.4071196678457423]\n",
      "1.28435978076\n",
      "[1.5444794632368617, 0.5298830198174739]\n",
      "[[0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]] [[0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]\n",
      " [0.15 0.15 0.35 0.35]\n",
      " [0.35 0.35 0.15 0.15]]\n",
      "[0.25 0.25 0.25 0.25]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.40165983 0.59834017]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.4958752062396878\n",
      "1.61716621898\n",
      "[0.9531293611994611, 1.0314637961333162]\n",
      "1.57725007909\n",
      "[1.0525073335971706, 0.9647523464137977]\n",
      "1.63291650175\n",
      "[0.9139163719291334, 1.0577870878944973]\n",
      "1.5561888568\n",
      "[1.1049428038666718, 0.9295529339685127]\n",
      "1.64729432034\n",
      "[0.8781203640190688, 1.0818165938728945]\n",
      "1.53311942243\n",
      "[1.1623780573231295, 0.8909972165329193]\n",
      "1.66041903221\n",
      "[0.8454441769617428, 1.1037517949773863]\n",
      "1.50785102501\n",
      "[1.2252880007083828, 0.8487663939094964]\n",
      "1.67239965446\n",
      "[0.8156163942524022, 1.1237748904224767]\n",
      "1.48763932413\n",
      "[1.2756084439614874, 0.8149867781762542]\n",
      "1.68333575306\n",
      "[0.7883891296040696, 1.1420522838202627]\n",
      "1.47133093131\n",
      "[1.3162109424902315, 0.7877307225963804]\n",
      "1.69331825667\n",
      "[0.7635360007684241, 1.158735943334432]\n",
      "1.45644183618\n",
      "[1.353279859683739, 0.7628467252089608]\n",
      "1.7024302013\n",
      "[0.7408502755286893, 1.173964646256894]\n",
      "1.44284890849\n",
      "[1.3871217491263523, 0.7401289996255752]\n",
      "1.71074741161\n",
      "[0.7201431756213401, 1.1878651175683617]\n",
      "1.43043963508\n",
      "[1.4180167312719238, 0.7193895037567984]\n",
      "1.71833912395\n",
      "[0.70124232545603, 1.200553071297314]\n",
      "1.41911121534\n",
      "[1.446220745670285, 0.7004564279148506]\n",
      "1.72526855616\n",
      "[0.6839903335402314, 1.2121341627956033]\n",
      "1.40876973175\n",
      "[1.4719676158217545, 0.6831728086971945]\n",
      "1.73159342852\n",
      "[0.6682434954797053, 1.2227048594014078]\n",
      "1.39932938987\n",
      "[1.4954709414829068, 0.6673952587003948]\n",
      "1.73736643999\n",
      "[0.6538706083214476, 1.2323532363590817]\n",
      "1.39071182202\n",
      "[1.5169258322057606, 0.6529928028124098]\n",
      "1.74263570349\n",
      "[0.640751886835682, 1.2411597043083331]\n",
      "1.38284544965\n",
      "[1.5365104949009942, 0.6398458124971101]\n",
      "1.74744514369\n",
      "[0.6287779731015206, 1.2491976741395707]\n",
      "1.37566489953\n",
      "[1.5543876872760731, 0.6278450301156442]\n",
      "1.75183486057\n",
      "[0.6178490314706581, 1.2565341645358232]\n",
      "1.36911046943\n",
      "[1.5707060481124333, 0.6168906759245246]\n",
      "1.75584146153\n",
      "[0.6078739216386254, 1.263230357081825]\n",
      "1.3631276392\n",
      "[1.5856013145121786, 0.606891630949963]\n",
      "1.75949836485\n",
      "[0.5987694431571756, 1.2693421034153787]\n",
      "1.35766662351\n",
      "[1.599197435463559, 0.5977646894623804]\n",
      "1.76283607688\n",
      "[0.5904596452778326, 1.2749203885225513]\n",
      "1.35268196276\n",
      "[1.61160759034437, 0.5894338752651512]\n",
      "1.76588244531\n",
      "[0.5828751965287441, 1.280011753934498]\n",
      "1.34813214896\n",
      "[1.6229351203018878, 0.5818298164684687]\n",
      "1.76866289038\n",
      "[0.5759528088979494, 1.2846586842675363]\n",
      "1.3439792837\n",
      "[1.6332743798148812, 0.5748891738441881]\n",
      "1.77120061616\n",
      "[0.5696347119289775, 1.28889996025757]\n",
      "1.34018876549\n",
      "[1.642711515155545, 0.5685541182520347]\n",
      "1.77351680345\n",
      "[0.5638681724321928, 1.2927709811731085]\n",
      "1.33672900393\n",
      "[1.65132517592444, 0.562771852993236]\n",
      "1.77563078597\n",
      "[0.5586050558801584, 1.2963040592462238]\n",
      "1.33357115857\n",
      "[1.6591871653274686, 0.5574941772860226]\n",
      "1.7775602112\n",
      "[0.553801425889992, 1.29952868853608]\n",
      "1.33068890019\n",
      "[1.6663630343980302, 0.5526770873701725]\n",
      "1.77932118728\n",
      "[0.5494171785026235, 1.30247179043365]\n",
      "1.32805819274\n",
      "[1.6729126249374446, 0.548280412036478]\n",
      "1.78092841715\n",
      "[0.5454157082501717, 1.305157937827383]\n",
      "1.32565709411\n",
      "[1.6788905655502606, 0.5442674796431489]\n",
      "1.78239532104\n",
      "[0.5417636032604215, 1.307609559776552]\n",
      "1.3234655741\n",
      "[1.68434672478575, 0.5406048139264108]\n",
      "1.78373414821\n",
      "[0.5384303668834517, 1.309847128380544]\n",
      "1.32146534818\n",
      "[1.6893266250607366, 0.5372618561382054]\n",
      "1.7849560791\n",
      "[0.535388163541606, 1.3118893293872609]\n",
      "1.31963972564\n",
      "[1.693871820729753, 0.5342107112514342]\n",
      "[[0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]] [[0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]\n",
      " [0.1 0.1 0.4 0.4]\n",
      " [0.4 0.4 0.1 0.1]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.34566543 0.65433457]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.5054247287635618\n",
      "1.6697071904\n",
      "[0.9555274483063245, 1.0234935224946573]\n",
      "1.63508186281\n",
      "[1.0556975094086933, 0.9705766671720886]\n",
      "1.68335423324\n",
      "[0.9160469520798885, 1.0443498909932496]\n",
      "1.61678942753\n",
      "[1.1086169902508072, 0.9426208839705011]\n",
      "1.69580440544\n",
      "[0.8800289676669861, 1.0633771177835007]\n",
      "1.59673465006\n",
      "[1.1666348755351603, 0.9119717657816245]\n",
      "1.70716308135\n",
      "[0.8471686501081661, 1.080736243364314]\n",
      "1.57474647374\n",
      "[1.2302460272353342, 0.8783678917859796]\n",
      "1.71752629001\n",
      "[0.8171881902164291, 1.096574026042452]\n",
      "1.55063698323\n",
      "[1.299994078884689, 0.8415220766038555]\n",
      "1.7269815553\n",
      "[0.7898343839974955, 1.1110242259353085]\n",
      "1.5241996916\n",
      "[1.3764763911735214, 0.8011187524011516]\n",
      "1.73560865865\n",
      "[0.7648764260521844, 1.124208770650611]\n",
      "1.49795413949\n",
      "[1.4524040066401136, 0.7610084579836686]\n",
      "1.74348033186\n",
      "[0.7421039054185932, 1.1362388140232156]\n",
      "1.48308781975\n",
      "[1.4954118353148933, 0.7382887049688897]\n",
      "1.75066288656\n",
      "[0.7213249845299505, 1.1472156981173534]\n",
      "1.46951997833\n",
      "[1.5346632041294404, 0.7175533776473907]\n",
      "1.75721678634\n",
      "[0.7023647439308128, 1.1572318276636888]\n",
      "1.45713718356\n",
      "[1.5704862676486304, 0.6986291217510215]\n",
      "1.7631971671\n",
      "[0.6850636771415977, 1.1663714651774904]\n",
      "1.44583591581\n",
      "[1.6031805050676202, 0.681357731354101]\n",
      "1.76865431019\n",
      "[0.669276321616697, 1.174711454182622]\n",
      "1.43552170091\n",
      "[1.6330192274157007, 0.6655948243924087]\n",
      "1.77363407305\n",
      "[0.6548700131281575, 1.1823218772334902]\n",
      "1.42610831933\n",
      "[1.6602518653043856, 0.6512086341141105]\n",
      "1.77817828106\n",
      "[0.641723752145468, 1.189266654772787]\n",
      "1.41751708457\n",
      "[1.68510605646837, 0.6380789062941389]\n",
      "1.78232508417\n",
      "[0.6297271718899444, 1.1956040902775769]\n",
      "1.40967618472\n",
      "[1.7077895506541256, 0.6260958929383691]\n",
      "1.78610928179\n",
      "[0.6187795987347746, 1.2013873666219261]\n",
      "1.40252008161\n",
      "[1.7284919478651424, 0.6151594340205077]\n",
      "1.7895626185\n",
      "[0.6087891965122613, 1.206664998113862]\n",
      "1.39598896253\n",
      "[1.7473862845654604, 0.6051781195380812]\n",
      "1.79271405352\n",
      "[0.5996721870896582, 1.2114812422419088]\n",
      "1.39002823984\n",
      "[1.7646304811597215, 0.5960685248519013]\n",
      "1.7955900061\n",
      "[0.591352140294433, 1.2158764747863915]\n",
      "1.38458809443\n",
      "[1.7803686628981037, 0.5877545128913899]\n",
      "1.79821457923\n",
      "[0.5837593269174919, 1.2198875316085316]\n",
      "1.37962305901\n",
      "[1.7947323652879779, 0.5801665973715556]\n",
      "1.80060976339\n",
      "[0.5768301291067801, 1.2235480201219227]\n",
      "1.37509163792\n",
      "[1.8078416341216093, 0.573241361681173]\n",
      "1.80279562235\n",
      "[0.5705065029904192, 1.2268886031726958]\n",
      "1.37095596006\n",
      "[1.8198060293425273, 0.5669209285701226]\n",
      "1.80479046246\n",
      "[0.5647354888442428, 1.2299372578034005]\n",
      "1.36718146227\n",
      "[1.83072554116446, 0.5611524761910851]\n",
      "1.806610987\n",
      "[0.559468764548568, 1.2327195111484721]\n",
      "1.36373660033\n",
      "[1.8406914261192706, 0.5558877964403548]\n",
      "1.80827243692\n",
      "[0.5546622384679, 1.2352586555037393]\n",
      "1.3605925852\n",
      "[1.8497869700376872, 0.5510828918978783]\n",
      "1.80978871916\n",
      "[0.5502756782393594, 1.2375759444264265]\n",
      "1.35772314235\n",
      "[1.858088184353097, 0.546697607990731]\n",
      "1.81117252365\n",
      "[0.5462723722744524, 1.2396907715536702]\n",
      "1.35510429212\n",
      "[1.8656644415590307, 0.5426952972998824]\n",
      "1.81243543001\n",
      "[0.5426188210678453, 1.2416208336748868]\n",
      "1.35271414922\n",
      "[1.872579055140509, 0.5390425131997647]\n",
      "1.81358800487\n",
      "[0.5392844556688985, 1.2433822794548621]\n",
      "1.35053273981\n",
      "[1.8788898088336958, 0.5357087302661848]\n",
      "1.81463989063\n",
      "[0.5362413809095466, 1.2449898450788046]\n",
      "1.34854183453\n",
      "[1.8846494396434872, 0.5326660891125405]\n",
      "[[0.45 0.45 0.05 0.05]\n",
      " [0.05 0.05 0.45 0.45]\n",
      " [0.45 0.45 0.05 0.05]\n",
      " [0.05 0.05 0.45 0.45]] [[0.05 0.05 0.45 0.45]\n",
      " [0.45 0.45 0.05 0.05]\n",
      " [0.05 0.05 0.45 0.45]\n",
      " [0.45 0.45 0.05 0.05]]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "[0.30006999 0.69993001]\n",
      "[0.25 0.25 0.25 0.25]\n",
      "simulated value, 1.5012249387530623\n",
      "1.71459332874\n",
      "[0.9511336618581039, 1.0209496972519758]\n",
      "1.6845679534\n",
      "[1.0511949010599502, 0.9780520145598699]\n",
      "1.72641576431\n",
      "[0.9117347354801446, 1.0378405798320125]\n",
      "1.66868857869\n",
      "[1.104113803558552, 0.955364925074398]\n",
      "1.73719563323\n",
      "[0.8758102206294329, 1.0532419325558675]\n",
      "1.65126575474\n",
      "[1.1621763368296283, 0.9304726875963265]\n",
      "1.74702568482\n",
      "[0.8430510250400716, 1.067286267693535]\n",
      "1.6321469142\n",
      "[1.225890940041315, 0.9031573269908592]\n",
      "1.75599028146\n",
      "[0.8131760063864732, 1.0800941149763135]\n",
      "1.61116389217\n",
      "[1.2958180321231256, 0.8731785836569286]\n",
      "1.76416617964\n",
      "[0.785929369341383, 1.0917751375152156]\n",
      "1.58813122692\n",
      "[1.3725756746168227, 0.8402714857821307]\n",
      "1.77162323458\n",
      "[0.7610783175565842, 1.1024291384303844]\n",
      "1.56284425453\n",
      "[1.456845921506292, 0.8041436270799454]\n",
      "1.77842503633\n",
      "[0.7384109335629228, 1.1121469697682385]\n",
      "1.53507696817\n",
      "[1.5493819531243156, 0.7644721083819898]\n",
      "1.78462948479\n",
      "[0.7177342627678747, 1.121011353919087]\n",
      "1.50548194855\n",
      "[1.6480090078512006, 0.7221892810626497]\n",
      "1.79028930966\n",
      "[0.6988725804873825, 1.1290976265653379]\n",
      "1.49021092111\n",
      "[1.6989005591429978, 0.7003713460016949]\n",
      "1.79545254116\n",
      "[0.6816658233472336, 1.136474409162136]\n",
      "1.47815230238\n",
      "[1.7390865791026167, 0.6831430251590068]\n",
      "1.80016293635\n",
      "[0.6659681684805686, 1.1432042180556876]\n",
      "1.46715156812\n",
      "[1.7757471400178713, 0.6674261189723384]\n",
      "1.80446036556\n",
      "[0.6516467457764201, 1.149344016560709]\n",
      "1.45711552673\n",
      "[1.8091928081169073, 0.653087483263023]\n",
      "1.80838116283\n",
      "[0.6385804700366942, 1.1549457156314116]\n",
      "1.44795926047\n",
      "[1.8397065764884637, 0.64000579485116]\n",
      "1.81195844387\n",
      "[0.6266589813071037, 1.160056628156769]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.43960538014\n",
      "[1.8675463489626734, 0.628070486680431]\n",
      "1.81522239473\n",
      "[0.615781682887753, 1.164719881379122]\n",
      "1.43198334866\n",
      "[1.8929471942734848, 0.6171807814264674]\n",
      "1.81820053397\n",
      "[0.6058568676239154, 1.1689747914658042]\n",
      "1.42502886695\n",
      "[1.9161233927307155, 0.6072448140593032]\n",
      "1.8209179509\n",
      "[0.5968009240461342, 1.1728572038482217]\n",
      "1.41868331604\n",
      "[1.937270295319819, 0.5981788348207461]\n",
      "1.82339752205\n",
      "[0.5885376147875011, 1.1763998025746727]\n",
      "1.41289325014\n",
      "[1.9565660131025067, 0.5899064849541967]\n",
      "1.82566010811\n",
      "[0.5809974204687667, 1.179632391596176]\n",
      "1.40760993582\n",
      "[1.9741729529772576, 0.5823581383021786]\n",
      "1.82772473296\n",
      "[0.5741169429207914, 1.1825821506135292]\n",
      "1.40278893291\n",
      "[1.9902392142463576, 0.5754703025780973]\n",
      "1.82960874662\n",
      "[0.5678383622191167, 1.1852738678543473]\n",
      "1.39838971334\n",
      "[2.004899859000263, 0.5691850747343158]\n",
      "1.83132797354\n",
      "[0.5621089425459194, 1.1877301519170993]\n",
      "1.39437531425\n",
      "[2.0182780680493453, 0.5634496453977021]\n",
      "1.83289684754\n",
      "[0.5568805823781078, 1.1899716246119]\n",
      "1.39071202228\n",
      "[2.0304861929886724, 0.5582158478344278]\n",
      "1.83432853473\n",
      "[0.5521094049333812, 1.1920170965421317]\n",
      "1.38736908623\n",
      "[2.041626713957395, 0.5534397473448369]\n",
      "1.8356350454\n",
      "[0.5477553851945798, 1.1938837270044382]\n",
      "1.38431845532\n",
      "[2.0517931117365817, 0.5490812673826454]\n",
      "1.83682733601\n",
      "[0.5437820101816072, 1.1955871696349996]\n",
      "1.38153454088\n",
      "[2.0610706620056805, 0.5451038490458505]\n",
      "1.83791540203\n",
      "[0.5401559694539942, 1.197141705095509]\n",
      "1.37899399923\n",
      "[2.0695371588377593, 0.5414741409039835]\n",
      "1.83890836254\n",
      "[0.5368468731097457, 1.1985603619710932]\n",
      "1.37667553393\n",
      "[2.07726357384788, 0.5381617164117873]\n",
      "1.83981453741\n",
      "[0.5338269948007253, 1.199855026943289]\n",
      "1.37455971552\n",
      "[2.084314656809294, 0.5351388164164728]\n"
     ]
    }
   ],
   "source": [
    "from copy import deepcopy\n",
    "npi = 8\n",
    "\n",
    "# pu1 = 1/6\n",
    "# pu2 = 1/8\n",
    "nSmarg = int(nS / nU)\n",
    "pis1u1 = 1.0/4\n",
    "pis2u1 = 1.0/4\n",
    "p0 = 0.25\n",
    "p_e_s = np.asarray([[ p0 ,1-p0 ], [1-p0,p0]])\n",
    "p_e_su = np.asarray([[ p0 ,p0,1-p0,1-p0 ], [1-p0,1-p0,p0, p0]])\n",
    "\n",
    "\n",
    "pi_s1u1s = np.linspace(1.0/10, 0.45, npi)\n",
    "bias = np.zeros(npi)\n",
    "tight = True; quiet = True; sense_min = False\n",
    "ngams = 30\n",
    "logGams_full = np.linspace(0.1, 2.75, ngams)\n",
    "min_bnds = [None] * ngams; w_min_bnds = [None] * ngams\n",
    "max_bnds = [None] * ngams; w_max_bnds = [None] * ngams\n",
    "ms_min = [None] * ngams; ms_max = [None] * ngams; \n",
    "w_min_bnds_ = [None] * npi; w_max_bnds_ = [None] * npi; \n",
    "min_bnds_ = [None] * npi; max_bnds_ = [None] * npi; \n",
    "\n",
    "\n",
    "phi = np.asarray([1,2])\n",
    "\n",
    "for indpi,param in enumerate(pi_s1u1s): \n",
    "    pu1 = param; pu2 = param\n",
    "    [P,Pi] = get_P(pu1, pu2, pis1u1, pis2u1)\n",
    "    print P[:,0,:], P[:,1,:]\n",
    "    \n",
    "    # simulate\n",
    "    [ stateChangeHist, stateHist, a_s, s_a_sprime, p_infty_b_su_long, distr_hist ] =  simulate_rollouts( \n",
    "        nS, nA, P, Pi, state_dist, 10000 )\n",
    "    p_infty_b_s = (reshape_byxrow(p_infty_b_su_long.T, nU).T).flatten()\n",
    "    [ p_a1_su, joint_s_a_sprime, s_a_giv_sprime ] = get_auxiliary_info_from_traj(stateChangeHist, \n",
    "                                            stateHist, a_s, s_a_sprime, p_infty_b_su_long, distr_hist, nA,nS)\n",
    "    # agg history and process\n",
    "    [aggStateHist, p_a1_s, p_e_s, agg_s_a_sprime, joint_s_a_sprime_agg, s_a_giv_sprime_agg] = agg_history(\n",
    "                stateHist, s_a_sprime, p_infty_b_s, a_s, p_e_su, nA, nS, nSmarg, nU)\n",
    "\n",
    "    print p_infty_b_s\n",
    "    \n",
    "    \n",
    "    # Simulate from evaluation policy: \n",
    "    [ stateChangeHist_e, stateHist_e, a_s_e, s_a_sprime_e, p_infty_e, distr_hist_e ] =  simulate_rollouts( \n",
    "        nS, nA, P, p_e_su, state_dist, n )\n",
    "    print 'simulated value,', np.dot( (reshape_byxrow(p_infty_e.T, nU).T).flatten(), phi) \n",
    "    \n",
    "    for ind,logGam in enumerate(logGams_full): \n",
    "        sense_min = False\n",
    "        [a_bnd, b_bnd] = get_bnds_as( p_a1_s, logGam )\n",
    "        [objVal, w_, m] = primal_opt_outer_L1_test_function(gamma, phi, a_bnd,b_bnd, s_a_giv_sprime_agg, p_infty_b_s, p_e_s, p_a1_s, nSmarg, nA, tight, sense_min, quiet)\n",
    "\n",
    "        print objVal\n",
    "        print w_\n",
    "        ms_min[ind] = m \n",
    "        min_bnds[ind] = objVal; w_min_bnds[ind] = w_; \n",
    "        sense_min = True\n",
    "        [objVal, w_, m] = primal_opt_outer_L1_test_function(gamma, phi, a_bnd,b_bnd, s_a_giv_sprime_agg, p_infty_b_s, p_e_s, p_a1_s, nSmarg, nA, tight, sense_min, quiet)\n",
    "        print objVal\n",
    "        max_bnds[ind] = objVal; w_max_bnds[ind] = w_; \n",
    "        ms_max[ind] = m \n",
    "        print w_  \n",
    "\n",
    "    w_min_bnds_[indpi] = deepcopy(w_min_bnds); w_max_bnds_[indpi] = deepcopy(w_max_bnds)\n",
    "    min_bnds_[indpi] = deepcopy(min_bnds); max_bnds_[indpi] = deepcopy(max_bnds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_e_su = np.asarray([ p0 ,1-p0 ])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_bnds[m_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(3,3))\n",
    "for m_ in range(npi): \n",
    "    \n",
    "    if m_==0: \n",
    "        plt.plot(np.exp(logGams_full),min_bnds_[m_],color=((m_)*1.0/(npi), 0.3,0),label=r'$p_{u_1}=$'+str(np.round(pi_s1u1s[m_],2)) if m_%2 ==0 else None ) \n",
    "    else:\n",
    "        plt.plot(np.exp(logGams_full),min_bnds_[m_],color=((m_)*1.0/(npi), 0.3,0),label=np.round(pi_s1u1s[m_],2) if m_%2 ==0 else None ) \n",
    "    plt.plot(np.exp(logGams_full),max_bnds_[m_],color=((m_)*1.0/(npi), 0.3,0))\n",
    "plt.axvline(3,linestyle='--',color='black',linewidth=1)\n",
    "plt.plot(3,1.5,marker='*',markersize=12, color='black')\n",
    "plt.xlabel(r'$\\Gamma$ parameter')\n",
    "plt.ylabel(r'${R}_e$')\n",
    "plt.legend()\n",
    "# plt.savefig('conf-randomwalk-varyingtransitions.pdf',bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_u0 = np.zeros([nS,nA]); \n",
    "for a in range(nA): \n",
    "    R_u0[:,a] = R_u0_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(1 + pis2u2 - 2*pu1 + pis2u1*(-1 + 4*pu1) + 2*pu2 - 4*pis2u2*pu2)/(2.*(2 + pis1u1 - pis2u1 + pis2u2 - 4*pis1u1*pu1 + 4*pis2u1*pu1 -  4*pis2u2*pu2 + pis1u2*(-1 + 4*pu2))),\n",
    " (1 + pis2u2 - 2*pu1 + pis2u1*(-1 + 4*pu1) + 2*pu2 - 4*pis2u2*pu2)/(2.*(2 + pis1u1 - pis2u1 + pis2u2 - 4*pis1u1*pu1 + 4*pis2u1*pu1 - 4*pis2u2*pu2 + pis1u2*(-1 + 4*pu2))),\n",
    "1.0/(2 + (2*(1 + pis2u2 - 2*pu1 + pis2u1*(-1 + 4*pu1) + 2*pu2 - 4*pis2u2*pu2))/(1 + pis1u1 + 2*pu1 - 4*pis1u1*pu1 - 2*pu2 + pis1u2*(-1 + 4*pu2))),\n",
    "1.0/(2 + (2*(1 + pis2u2 - 2*pu1 + pis2u1*(-1 + 4*pu1) + 2*pu2 - 4*pis2u2*pu2))/(1 + pis1u1 + 2*pu1 - 4*pis1u1*pu1 - 2*pu2 + pis1u2*(-1 + 4*pu2)))]\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}