2.4 Incremental Implementation - Simple Bandit algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.4 Incremental Implementation - Simple Bandit algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "np.random.seed(77)\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 15\n",
    "plt.rcParams['ytick.labelsize'] = 15\n",
    "plt.rcParams[\"figure.figsize\"] = (11,4)\n",
    "\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = '10_armed_testbed'\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bandit(action):\n",
    "    return np.random.normal(q_rewards[action],1)\n",
    "\n",
    "def action_selection(t, q, epsilon = 0):\n",
    "\n",
    "    rand = np.random.rand(1)[0]\n",
    "\n",
    "    if t <= 0 or epsilon >= rand:            \n",
    "        return np.random.choice(NUM_OF_ACTION,1)[0]\n",
    "    else:\n",
    "        return argmax(q)\n",
    "    \n",
    "def argmax(array):\n",
    "    top_index = [0]\n",
    "    top = array[0]\n",
    "    \n",
    "    for i in range(1,len(array)):\n",
    "        if array[i] > top:\n",
    "            top_index = [i]\n",
    "            top = array[i]\n",
    "        elif array[i] == top:\n",
    "            top_index.append(i)\n",
    "        \n",
    "    np.random.shuffle(top_index)\n",
    "        \n",
    "    return top_index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## simple bandit algorithm\n",
    "\n",
    "STEP = 1000\n",
    "RUN = 2000\n",
    "NUM_OF_ACTION = 10\n",
    "\n",
    "q_rewards = []\n",
    "for _ in range(NUM_OF_ACTION):\n",
    "    tmp = np.random.normal(0,1)\n",
    "    q_rewards.append(tmp)\n",
    "    \n",
    "optimal_action = np.argmax(q_rewards)\n",
    "\n",
    "g_OPTIMAL_ACTIONS = [0]*STEP\n",
    "g_AVERAGE_REWARDS = [0]*STEP\n",
    "\n",
    "e01_OPTIMAL_ACTIONS = [0]*STEP\n",
    "e01_AVERAGE_REWARDS = [0]*STEP\n",
    "\n",
    "e001_OPTIMAL_ACTIONS = [0]*STEP\n",
    "e001_AVERAGE_REWARDS = [0]*STEP\n",
    "\n",
    "\n",
    "## 10-armed testbed with simple bandit algorithm\n",
    "for i in tqdm_notebook(range(RUN)):\n",
    "\n",
    "    # greedy\n",
    "    Q = [0]*NUM_OF_ACTION\n",
    "    N = [0]*NUM_OF_ACTION\n",
    "\n",
    "    for t in range(STEP):\n",
    "        A = action_selection(t, Q, 0)\n",
    "        R = bandit(A)\n",
    "        N[A] += 1\n",
    "        Q[A] += (1/N[A])*(R-Q[A])\n",
    "        \n",
    "        g_AVERAGE_REWARDS[t] += R/RUN\n",
    "        if A == optimal_action:\n",
    "            g_OPTIMAL_ACTIONS[t] += 1/RUN*100\n",
    "            \n",
    "    # ε = 0.1\n",
    "    Q = [0]*NUM_OF_ACTION\n",
    "    N = [0]*NUM_OF_ACTION\n",
    "\n",
    "    for t in range(STEP):\n",
    "        A = action_selection(t, Q, 0.1)\n",
    "        R = bandit(A)\n",
    "        N[A] += 1\n",
    "        Q[A] += (1/N[A])*(R-Q[A])\n",
    "        \n",
    "        e01_AVERAGE_REWARDS[t] += R/RUN\n",
    "        if A == optimal_action:\n",
    "            e01_OPTIMAL_ACTIONS[t] += 1/RUN*100\n",
    "            \n",
    "    # ε = 0.01\n",
    "    Q = [0]*NUM_OF_ACTION\n",
    "    N = [0]*NUM_OF_ACTION\n",
    "\n",
    "    for t in range(STEP):\n",
    "        A = action_selection(t, Q, 0.01)\n",
    "        R = bandit(A)\n",
    "        N[A] += 1\n",
    "        Q[A] += (1/N[A])*(R-Q[A])\n",
    "        \n",
    "        e001_AVERAGE_REWARDS[t] += R/RUN\n",
    "        if A == optimal_action:\n",
    "            e001_OPTIMAL_ACTIONS[t] += 1/RUN*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.arange(STEP), g_AVERAGE_REWARDS, 'g-', np.arange(STEP), e01_AVERAGE_REWARDS,'b-', np.arange(STEP), e001_AVERAGE_REWARDS,'r-')\n",
    "plt.legend([\"ε=0(greed)\", \"ε=0.1\", \"ε=0.01\"])\n",
    "save_fig(\"simple_vandit_average_reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.arange(STEP), g_OPTIMAL_ACTIONS, 'g-', np.arange(STEP), e01_OPTIMAL_ACTIONS,'b-', np.arange(STEP), e001_OPTIMAL_ACTIONS,'r-')\n",
    "plt.legend([\"ε=0(greed)\", \"ε=0.1\", \"ε=0.01\"])\n",
    "save_fig(\"simple_vandit_%Optimal_action\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}