agent_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "importing dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import annotations\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions.normal import Normal\n",
    "from stable_baselines3 import PPO, TD3\n",
    "import time\n",
    "import gymnasium as gym\n",
    "import pickle\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = (10, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Agent training, we choose to setup 2 different environments: \n",
    "- a default environment, with the rewards unchanged (with the exception of the healthy range, which we keep constant in all of our environments)\n",
    "- a changed environment, with our custom rewards in an attempt to see a difference in how the agent learns\n",
    "\n",
    "In each environment, we initialize 3 seeds and a different agent for every seed. The agent learns through a PPO algorithm using an MLP policy. \n",
    "For each seed, each different agent will learn for 2 million timesteps and then be tested on the environment for 1000 steps. Testing is always done in the default environment so we can compare the results. The goal is to compare the rewards the agents trained differently will receive when tested on the default environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default Env Agent Training and Testing Implementation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "training 3 agents on the default environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting seed 1\n",
      "Starting training\n",
      "Starting testing phase\n",
      "\n",
      "Starting seed 2\n",
      "Starting training\n",
      "Starting testing phase\n",
      "\n",
      "Starting seed 3\n",
      "Starting training\n",
      "Starting testing phase\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rewards_over_seeds = []\n",
    "\n",
    "for seed in [1,2,3]:\n",
    "    print(f\"Starting seed {seed}\")\n",
    "    steps_in_seed = 0\n",
    "    env_train = gym.make(\"Humanoid-v4\")\n",
    "    env_test = gym.make(\"Humanoid-v4\", healthy_z_range=(0.9,2.0))\n",
    "    # Record each episode reward\n",
    "    wrapped_env_train = gym.wrappers.RecordEpisodeStatistics(env_train)\n",
    "    wrapped_env_test = gym.wrappers.RecordEpisodeStatistics(env_test)\n",
    "\n",
    "    # Set seed\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    #initializing and training the agents\n",
    "    print(f\"Starting training\")\n",
    "    agent = PPO('MlpPolicy', wrapped_env_train, verbose=0, tensorboard_log=\"./logs_default\")\n",
    "    agent.learn(total_timesteps=2000000,progress_bar=True, tb_log_name=\"PPO_\"+str(seed), reset_num_timesteps=False) #training results are stored in tensorboards\n",
    "    agent.save(\"agent_default_\"+str(seed))\n",
    "    del agent\n",
    "    env_train.close()\n",
    "\n",
    "    print(f\"Starting testing phase\")\n",
    "    agent = PPO.load(\"agent_default_\"+str(seed), env = wrapped_env_test)\n",
    "    reward_over_episodes = []\n",
    "\n",
    "    #testing the agent\n",
    "    for episode in range(1000):\n",
    "        obs, info = wrapped_env_test.reset(seed=seed)\n",
    "        done = False\n",
    "\n",
    "        while not done:\n",
    "            action, _states = agent.predict(obs)\n",
    "            obs, reward, terminated, truncated, info = wrapped_env_test.step(action)\n",
    "            done = terminated or truncated\n",
    "        \n",
    "        reward_over_episodes.append(wrapped_env_test.return_queue[-1][0])\n",
    "    \n",
    "    rewards_over_seeds.append(reward_over_episodes)\n",
    "    env_test.close()\n",
    "    print()\n",
    "\n",
    "# Store the rewards for further analysis\n",
    "with open('rewards_default', 'wb') as f:  \n",
    "    pickle.dump(rewards_over_seeds, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Changed Agent Training and Testing Implementation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "training 3 agents on our custom environment, following the exact same structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting seed 1\n",
      "Starting 1st phase of training\n",
      "Starting 2nd phase of training\n",
      "Starting test phase\n",
      "\n",
      "Starting seed 2\n",
      "Starting 1st phase of training\n",
      "Starting 2nd phase of training\n",
      "Starting test phase\n",
      "\n",
      "Starting seed 3\n",
      "Starting 1st phase of training\n",
      "Starting 2nd phase of training\n",
      "Starting test phase\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rewards_over_seeds = []\n",
    "\n",
    "for seed in [1,2,3]:\n",
    "    print(f\"Starting seed {seed}\")\n",
    "    steps_in_seed = 0\n",
    "    env_train_1 = gym.make(\"Humanoid-v4\", healthy_reward = 10, forward_reward_weight = 0.625,  healthy_z_range=(0.9,2.0))\n",
    "    env_train_2 = gym.make(\"Humanoid-v4\", healthy_reward = 2.5, forward_reward_weight = 2.5, healthy_z_range=(0.9,2.0))\n",
    "    env_test = gym.make(\"Humanoid-v4\", healthy_z_range=(0.9,2.0))\n",
    "    # Record each episode reward\n",
    "    wrapped_env_train_1 = gym.wrappers.RecordEpisodeStatistics(env_train_1)\n",
    "    wrapped_env_train_2 = gym.wrappers.RecordEpisodeStatistics(env_train_2)\n",
    "    wrapped_env_test = gym.wrappers.RecordEpisodeStatistics(env_test)\n",
    "\n",
    "    # Set seed\n",
    "    torch.manual_seed(seed)\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    print(f\"Starting 1st phase of training\")\n",
    "    agent = PPO('MlpPolicy', wrapped_env_train_1, verbose=0, tensorboard_log=\"./logs_changed\")\n",
    "    agent.learn(total_timesteps=700000,progress_bar=True, tb_log_name=\"PPO_1_\"+str(seed), reset_num_timesteps=False)\n",
    "    agent.save(\"agent_changed_1_\"+str(seed))\n",
    "    del agent\n",
    "    env_train_1.close()\n",
    "    \n",
    "    print(f\"Starting 2nd phase of training\")\n",
    "    agent = PPO.load(\"agent_changed_1_\"+str(seed), env = wrapped_env_train_2)\n",
    "    agent.learn(total_timesteps=1300000,progress_bar=True, tb_log_name=\"PPO_2_\"+str(seed), reset_num_timesteps=False)\n",
    "    agent.save(\"agent_changed_2_\"+str(seed))\n",
    "    del agent\n",
    "    env_train_2.close()\n",
    "\n",
    "    print(f\"Starting testing phase\")\n",
    "    agent = PPO.load(\"agent_changed_2_\"+str(seed), env = wrapped_env_test)\n",
    "    reward_over_episodes = []\n",
    "\n",
    "\n",
    "    for episode in range(1000):\n",
    "        obs, info = wrapped_env_test.reset(seed=seed)\n",
    "        done = False\n",
    "\n",
    "        while not done:\n",
    "            action, _states = agent.predict(obs)\n",
    "            obs, reward, terminated, truncated, info = wrapped_env_test.step(action)\n",
    "            done = terminated or truncated\n",
    "            #wrapped_env_test.render()\n",
    "        \n",
    "        reward_over_episodes.append(wrapped_env_test.return_queue[-1][0])\n",
    "    \n",
    "    rewards_over_seeds.append(reward_over_episodes)\n",
    "    env_test.close()\n",
    "    print()\n",
    "\n",
    "# Store the rewards for further analysis\n",
    "with open('rewards_changed', 'wb') as f:  \n",
    "    pickle.dump(rewards_over_seeds, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ML2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}