Exp2 - SARSA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "103446b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|\u001b[34;1mR\u001b[0m: | : :G|\n",
      "| : | : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|Y|\u001b[43m \u001b[0m: |\u001b[35mB\u001b[0m: |\n",
      "+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import gym   \n",
    "import random\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "\n",
    "env = gym.make(\"Taxi-v3\")\n",
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "605886ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action Size:  6\n",
      "State Size:  500\n"
     ]
    }
   ],
   "source": [
    "action_size = env.action_space.n\n",
    "print(\"Action Size: \", action_size)\n",
    "\n",
    "state_size = env.observation_space.n\n",
    "print(\"State Size: \", state_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fdc2eee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sarsa_Agent():\n",
    "    \n",
    "    def __init__(self, gamma = 0.95, learning_rate = 0.10, epsilon = 0.2, nepisodes = 40000):\n",
    "        self.gamma = gamma\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epsilon = epsilon\n",
    "        self.nepisodes = nepisodes\n",
    "        self.Q = defaultdict(lambda: np.zeros(env.action_space.n)) \n",
    "\n",
    "    def greedy_policy(self, state):\n",
    "        return np.argmax(self.Q[state])\n",
    "\n",
    "    def epsilon_greedy_policy(self, state):\n",
    "        action = 0\n",
    "        if np.random.uniform() < self.epsilon:\n",
    "            action = np.random.choice(env.action_space.n)\n",
    "        else:\n",
    "            action = self.greedy_policy(state)\n",
    "        return action\n",
    "\n",
    "    def onpolicy_control(self, test_episodes):\n",
    "        \n",
    "        rewards = []\n",
    "        start = datetime.now()\n",
    "        for episode in range(self.nepisodes):\n",
    "            state = env.reset()\n",
    "            done = False\n",
    "            action = self.epsilon_greedy_policy(state)\n",
    "            while not done:\n",
    "                next_state, reward, done, info = env.step(action)\n",
    "                next_action = self.epsilon_greedy_policy(next_state)\n",
    "                self.Q[state][action] = self.Q[state][action] + self.learning_rate*(reward + self.gamma*self.Q[next_state][next_action] - self.Q[state][action])\n",
    "                state = next_state\n",
    "                action = next_action\n",
    "                \n",
    "            ep_avg, max_rew, min_rew = self.test_policy(test_episodes)\n",
    "            rewards.append(ep_avg)\n",
    "            \n",
    "            if episode % 100 == 0:\n",
    "                print(f'Episode: {episode} Average Reward: {ep_avg}')\n",
    "                print(f'Maximum Reward : {max_rew} Minimum Reward: {min_rew}')\n",
    "                \n",
    "            if ep_avg>9:\n",
    "                if ep_avg<9.5:\n",
    "                    print(f'Solved! Episode: {episode} Average Reward: {ep_avg}')\n",
    "                    \n",
    "                    fig = plt.figure()\n",
    "                    ax = fig.add_subplot(111)\n",
    "                    plt.plot(np.arange(len(rewards)), rewards)\n",
    "                    plt.ylabel('Average Reward')\n",
    "                    plt.xlabel('Episode #')\n",
    "                    plt.show()\n",
    "                    end = datetime.now()\n",
    "                    print(\"Solved in: \", (end-start))\n",
    "                    \n",
    "                    return self.Q\n",
    "                    \n",
    "                else:\n",
    "                    continue     \n",
    "        \n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111)\n",
    "        plt.plot(np.arange(len(rewards)), rewards)\n",
    "        plt.ylabel('Average Reward')\n",
    "        plt.xlabel('Episode #')\n",
    "        plt.show()\n",
    "        \n",
    "        end = datetime.now()\n",
    "        print(\"Not Solved\")\n",
    "        print((end-start))\n",
    "        return self.Q\n",
    "    \n",
    "    def test_policy(self, n):\n",
    "        \n",
    "        episode_rewards = []\n",
    "        max_rewards = []\n",
    "        min_rewards = []\n",
    "        \n",
    "        for episode in range(n):\n",
    "            state = env.reset()\n",
    "            done = False\n",
    "            cum_rewards = 0\n",
    "            while not done:\n",
    "                action = self.greedy_policy(state)\n",
    "                state, reward, done, info = env.step(action)\n",
    "                cum_rewards += reward\n",
    "                \n",
    "            episode_rewards.append(cum_rewards)\n",
    "        \n",
    "        max_rewards.append(max(episode_rewards))\n",
    "        min_rewards.append(min(episode_rewards))\n",
    "        \n",
    "                \n",
    "        return sum(episode_rewards)/n, max_rewards, min_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db9a19f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0 Average Reward: -209.0\n",
      "Maximum Reward : [-200] Minimum Reward: [-2000]\n",
      "Episode: 100 Average Reward: -200.0\n",
      "Maximum Reward : [-200] Minimum Reward: [-200]\n"
     ]
    }
   ],
   "source": [
    "a = Sarsa_Agent()\n",
    "Q = a.onpolicy_control(200)\n",
    "a.test_policy(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3557b8a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}