Q_Learning_Taxi_V3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "import random "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the environment\n",
    "# env = gym.make('Taxi-v3', render_mode=\"human\")\n",
    "env = gym.make('Taxi-v3', render_mode=\"rgb_array\")\n",
    "# env = gym.make(\"LunarLander-v2\", render_mode=\"human\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the Q-table to a zero matrix of size (state_space x action_space)\n",
    "ns = env.observation_space.n\n",
    "na = env.action_space.n\n",
    "q_table = np.zeros([ns, na], dtype=int)\n",
    "policy = np.full([ns, na], fill_value = 1 / na)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for Q-learning\n",
    "num_episodes = 10000\n",
    "max_steps_per_episode = 2000\n",
    "learning_rate = 0.2\n",
    "discount_rate = 0.99\n",
    "\n",
    "# Exploration parameters\n",
    "exploration_rate = 0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(314, {'prob': 1.0, 'action_mask': array([1, 1, 0, 0, 0, 0], dtype=int8)})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.metadata[\"render_fps\"]\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 1000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -4]\n",
      " [ 0  0  0  0  0  0]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0 10 -5 -3]]\n",
      "Episode: 2000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 1  0  1 10 -3 -3]]\n",
      "Episode: 3000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 2  0  3 10 -1 -3]]\n",
      "Episode: 4000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0 -1]]\n",
      "Episode: 5000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n",
      "Episode: 6000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n",
      "Episode: 7000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n",
      "Episode: 8000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n",
      "Episode: 9000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n",
      "Episode: 10000\n",
      "[[ 0  0  0  0  0  0]\n",
      " [ 0  0  0  0  0 -6]\n",
      " [ 0  0  0  0  0 -6]\n",
      " ...\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 0  0  0  0 -6 -6]\n",
      " [ 4  0  4 10  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "import time\n",
    "\n",
    "# The Q-learning algorithm\n",
    "rewards_all_episodes = []\n",
    "\n",
    "for episode in range(num_episodes):\n",
    "    state, prob = env.reset()\n",
    "    done = False\n",
    "    rewards_current_episode = 0\n",
    "    for step in range(max_steps_per_episode):\n",
    "        \n",
    "        # Exploration-exploitation trade-off\n",
    "        exploration_rate_threshold = random.uniform(0, 1)\n",
    "\n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            # Choose best Q\n",
    "            action = np.argmax(q_table[state,:])\n",
    "        else:\n",
    "            # Choose random action\n",
    "            action = env.action_space.sample()\n",
    "            \n",
    "        new_state, reward, terminated, info, done = env.step(action)\n",
    "            \n",
    "        # Update Q-table for Q(s,a)        \n",
    "        q_table[state, action] = q_table[state, action] * (1 - learning_rate) + \\\n",
    "            learning_rate * (reward + discount_rate * np.max(q_table[new_state, :]))\n",
    "        state = new_state\n",
    "        rewards_current_episode += reward\n",
    "        if terminated == True:\n",
    "            break\n",
    "        \n",
    "    # Print average reward every 1000 episodes    \n",
    "    if (episode + 1) % 1000 == 0:\n",
    "        print(f'Episode: {episode + 1}')\n",
    "        print(q_table)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0, -6],\n",
       "       [ 0,  0,  0,  0,  0, -6],\n",
       "       ...,\n",
       "       [ 0,  0,  0,  0, -6, -6],\n",
       "       [ 0,  0,  0,  0, -6, -6],\n",
       "       [ 4,  0,  4, 10,  0,  0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Play Time\n",
    "import time\n",
    "\n",
    "env = gym.make('Taxi-v3', render_mode=\"human\")\n",
    "state, prob = env.reset()\n",
    "done = False\n",
    "for step in range(100):\n",
    "    \n",
    "    action = np.argmax(q_table[state,:])    \n",
    "    new_state, reward, terminated, info, done = env.step(action)\n",
    "    state = new_state\n",
    "    \n",
    "    if terminated == True:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}