TD Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:44:46.535117Z",
     "start_time": "2020-06-20T14:44:46.125007Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:44:46.646553Z",
     "start_time": "2020-06-20T14:44:46.538019Z"
    }
   },
   "outputs": [],
   "source": [
    "class WindyGridWorld:\n",
    "    \n",
    "    def __init__(self, n, terminal_state, windy_blocks, on_policy = 'on-policy', verbose=False):\n",
    "        \n",
    "        self.grid_size = n\n",
    "        self.terminal_state = terminal_state\n",
    "        self.windy_blocks = windy_blocks\n",
    "        self.on_policy = on_policy\n",
    "        self.verbose = verbose\n",
    "        self.gamma = 0.1\n",
    "        \n",
    "        self.Q = np.zeros((self.grid_size, self.grid_size, 4)) + 0.25\n",
    "        \n",
    "        self.actions = [(0,-1), (0, 1), (-1,0), (1, 0)]# Left, Right, Up, Down\n",
    "\n",
    "        #Equi-probable action selection at each state for all actions\n",
    "        self.policy = np.random.randint(1,4,size = (self.grid_size, self.grid_size))\n",
    "        \n",
    "        print(\"Initial State-Action/Q Function\", *self.Q, sep='\\n',end='\\n\\n')\n",
    "        print(\"Initial Policy\", *self.policy, sep='\\n',end='\\n\\n')\n",
    "        \n",
    "        if self.verbose:\n",
    "            print(\"Shape of Q\", self.Q.shape)\n",
    "            print(\"Shape of Policy\", self.policy.shape)\n",
    "        \n",
    "    def _get_reward(self, x, y):\n",
    "        if x==self.terminal_state[0] and y==self.terminal_state[1]:\n",
    "            return 3\n",
    "        return -1\n",
    "    \n",
    "    def _get_next_state(self, x, y, a):\n",
    "        \n",
    "        if x+a[0]!=-1 and x+a[0]!=self.grid_size:\n",
    "            x = x + a[0]\n",
    "        if y+a[1]!=-1 and y+a[1]!=self.grid_size:\n",
    "            y = y + a[1]\n",
    "        if x in self.windy_blocks[0] and y in self.windy_blocks[1]:\n",
    "            if x-1!=-1:\n",
    "                x-=1\n",
    "        return x,y\n",
    "        \n",
    "    def play(self, epochs = 10, threshold = 0.1):\n",
    "        \n",
    "        if self.on_policy == 'on-policy':\n",
    "            res = self._play_on_policy(epochs, threshold)\n",
    "            print(\"Final Policy:\")\n",
    "            print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            print(\"Final State-Action/Q Function:\")\n",
    "            print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            return res\n",
    "        \n",
    "        elif self.on_policy == 'off-policy':\n",
    "            res = self._play_off_policy(epochs, threshold)\n",
    "            print(\"Final Policy:\")\n",
    "            print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            print(\"Final State-Action/Q Function:\")\n",
    "            print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            return res\n",
    "        \n",
    "        elif self.on_policy == 'expected':\n",
    "            res = self._play_expected_sarsa(epochs, threshold)\n",
    "            print(\"Final Policy:\")\n",
    "            print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            print(\"Final State-Action/Q Function:\")\n",
    "            print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            return res\n",
    "        \n",
    "        elif self.on_policy == 'double':\n",
    "            res = self._play_double_policy(epochs, threshold)\n",
    "            print(\"Final Policy:\")\n",
    "            print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            print(\"Final State-Action/Q Function:\")\n",
    "            print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            return res\n",
    "        \n",
    "    def _get_action(self, x,y):\n",
    "        \n",
    "        m = np.round(np.min(self.Q[x][y])-1, 3)\n",
    "        probs = (np.round(self.Q[x][y], 5)-m) / np.sum(np.round(self.Q[x][y], 5) - m)\n",
    "        try:\n",
    "            \n",
    "            return np.random.choice([0,1,2,3],p=probs)\n",
    "        except:\n",
    "            print(m,probs)\n",
    "            \n",
    "    def _get_action_double(self, x,y):\n",
    "        \n",
    "        m = np.round(np.min(self.Q[x][y] + self.Q1[x][y])-1, 3)\n",
    "        probs = (np.round(self.Q[x][y] + self.Q1[x][y], 5)-m) / np.sum(np.round(self.Q[x][y] + self.Q1[x][y], 5) - m)\n",
    "        try:\n",
    "            \n",
    "            return np.random.choice([0,1,2,3],p=probs)\n",
    "        except:\n",
    "            print(m,probs)\n",
    "    \n",
    "    \n",
    "    def _play_on_policy(self, epochs, threshold):\n",
    "        \n",
    "        iters = 0\n",
    "        while True:\n",
    "            \n",
    "            iters+=1\n",
    "            x=0\n",
    "            y=0\n",
    "            diff = 0\n",
    "            \n",
    "            while True:\n",
    "                \n",
    "                    action = self._get_action(x,y)\n",
    "                    new_x, new_y = self._get_next_state(x,y,self.actions[action])\n",
    "                    reward = self._get_reward(new_x, new_y)\n",
    "\n",
    "                    if self.verbose:\n",
    "                        print([x,y,action,reward])\n",
    "                    \n",
    "                    self.Q[x,y,action] = self.Q[x,y,action] + 0.1*(reward + self.gamma*self.Q[new_x, new_y, action] - self.Q[x,y,action])\n",
    "                    \n",
    "                    self.policy[x,y] = np.argmax(self.Q[x][y])\n",
    "                    \n",
    "                    x, y = new_x, new_y\n",
    "                    \n",
    "                    if reward>0:\n",
    "                        break\n",
    "\n",
    "            if self.verbose:\n",
    "                print(\"Q function at iter:\", iters)\n",
    "                print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if self.verbose:\n",
    "                print(\"Policy at iter:\", iters)\n",
    "                print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if epochs==iters:\n",
    "                return diff, iters\n",
    "\n",
    "    def _play_off_policy(self, epochs, threshold):\n",
    "        \n",
    "        iters = 0\n",
    "        while True:\n",
    "            \n",
    "            iters+=1\n",
    "            x=0\n",
    "            y=0\n",
    "            diff = 0\n",
    "            \n",
    "            while True:\n",
    "                \n",
    "                    action = self._get_action(x,y)\n",
    "                    new_x, new_y = self._get_next_state(x,y,self.actions[action])\n",
    "                    reward = self._get_reward(new_x, new_y)\n",
    "\n",
    "                    if self.verbose:\n",
    "                        print([x,y,action,reward])\n",
    "                    \n",
    "                    self.Q[x,y,action] = self.Q[x,y,action] + 0.1*(reward + self.gamma * np.max(self.Q[new_x, new_y]) - self.Q[x,y,action])\n",
    "                    \n",
    "                    self.policy[x,y] = np.argmax(self.Q[x][y])\n",
    "                    \n",
    "                    x, y = new_x, new_y\n",
    "                    \n",
    "                    if reward>0:\n",
    "                        break\n",
    "\n",
    "            if self.verbose:\n",
    "                print(\"Q function at iter:\", iters)\n",
    "                print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if self.verbose:\n",
    "                print(\"Policy at iter:\", iters)\n",
    "                print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if epochs==iters:\n",
    "                return diff, iters\n",
    "            \n",
    "    def _play_expected_sarsa(self, epochs, threshold):\n",
    "        \n",
    "        iters = 0\n",
    "        while True:\n",
    "            \n",
    "            iters+=1\n",
    "            x=0\n",
    "            y=0\n",
    "            diff = 0\n",
    "            \n",
    "            while True:\n",
    "                \n",
    "                    action = self._get_action(x,y)\n",
    "                    new_x, new_y = self._get_next_state(x,y,self.actions[action])\n",
    "                    reward = self._get_reward(new_x, new_y)\n",
    "\n",
    "                    if self.verbose:\n",
    "                        print([x,y,action,reward])\n",
    "                    \n",
    "                    m = np.round(np.min(self.Q[x][y])-1, 3)\n",
    "                    probs = (np.round(self.Q[x][y], 5)-m) / np.sum(np.round(self.Q[x][y], 5) - m)\n",
    "                    self.Q[x,y,action] = self.Q[x,y,action] + 0.1*(reward + self.gamma * np.sum(self.Q[new_x, new_y] * probs) - self.Q[x,y,action])\n",
    "                    \n",
    "                    self.policy[x,y] = np.argmax(self.Q[x][y])\n",
    "                    \n",
    "                    x, y = new_x, new_y\n",
    "                    \n",
    "                    if reward>0:\n",
    "                        break\n",
    "\n",
    "            if self.verbose:\n",
    "                print(\"Q function at iter:\", iters)\n",
    "                print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if self.verbose:\n",
    "                print(\"Policy at iter:\", iters)\n",
    "                print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if epochs==iters:\n",
    "                return diff, iters\n",
    "            \n",
    "    def _play_double_policy(self, epochs, threshold):\n",
    "        \n",
    "        iters = 0\n",
    "        self.Q1 = np.zeros((self.grid_size, self.grid_size, 4)) + 0.25\n",
    "        while True:\n",
    "            \n",
    "            iters+=1\n",
    "            x=0\n",
    "            y=0\n",
    "            diff = 0\n",
    "            \n",
    "            while True:\n",
    "                \n",
    "                    action = self._get_action_double(x,y)\n",
    "                    new_x, new_y = self._get_next_state(x,y,self.actions[action])\n",
    "                    reward = self._get_reward(new_x, new_y)\n",
    "\n",
    "                    if self.verbose:\n",
    "                        print([x,y,action,reward])\n",
    "                    \n",
    "                    if np.random.binomial(1,0.5,1):\n",
    "                        self.Q[x,y,action] = self.Q[x,y,action] + 0.1*(reward + self.gamma * self.Q1[new_x, new_y, np.argmax(self.Q[new_x, new_y])] - self.Q[x,y,action])\n",
    "                    else:\n",
    "                        self.Q1[x,y,action] = self.Q1[x,y,action] + 0.1*(reward + self.gamma * self.Q[new_x, new_y, np.argmax(self.Q1[new_x, new_y])] - self.Q1[x,y,action])\n",
    "                        \n",
    "                    self.policy[x,y] = np.argmax(self.Q[x][y] + self.Q1[x][y])\n",
    "                    \n",
    "                    x, y = new_x, new_y\n",
    "                    \n",
    "                    if reward>0:\n",
    "                        break\n",
    "\n",
    "            if self.verbose:\n",
    "                print(\"Q function at iter:\", iters)\n",
    "                print(*self.Q, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if self.verbose:\n",
    "                print(\"Policy at iter:\", iters)\n",
    "                print(*self.policy, sep = '\\n', end = '\\n\\n')\n",
    "            \n",
    "            if epochs==iters:\n",
    "                return diff, iters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# On Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:45:16.279634Z",
     "start_time": "2020-06-20T14:44:46.649052Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial State-Action/Q Function\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "\n",
      "Initial Policy\n",
      "[2 2 3 3]\n",
      "[3 1 1 3]\n",
      "[3 1 3 3]\n",
      "[2 2 2 1]\n",
      "\n",
      "Final Policy:\n",
      "[0 0 0 3]\n",
      "[0 0 1 3]\n",
      "[1 1 1 2]\n",
      "[1 1 2 2]\n",
      "\n",
      "Final State-Action/Q Function:\n",
      "[[-1.11111111 -1.11111111 -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11111111 -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11111111 -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11111111 -1.11111111  3.025     ]]\n",
      "[[-1.11111111 -1.11111111 -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11111111 -1.11111111 -1.11111111]\n",
      " [-1.11111111  3.025      -1.11111111 -1.11111111]\n",
      " [ 0.25        0.25        0.25        0.25      ]]\n",
      "[[-1.11111111 -1.06975    -1.11111111 -1.11111111]\n",
      " [-1.11111111 -0.6975     -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11109011 -1.11111111 -1.11111111]\n",
      " [-1.11108399 -1.1109189   3.025      -1.10558908]]\n",
      "[[-1.11111111 -1.11111088 -1.11111111 -1.11111111]\n",
      " [-1.11111111 -1.11110885 -1.11111111 -1.11111111]\n",
      " [ 0.25        0.25        0.25        0.25      ]\n",
      " [-1.06398939 -1.05599216 -0.6965136  -1.08519129]]\n",
      "\n",
      "(0, 10000)\n"
     ]
    }
   ],
   "source": [
    "blocks = [[1,2,3],[2]]\n",
    "game = WindyGridWorld(4, (1,3), windy_blocks = blocks, on_policy = 'on-policy', verbose=False)\n",
    "print(game.play(epochs=10000, threshold=0.00000000000001))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Off-Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:45:46.575797Z",
     "start_time": "2020-06-20T14:45:16.282728Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial State-Action/Q Function\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "\n",
      "Initial Policy\n",
      "[2 3 1 2]\n",
      "[3 3 3 1]\n",
      "[3 3 1 1]\n",
      "[2 1 3 2]\n",
      "\n",
      "Final Policy:\n",
      "[1 1 1 3]\n",
      "[1 1 1 1]\n",
      "[1 1 1 2]\n",
      "[1 2 3 2]\n",
      "\n",
      "Final State-Action/Q Function:\n",
      "[[-1.1106975 -1.106975  -1.1106975 -1.1106975]\n",
      " [-1.1106975 -1.06975   -1.106975  -1.106975 ]\n",
      " [-1.106975  -0.6975    -1.06975   -1.06975  ]\n",
      " [-1.06975   -0.6975    -0.6975     3.025    ]]\n",
      "[[-1.1106975 -1.106975  -1.1106975 -1.106975 ]\n",
      " [-1.1106975 -1.06975   -1.106975  -1.06975  ]\n",
      " [-1.106975   3.025     -1.06975   -0.6975   ]\n",
      " [ 0.25       0.25       0.25       0.25     ]]\n",
      "[[-1.106975   -1.06975    -1.1106975  -1.1106975 ]\n",
      " [-1.106975   -0.6975     -1.106975   -1.106975  ]\n",
      " [-1.06975    -0.6975     -1.06975    -1.06975   ]\n",
      " [-0.69749964 -0.69749984  3.025      -1.06957767]]\n",
      "[[-1.1106975  -1.106975   -1.106975   -1.1106975 ]\n",
      " [-1.1106975  -1.06975    -1.06975    -1.106975  ]\n",
      " [ 0.25        0.25        0.25        0.25      ]\n",
      " [-1.06519888 -1.03637454 -0.6967627  -1.06240299]]\n",
      "\n",
      "(0, 10000)\n"
     ]
    }
   ],
   "source": [
    "blocks = [[1,2,3],[2]]\n",
    "game = WindyGridWorld(4, (1,3), windy_blocks = blocks, on_policy = 'off-policy', verbose=False)\n",
    "print(game.play(epochs=10000, threshold=0.00000000000001))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected SARSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:46:34.602979Z",
     "start_time": "2020-06-20T14:45:46.578033Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial State-Action/Q Function\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "\n",
      "Initial Policy\n",
      "[3 2 3 2]\n",
      "[3 1 1 2]\n",
      "[3 2 1 2]\n",
      "[1 2 2 3]\n",
      "\n",
      "Final Policy:\n",
      "[1 1 1 3]\n",
      "[1 1 1 2]\n",
      "[1 1 1 2]\n",
      "[1 2 2 2]\n",
      "\n",
      "Final State-Action/Q Function:\n",
      "[[-1.11110882 -1.11102985 -1.11110882 -1.11110518]\n",
      " [-1.11110881 -1.10809693 -1.1110297  -1.11095893]\n",
      " [-1.11102389 -0.99734621 -1.10787627 -1.10787627]\n",
      " [-1.10922278 -0.85786578 -0.85786578  3.025     ]]\n",
      "[[-1.11110518 -1.11095904 -1.11110882 -1.11103425]\n",
      " [-1.11110517 -1.10809883 -1.11102974 -1.10812804]\n",
      " [-1.11088125  3.025      -1.1040668  -0.85428082]\n",
      " [ 0.25        0.25        0.25        0.25      ]]\n",
      "[[-1.11103409 -1.1081259  -1.11110518 -1.11110505]\n",
      " [-1.11102787 -0.99236187 -1.11095475 -1.11095389]\n",
      " [-1.10790236 -1.00169331 -1.10788662 -1.10792918]\n",
      " [-1.04642982 -0.85482723  3.025      -1.10264078]]\n",
      "[[-1.11110506 -1.11095805 -1.11103425 -1.11110506]\n",
      " [-1.11110505 -1.10813142 -1.10812816 -1.11095785]\n",
      " [ 0.25        0.25        0.25        0.25      ]\n",
      " [-1.10617846 -1.10248901 -0.98486873 -1.10086134]]\n",
      "\n",
      "(0, 10000)\n"
     ]
    }
   ],
   "source": [
    "blocks = [[1,2,3],[2]]\n",
    "game = WindyGridWorld(4, (1,3), windy_blocks = blocks, on_policy = 'expected', verbose=False)\n",
    "print(game.play(epochs=10000, threshold=0.00000000000001))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Double Q-Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T14:47:01.966208Z",
     "start_time": "2020-06-20T14:46:34.604912Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial State-Action/Q Function\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "[[0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]\n",
      " [0.25 0.25 0.25 0.25]]\n",
      "\n",
      "Initial Policy\n",
      "[3 2 3 2]\n",
      "[1 1 3 2]\n",
      "[2 2 3 1]\n",
      "[3 3 1 2]\n",
      "\n",
      "Final Policy:\n",
      "[1 1 1 3]\n",
      "[1 1 1 2]\n",
      "[1 1 1 2]\n",
      "[1 2 1 2]\n",
      "\n",
      "Final State-Action/Q Function:\n",
      "[[-1.1106975 -1.106975  -1.1106975 -1.1106975]\n",
      " [-1.1106975 -1.06975   -1.106975  -1.106975 ]\n",
      " [-1.106975  -0.6975    -1.06975   -1.06975  ]\n",
      " [-1.06975   -0.6975    -0.6975     3.025    ]]\n",
      "[[-1.1106975  -1.106975   -1.1106975  -1.106975  ]\n",
      " [-1.1106975  -1.06975    -1.106975   -1.06975   ]\n",
      " [-1.10697487  3.025      -1.06974995 -0.6975    ]\n",
      " [ 0.25        0.25        0.25        0.25      ]]\n",
      "[[-1.106975   -1.06975    -1.1106975  -1.1106975 ]\n",
      " [-1.106975   -0.6975     -1.106975   -1.106975  ]\n",
      " [-1.06974999 -0.6975     -1.06974999 -1.06975   ]\n",
      " [-0.69222443 -0.68820705  3.025      -1.02339897]]\n",
      "[[-1.1106975  -1.106975   -1.106975   -1.1106975 ]\n",
      " [-1.1106975  -1.06975    -1.06975    -1.106975  ]\n",
      " [ 0.25        0.25        0.25        0.25      ]\n",
      " [-0.79546107 -0.88749907 -0.54124042 -0.91800273]]\n",
      "\n",
      "(0, 10000)\n"
     ]
    }
   ],
   "source": [
    "blocks = [[1,2,3],[2]]\n",
    "game = WindyGridWorld(4, (1,3), windy_blocks = blocks, on_policy = 'double', verbose=False)\n",
    "print(game.play(epochs=10000, threshold=0.00000000000001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}