Q_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNXb8kBTtgIlVZmOqycpNd+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ba6e007e5bb24afa99bd90f3bfaa160e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_591240c4bfe3482bbd7610b152fe000d",
              "IPY_MODEL_2af2646558304fdc841aded14f81a73b",
              "IPY_MODEL_2918af6f375241e5bdc746d1cea71172"
            ],
            "layout": "IPY_MODEL_833540cba7cb41e0ad2e426c9a1d5263"
          }
        },
        "591240c4bfe3482bbd7610b152fe000d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c79acd132727453f8062bbe4d369e914",
            "placeholder": "​",
            "style": "IPY_MODEL_eea1c5f195434387b153ebdab787d6aa",
            "value": "100%"
          }
        },
        "2af2646558304fdc841aded14f81a73b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ca494539ec54014a9fc3f3224947d66",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_56ec40ac081e4d3ba7ed3c921792b2de",
            "value": 100
          }
        },
        "2918af6f375241e5bdc746d1cea71172": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_23cf95541b8e42a4a767a2cb6dc3d3fd",
            "placeholder": "​",
            "style": "IPY_MODEL_b87e03faa74a486abd147b749be1a17e",
            "value": " 100/100 [00:00&lt;00:00, 1812.88it/s]"
          }
        },
        "833540cba7cb41e0ad2e426c9a1d5263": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c79acd132727453f8062bbe4d369e914": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eea1c5f195434387b153ebdab787d6aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ca494539ec54014a9fc3f3224947d66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56ec40ac081e4d3ba7ed3c921792b2de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "23cf95541b8e42a4a767a2cb6dc3d3fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b87e03faa74a486abd147b749be1a17e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aashay96/AngelList-Startups/blob/master/Q_learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "kgyn7vgn9KHZ"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install pyglet==1.5.1 \n",
        "!apt install python-opengl\n",
        "!apt install ffmpeg\n",
        "!apt install xvfb\n",
        "!pip3 install pyvirtualdisplay\n",
        "\n",
        "# Virtual display\n",
        "from pyvirtualdisplay import Display\n",
        "\n",
        "virtual_display = Display(visible=0, size=(1400, 900))\n",
        "virtual_display.start()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install gym==0.24 # We install the newest gym version for the Taxi-v3 \"rgb_array version\"\n",
        "!pip install pygame\n",
        "!pip install numpy\n",
        "\n",
        "!pip install huggingface_hub\n",
        "!pip install pickle5\n",
        "!pip install pyyaml==6.0 # avoid key error metadata\n",
        "!pip install imageio imageio_ffmpeg"
      ],
      "metadata": {
        "id": "eYmaemxIPgUU"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "import random\n",
        "import imageio\n",
        "import os\n",
        "\n",
        "import pickle5 as pickle\n",
        "from tqdm.notebook import tqdm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmYau4WQPssh",
        "outputId": "3fbe21cd-58f9-43bc-e369-34ec9060cfad"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Warning: Gym version v0.24.0 has a number of critical issues with `gym.make` such that the `reset` and `step` functions are called before returning the environment. It is recommend to downgrading to v0.23.1 or upgrading to v0.25.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env = gym.make('FrozenLake-v1', desc=None, map_name=\"4x4\", is_slippery=False)"
      ],
      "metadata": {
        "id": "kpPtKiKVPuCN"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "env.reset()\n",
        "print(\"_____OBSERVATION SPACE_____ \\n\")\n",
        "print(\"Observation Space\", env.observation_space)\n",
        "print(\"Sample observation\", env.observation_space.sample())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HeKRoaWiRNNM",
        "outputId": "327ad770-a4d1-43c5-f259-f9e931644709"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "_____OBSERVATION SPACE_____ \n",
            "\n",
            "Observation Space Discrete(16)\n",
            "Sample observation 15\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "state_space = env.observation_space.n\n",
        "print(\"There are \", state_space, \" possible states\")\n",
        "\n",
        "action_space = env.action_space.n\n",
        "print(\"There are \", action_space, \" possible actions\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B-hqE9YNRQSF",
        "outputId": "d276d8ba-90b3-40f0-b698-e3898ea7e00b"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are  16  possible states\n",
            "There are  4  possible actions\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def initialize_q_table(state_space, action_space):\n",
        "  Qtable = np.zeros((state_space,action_space))\n",
        "  return Qtable"
      ],
      "metadata": {
        "id": "Bf3V57j5RvBn"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Qtable_frozenlake = initialize_q_table(state_space, action_space)"
      ],
      "metadata": {
        "id": "6rUKR8z4SOOV"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def epsilon_greedy_policy(Qtable, state, epsilon):\n",
        "  # Randomly generate a number between 0 and 1\n",
        "  random_num = np.random.random()\n",
        "  # if random_num > greater than epsilon --> exploitation\n",
        "  if random_num > epsilon:\n",
        "    # Take the action with the highest value given a state\n",
        "    # np.argmax can be useful here\n",
        "    action = Qtable[state].argmax()\n",
        "  # else --> exploration\n",
        "  else:\n",
        "    action = env.action_space.sample() # Take a random action \n",
        "  \n",
        "  return action"
      ],
      "metadata": {
        "id": "vCbvkCO3SQVx"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def greedy_policy(Qtable, state):\n",
        "  # Exploitation: take the action with the highest state, action value\n",
        "  action = Qtable[state].argmax()\n",
        "  \n",
        "  return action"
      ],
      "metadata": {
        "id": "iinxfvzlY_14"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_training_episodes = 10000  # Total training episodes\n",
        "learning_rate = 0.7          # Learning rate\n",
        "\n",
        "# Evaluation parameters\n",
        "n_eval_episodes = 100        # Total number of test episodes\n",
        "\n",
        "# Environment parameters\n",
        "env_id = \"FrozenLake-v1\"     # Name of the environment\n",
        "max_steps = 99               # Max steps per episode\n",
        "gamma = 0.95                 # Discounting rate\n",
        "eval_seed = []               # The evaluation seed of the environment\n",
        "\n",
        "# Exploration parameters\n",
        "max_epsilon = 1.0             # Exploration probability at start\n",
        "min_epsilon = 0.05            # Minimum exploration probability \n",
        "decay_rate = 0.0005      "
      ],
      "metadata": {
        "id": "lAJgd_xQZewH"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(n_training_episodes, min_epsilon, max_epsilon, decay_rate, env, max_steps, Qtable):\n",
        "  for episode in range(n_training_episodes):\n",
        "    # Reduce epsilon (because we need less and less exploration)\n",
        "    epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*episode)\n",
        "    # Reset the environment\n",
        "    state = env.reset()\n",
        "    step = 0\n",
        "    done = False\n",
        "\n",
        "    # repeat\n",
        "    for step in range(max_steps):\n",
        "      # Choose the action At using epsilon greedy policy\n",
        "      action = epsilon_greedy_policy(Qtable,state, epsilon)\n",
        "\n",
        "      # Take action At and observe Rt+1 and St+1\n",
        "      # Take the action (a) and observe the outcome state(s') and reward (r)\n",
        "      new_state, reward, done, info = env.step(action)\n",
        "\n",
        "      # Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * max Q(s',a') - Q(s,a)]\n",
        "      Qtable[state][action] = Qtable[state][action] + learning_rate*(reward + gamma*np.max(Qtable[new_state]) - Qtable[state][action] )\n",
        "\n",
        "      # If done, finish the episode\n",
        "      if done:\n",
        "        break\n",
        "      \n",
        "      # Our state is the new state\n",
        "      state = new_state\n",
        "  return Qtable"
      ],
      "metadata": {
        "id": "RcljBH5Tb88-"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Qtable_frozenlake = train(n_training_episodes, min_epsilon, max_epsilon, decay_rate, env, max_steps, Qtable_frozenlake)"
      ],
      "metadata": {
        "id": "TMJdcsHbe4bK"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_agent(env, max_steps, n_eval_episodes, Q, seed):\n",
        "  \"\"\"\n",
        "  Evaluate the agent for ``n_eval_episodes`` episodes and returns average reward and std of reward.\n",
        "  :param env: The evaluation environment\n",
        "  :param n_eval_episodes: Number of episode to evaluate the agent\n",
        "  :param Q: The Q-table\n",
        "  :param seed: The evaluation seed array (for taxi-v3)\n",
        "  \"\"\"\n",
        "  episode_rewards = []\n",
        "  for episode in tqdm(range(n_eval_episodes)):\n",
        "    if seed:\n",
        "      state = env.reset(seed=seed[episode])\n",
        "    else:\n",
        "      state = env.reset()\n",
        "    step = 0\n",
        "    done = False\n",
        "    total_rewards_ep = 0\n",
        "    \n",
        "    for step in range(max_steps):\n",
        "      # Take the action (index) that have the maximum expected future reward given that state\n",
        "      action = np.argmax(Q[state][:])\n",
        "      new_state, reward, done, info = env.step(action)\n",
        "      total_rewards_ep += reward\n",
        "        \n",
        "      if done:\n",
        "        break\n",
        "      state = new_state\n",
        "    episode_rewards.append(total_rewards_ep)\n",
        "  mean_reward = np.mean(episode_rewards)\n",
        "  std_reward = np.std(episode_rewards)\n",
        "\n",
        "  return mean_reward, std_reward"
      ],
      "metadata": {
        "id": "_tVnjIW7fIUa"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kbnXIeAbf9lO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mean_reward, std_reward = evaluate_agent(env, max_steps, n_eval_episodes, Qtable_frozenlake, eval_seed)\n",
        "print(f\"Mean_reward={mean_reward:.2f} +/- {std_reward:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "ba6e007e5bb24afa99bd90f3bfaa160e",
            "591240c4bfe3482bbd7610b152fe000d",
            "2af2646558304fdc841aded14f81a73b",
            "2918af6f375241e5bdc746d1cea71172",
            "833540cba7cb41e0ad2e426c9a1d5263",
            "c79acd132727453f8062bbe4d369e914",
            "eea1c5f195434387b153ebdab787d6aa",
            "6ca494539ec54014a9fc3f3224947d66",
            "56ec40ac081e4d3ba7ed3c921792b2de",
            "23cf95541b8e42a4a767a2cb6dc3d3fd",
            "b87e03faa74a486abd147b749be1a17e"
          ]
        },
        "id": "7-AhJUQffMud",
        "outputId": "dd2fb872-cf76-458c-8727-d4b522877b45"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/100 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ba6e007e5bb24afa99bd90f3bfaa160e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean_reward=1.00 +/- 0.00\n"
          ]
        }
      ]
    }
  ]
}