Q_Learning_Chess_Game.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO24xAcPtIzcaLRfVYKrmBp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayudixit/ML_Training/blob/main/Q_Learning_Chess_Game.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 412
        },
        "id": "jTVoMX9sIVZF",
        "outputId": "413e90f7-0491-435b-f077-599d9c0b6b5b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: python-chess in /usr/local/lib/python3.10/dist-packages (1.999)\n",
            "Requirement already satisfied: chess<2,>=1 in /usr/local/lib/python3.10/dist-packages (from python-chess) (1.11.1)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-fa34979f6d28>\u001b[0m in \u001b[0;36m<cell line: 60>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mepisode\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m       \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Episode{episode}/{n_episodes}:Total Reward:{reward}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0mtrain_agent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtest_agent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m   \u001b[0mboard\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchess_Board\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-2-fa34979f6d28>\u001b[0m in \u001b[0;36mtrain_agent\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m   \u001b[0;34m\"\"\"Train the Q-learning agent by playing multiple Games\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mepisode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_episodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m     \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplay_game\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mepisode\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m       \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Episode{episode}/{n_episodes}:Total Reward:{reward}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-2-fa34979f6d28>\u001b[0m in \u001b[0;36mplay_game\u001b[0;34m()\u001b[0m\n\u001b[1;32m     42\u001b[0m   \u001b[0mdone\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m   \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m     \u001b[0mcurrent_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m     \u001b[0maction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchooose_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mboard\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/chess/__init__.py\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(self, stack)\u001b[0m\n\u001b[1;32m   3875\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3876\u001b[0m             \u001b[0mstack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3877\u001b[0;31m             \u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmove\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmove\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3878\u001b[0m             \u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/chess/__init__.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   3875\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3876\u001b[0m             \u001b[0mstack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3877\u001b[0;31m             \u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmove\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmove\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_stack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3878\u001b[0m             \u001b[0mboard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/copy.py\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_reconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/copy.py\u001b[0m in \u001b[0;36m_reconstruct\u001b[0;34m(x, memo, func, args, state, listiter, dictiter, deepcopy)\u001b[0m\n\u001b[1;32m    278\u001b[0m                 \u001b[0mslotstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m                 \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mslotstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mslotstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "from IPython import get_ipython\n",
        "from IPython.display import display\n",
        "!pip install python-chess\n",
        "import chess\n",
        "import random\n",
        "import numpy as np\n",
        "board=chess.Board()\n",
        "Q={}\n",
        "alpha=0.1\n",
        "gamma=0.5\n",
        "epsilon=0.1\n",
        "n_episodes = 1000\n",
        "def state_to_str(state):\n",
        "  return state.fen()\n",
        "def get_valid_moves(state):\n",
        "  return list(state.legal_moves)\n",
        "def update_q_value(state,action,reward,next_state):\n",
        "  current_q = Q.get((state_to_str(state),action),0)\n",
        "  max_future_q=max([Q.get((state_to_str(next_state),next_action),0)for next_action in get_valid_moves(next_state)])\n",
        "  Q[(state_to_str(state),action)]=current_q+alpha*(reward+gamma*max_future_q-current_q)\n",
        "def chooose_action(state):\n",
        "  if random.uniform(0,1)<epsilon:\n",
        "    return random.choice(get_valid_moves(state))\n",
        "  else:\n",
        "    valid_moves = get_valid_moves(state)\n",
        "    q_values = [Q.get((state_to_str(state),move),0) for move in valid_moves]\n",
        "    max_q = max(q_values)\n",
        "    best_moves = [valid_moves[i] for i,q in enumerate(q_values) if q==max_q]\n",
        "    return random.choice(best_moves)\n",
        "def get_reward(state,action,next_state):\n",
        "  if next_state.is_checkmate():\n",
        "    return 1\n",
        "  elif next_state.is_stalemate() or next_state.is_insufficient_material():\n",
        "    return 0.5\n",
        "  elif next_state.is_game_over():\n",
        "    return -1\n",
        "  else:\n",
        "    return 0\n",
        "def play_game():\n",
        "  board = chess.Board()\n",
        "  total_reward=0\n",
        "  done=False\n",
        "  while not done:\n",
        "    current_state=board.copy()\n",
        "    action=chooose_action(board)\n",
        "    board.push(action)\n",
        "  reward=get_reward(current_state,action,board)\n",
        "  total_reward+=reward\n",
        "  next_state=board.copy()\n",
        "  update_q_value(current_state,action,reward,next_state)\n",
        "  if board.is_game_over():\n",
        "    done=True\n",
        "  return total_reward\n",
        "def train_agent():\n",
        "  \"\"\"Train the Q-learning agent by playing multiple Games\"\"\"\n",
        "  for episode in range(n_episodes):\n",
        "    reward = play_game()\n",
        "    if episode % 100 == 0:\n",
        "      print(f\"Episode{episode}/{n_episodes}:Total Reward:{reward}\")\n",
        "train_agent()\n",
        "def test_agent():\n",
        "  board = chess_Board()\n",
        "  while not board.is_game_over():\n",
        "    if board.turn == chess.WHITE:\n",
        "      action = chooose_action(board)\n",
        "      print(\"Agent plays:\",action)\n",
        "    else:\n",
        "      action = random.choice(get_valid_moves(board))\n",
        "      print(\"Opponent plays:\",action)\n",
        "    board.push(action)\n",
        "    print(board)\n",
        "test_agent()"
      ]
    }
  ]
}