TensorFlow_with_GPU.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TensorFlow with GPU",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 2",
      "name": "python2"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/Joejiong/reinforcement-learning-playground/blob/master/TensorFlow_with_GPU.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "wCx_tUSHLgEL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Reinforcement Learning playground 1\n",
        "\n",
        "Hi here's the first reinforcement learning playground where YOU can make machine do some small game and feel like you are better than them. lol\n",
        "\n",
        "good to go!"
      ]
    },
    {
      "metadata": {
        "id": "Z2D_47PwCBLw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import time\n",
        "\n",
        "N_STATES = 6   # 1维世界的宽度\n",
        "ACTIONS = ['left', 'right']     # 探索者的可用动作\n",
        "EPSILON = 0.9   # 贪婪度 greedy\n",
        "ALPHA = 0.1     # 学习率\n",
        "GAMMA = 0.9    # 奖励递减值\n",
        "MAX_EPISODES = 13   # 最大回合数\n",
        "FRESH_TIME = 0.3    # 移动间隔时间"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "H2ZEexCECCKv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def build_q_table(n_states, actions):\n",
        "    table = pd.DataFrame(\n",
        "        np.zeros((n_states, len(actions))),     # q_table 全 0 初始\n",
        "        columns=actions,    # columns 对应的是行为名称\n",
        "    )\n",
        "    return table"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IH8AOdkDCI4l",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# on an arbitary state choose an action\n",
        "def choose_action(state, q_table):\n",
        "  state_actions = q_table.iloc[state, :]\n",
        "  if(np.random.uniform() > EPSILON) or (state_actions.all() == 0):\n",
        "    action_name = np.random.choice(ACTIONS)\n",
        "  else:\n",
        "    action_name = state_actions.argmax()\n",
        "  return action_name\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hDugE92TDiwU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_env_feedback(S,A):\n",
        "  if A == 'right':\n",
        "    if S == N_STATES-2:\n",
        "      S_ = 'terminal'\n",
        "      R = 1\n",
        "    else:\n",
        "      S_=S+1\n",
        "      R = 0\n",
        "  else:\n",
        "    R = 0\n",
        "    if S == 0:\n",
        "      S_ = S  # reach the wall\n",
        "    else:\n",
        "      S_ = S - 1\n",
        "  return S_, R"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QFTGPFxpETQs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def update_env(S, episode, step_counter):\n",
        "  env_list = ['-']*(N_STATES-1)+['T']\n",
        "  if S == 'terminal':\n",
        "    interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)\n",
        "    print('\\r{}'.format(interaction))\n",
        "    time.sleep(2)\n",
        "    print('\\r                                ')\n",
        "  else:\n",
        "    env_list[S] = 'o'\n",
        "    interaction = ''.join(env_list)\n",
        "    print('\\r{}'.format(interaction))\n",
        "    time.sleep(FRESH_TIME)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Abk-bVFvFjN0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def rl():\n",
        "  q_table = build_q_table(N_STATES, ACTIONS)\n",
        "  for episode in range(MAX_EPISODES):\n",
        "    step_counter = 0\n",
        "    S = 0\n",
        "    is_terminated = False\n",
        "    update_env(S, episode, step_counter)\n",
        "    while not is_terminated:\n",
        "      \n",
        "      A = choose_action(S, q_table)\n",
        "      S_, R = get_env_feedback(S,A)\n",
        "      q_predict = q_table.loc[S,A]\n",
        "      if S_ !=\"terminal\":\n",
        "        q_target = R + GAMMA*q_table.iloc[S_, :].max()\n",
        "      else:\n",
        "        q_target = R\n",
        "        is_terminated = True\n",
        "        \n",
        "      q_table.loc[S,A] += ALPHA*(q_target - q_predict)\n",
        "      S = S_\n",
        "      \n",
        "      update_env(S, episode, step_counter+1)\n",
        "      \n",
        "      step_counter += 1\n",
        "  return q_table"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zrhgSoL5GGGr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3114
        },
        "outputId": "5c3c0c40-af19-4e50-dd04-24db5f4fba82"
      },
      "cell_type": "code",
      "source": [
        "if __name__ == \"__main__\":\n",
        "  q_table = rl()\n",
        "  print('\\r\\nQ-table:\\n')\n",
        "  print(q_table)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 1: total_steps = 37\n",
            "                                \n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "---o-T\n",
            "--o--T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "---o-T\n",
            "----oT\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:6: FutureWarning: 'argmax' is deprecated. Use 'idxmax' instead. The behavior of 'argmax' will be corrected to return the positional maximum in the future. Use 'series.values.argmax' to get the position of the maximum now.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Episode 2: total_steps = 40\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 3: total_steps = 7\n",
            "                                \n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 4: total_steps = 11\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 5: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 6: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 7: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 8: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 9: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 10: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 11: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 12: total_steps = 5\n",
            "                                \n",
            "o----T\n",
            "-o---T\n",
            "o----T\n",
            "o----T\n",
            "-o---T\n",
            "--o--T\n",
            "---o-T\n",
            "----oT\n",
            "Episode 13: total_steps = 8\n",
            "                                \n",
            "\n",
            "Q-table:\n",
            "\n",
            "       left     right\n",
            "0  0.000507  0.007062\n",
            "1  0.000507  0.028651\n",
            "2  0.000045  0.114675\n",
            "3  0.000263  0.345619\n",
            "4  0.002268  0.745813\n",
            "5  0.000000  0.000000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7CQLVDTjIN6a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "0a533214-622d-421e-bbc8-97376b8a6cd2"
      },
      "cell_type": "code",
      "source": [
        "!echo \"# reinforcement-learning-playground\" >> README.md\n",
        "!git init\n",
        "!git add README.md\n",
        "!git commit -m \"first commit\"\n",
        "!git remote add origin https://github.com/Joejiong/reinforcement-learning-playground.git\n",
        "!git push -u origin master"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
           
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "EOhVDAA-OFsg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}