HW #4_ Monte Carlo On-Policy in Grid-Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br><br>\n",
    "\n",
    "<font color=#0C2C76 ><font>\n",
    "<p style = \"font-family:courier;\">\n",
    "    <font size=\"7\">\n",
    "        Interactive Learning\n",
    "    </font>    \n",
    "</p>\n",
    "\n",
    " <br>\n",
    "    \n",
    "<font color=#CF3A69><font>\n",
    "<p style = \"font-family:optima;\">\n",
    "    <font size=\"6\">\n",
    "        Homework #4 - Model Free Learning in MDPs\n",
    "    </font>    \n",
    "<font color=#000000> <font>\n",
    "</p>\n",
    "\n",
    "> $\\textit{Sahand Khoshdel - 810196607}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "MAP_LENGTH = 10\n",
    "MAP_WIDTH = 10\n",
    "MAX_EPOCHS = 5000\n",
    "\n",
    "nS = MAP_LENGTH*MAP_WIDTH\n",
    "nA = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pprint\n",
    "import io\n",
    "import numpy as np\n",
    "import sys\n",
    "from gym.envs.toy_text import discrete\n",
    "from colorama import Fore, Back, Style\n",
    "\n",
    "UP = 0\n",
    "RIGHT = 1\n",
    "DOWN = 2\n",
    "LEFT = 3\n",
    "\n",
    "class GridworldEnv(discrete.DiscreteEnv):\n",
    "  metadata = {'render.modes': ['human', 'ansi']}\n",
    "\n",
    "  def __init__(self, shape=[10,10]):\n",
    "    if not isinstance(shape, (list, tuple)) or not len(shape) == 2:\n",
    "        raise ValueError('shape argument must be a list/tuple of length 2')\n",
    "        \n",
    "    self.shape = shape\n",
    "\n",
    "    nS = np.prod(shape)\n",
    "    nA = 4\n",
    "\n",
    "    MAX_Y = shape[0]\n",
    "    MAX_X = shape[1]\n",
    "\n",
    "    P = {}\n",
    "    grid = np.arange(nS).reshape(shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      # P[s][a] = (prob, next_state, reward, is_done)\n",
    "      P[s] = {a : [] for a in range(nA)}\n",
    "\n",
    "      #Reaching target\n",
    "      is_done = lambda s: s == 0\n",
    "\n",
    "      #Due to existance of walls, these moves are not available\n",
    "      no_up = lambda s: s in [44,45,63,64]\n",
    "      no_down = lambda s: s in [23,24,25,44]\n",
    "      no_left = lambda s: s in [44,36,55]\n",
    "      no_right = lambda s: s in [32,42,52]\n",
    "\n",
    "      #Barries in the path\n",
    "      pit = lambda s: s in [13,62,41]\n",
    "      wall = lambda s: s in [43,33,53,34,35,54]\n",
    "\n",
    "      #Rewards of each state\n",
    "      reward = 1.0 if is_done(s) else 0.0\n",
    "      reward = -10.0 if pit(s) else reward\n",
    "      \n",
    "      ns_up = s if y == 0 else s - MAX_X\n",
    "      ns_right = s if x == (MAX_X - 1) else s + 1\n",
    "      ns_down = s if y == (MAX_Y - 1) else s + MAX_X\n",
    "      ns_left = s if x == 0 else s - 1\n",
    "      \n",
    "      P[s][UP] = [(1, ns_up, reward, is_done(ns_up))]\n",
    "      P[s][RIGHT] = [(1.0, ns_right, reward, is_done(ns_right))]\n",
    "      P[s][DOWN] = [(1, ns_down, reward, is_done(ns_down))]\n",
    "      P[s][LEFT] = [(1, ns_left, reward, is_done(ns_left))]\n",
    "\n",
    "      #Terminal state\n",
    "      if is_done(s):\n",
    "        P[s][UP] = [(1.0, s, reward, True)]\n",
    "        P[s][RIGHT] = [(1.0, s, reward, True)]\n",
    "        P[s][DOWN] = [(1.0, s, reward, True)]\n",
    "        P[s][LEFT] = [(1.0, s, reward, True)]\n",
    "\n",
    "      #Encountering walls\n",
    "      if no_up(s):\n",
    "        P[s][UP] = [(1.0, s, -3.0 , False)]                \n",
    "      if no_down(s):\n",
    "        P[s][DOWN] = [(1.0, s, -3.0 , False)]                \n",
    "      if no_right(s):\n",
    "        P[s][RIGHT] = [(1.0, s, -3.0 , False)]  \n",
    "      if no_left(s):\n",
    "        P[s][LEFT] = [(1.0, s, -3.0 , False)] \n",
    "\n",
    "      it.iternext() \n",
    "\n",
    "    # Initial state distribution is uniform\n",
    "    isd = np.zeros(100)\n",
    "    isd[44]=1\n",
    "    self.P = P\n",
    "\n",
    "    super(GridworldEnv, self).__init__(nS, nA, P, isd)\n",
    "\n",
    "  def _render(self, mode='human', close=False):\n",
    "    if close:\n",
    "      return\n",
    "\n",
    "    outfile = io.StringIO() if mode == 'ansi' else sys.stdout\n",
    "\n",
    "    grid = np.arange(self.nS).reshape(self.shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      if self.s == s and not s==0:\n",
    "        output =  Back.CYAN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL if s not in [13,62,41] else Back.BLUE + Style.BRIGHT + Fore.WHITE + \"  웃 \" + Style.RESET_ALL\n",
    "      elif s == 0:\n",
    "          output = Back.GREEN + Style.BRIGHT + Fore.WHITE + \"     \" + Style.RESET_ALL if not self.s==0 else Back.GREEN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL\n",
    "      elif s in [13,62,41]:\n",
    "          output = Back.BLUE + \"     \" + Style.RESET_ALL\n",
    "      elif s in [43,33,53,34,35,54]:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +   \"  █  \"  + Style.RESET_ALL\n",
    "      else:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +  \"  .  \" + Style.RESET_ALL\n",
    "\n",
    "      if x == 0:\n",
    "          output = output.lstrip()\n",
    "      if x == self.shape[1] - 1:\n",
    "          output = output.rstrip()\n",
    "\n",
    "      outfile.write(output)\n",
    "\n",
    "      if x == self.shape[1] - 1:\n",
    "          outfile.write(\"\\n\")\n",
    "\n",
    "      it.iternext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44\n",
      "\u001b[42m\u001b[1m\u001b[37m     \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[44m     \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[44m     \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[37m  웃 \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  █  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[44m     \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n",
      "\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\u001b[46m\u001b[1m\u001b[30m  .  \u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env =  GridworldEnv()\n",
    "print(env.reset())\n",
    "env._render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent Reached Terminal State 1000 times with an Average Regret of: 245.324\n"
     ]
    }
   ],
   "source": [
    "total_rewards = []\n",
    "actions_taken = []\n",
    "max_epoch = 1000\n",
    "for e in range(max_epoch):\n",
    "    rewards = []\n",
    "    env.reset()\n",
    "    s = init\n",
    "    while(True):\n",
    "        a = 0 \n",
    "        action = env.action_space.sample()\n",
    "        #print(f\"action chosen: {action}\")\n",
    "        state, reward, done, info = env.step(action) # Take a random action\n",
    "        a = a + 1 \n",
    "        #print(f\"S':{state}\")\n",
    "        #print(f\"Probability taking action:{info}\")\n",
    "        #print(f\"Reward:{reward}\\n\\n\\n\")\n",
    "        rewards.append(reward)\n",
    "        if(done==True):\n",
    "            total_reward = sum(rewards)\n",
    "            #print(f\"Agent Reached Goal with Total Reward = {total_reward}\")\n",
    "            total_rewards.append(total_reward)\n",
    "            break\n",
    "avg_reward = sum(total_rewards)/len(total_rewards)\n",
    "print(f\"Agent Reached Terminal State {max_epoch} times with an Average Regret of: {abs(avg_reward)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "def e_greedy(q_function_matrix,e = 0.3):\n",
    "    #print(f\"epsilon : {e}\")\n",
    "    policy_matrix = np.ones((nS,nA))/nA\n",
    "    \n",
    "    for s in range(nS):\n",
    "        for a in range(nA):\n",
    "            \n",
    "            policy_matrix[s][a] = (e/nA)\n",
    "            \n",
    "            opt_action = np.argmax(q_function_matrix[s])\n",
    "            \n",
    "            if(a==opt_action):\n",
    "                policy_matrix[s][a] =  policy_matrix[s][a] + (1-e)\n",
    "                \n",
    "            return policy_matrix\n",
    "        \n",
    "        \n",
    "def boltzman(q_function_matrix,T):\n",
    "    \n",
    "    policy_matrix = np.zeros((nS,nA))\n",
    "    \n",
    "    for s in range(nS):\n",
    "        for a in range(nA):\n",
    "            \n",
    "            policy_matrix[s][a] = (np.exp(q_function_matrix[s][a])/np.sum(np.exp(q_function_matrix[s])))\n",
    "            return policy_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "def next_step_generator(current_state,policy_matrix):\n",
    "    #print(policy_matrix[current_state])\n",
    "    action = np.random.choice(nA, 1, p = policy_matrix[current_state]) # Choosing action policy based\n",
    "    next_state, reward, done, info = env.step(action[0]) # Take chosen action, observe reward, next state\n",
    "   \n",
    "    return next_state,action,reward,done\n",
    "\n",
    "# yed for too main bayad bezairi !\n",
    "\n",
    "def monte_carlo_on_policy(current_state,policy_matrix,q_function_matrix, soft_policy = \"e-greedy\",e_value = 0.3, gamma = 0.9,emt=False):\n",
    "    states = []\n",
    "    rewards = []\n",
    "    actions = []\n",
    "    \n",
    "    returns = [[[] for a in range(nA)] for s in range(nS)] # create empty list for returns for each state-action pair \n",
    "    \n",
    "    for i in itertools.count():# iterating over states till an episode finishes\n",
    "\n",
    "        if(soft_policy == \"e-greedy\"):\n",
    "\n",
    "            policy_matrix = e_greedy(q_function_matrix,e_value) # policy construction\n",
    "            \n",
    "            \n",
    "        step_next_state, step_action, step_reward, done = next_step_generator(current_state, policy_matrix) # episode generation\n",
    "        if(done==True):\n",
    "            #print(\"Agent Reached Terminal State! Total Reward:\")\n",
    "            break\n",
    "        states.append(step_next_state)\n",
    "        rewards.append(step_reward)\n",
    "        actions.append(step_action)\n",
    "        \n",
    "        current_state = step_next_state\n",
    "        #print(current_state)\n",
    "\n",
    "    G = 0\n",
    "    states_reversed = states[::-1]\n",
    "    actions_reversed = actions[::-1]\n",
    "    rewards_reversed = rewards[::-1]\n",
    "\n",
    "    for i,state in enumerate(states_reversed):\n",
    "\n",
    "        action = actions_reversed[i][0]\n",
    "        reward = rewards_reversed[i]\n",
    "\n",
    "        G = G + (reward*gamma)\n",
    "        #print(f\"s: {state} ,states:{states}\\n\\n\")\n",
    "        if( (state not in states[:len(states)-i-1]) and ((action not in actions[:len(actions)-i-1]))):\n",
    "            returns[state][action].append(G)\n",
    "            \n",
    "            q_function_matrix[state][action]  = sum(returns[state][action])/len(returns[state][action])\n",
    "            if(emt==True):\n",
    "                print(f\"q function for state 4: {q_function_matrix[2]}\")\n",
    "            #print(q_function_matrix)\n",
    "            policy_matrix = e_greedy(q_function_matrix[state])\n",
    "            \n",
    "    sum_rewards = sum(rewards)\n",
    "    \n",
    "    return sum_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n",
      "q function for state 4: [0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "total_rewards = []\n",
    "epsilon = 1\n",
    "\n",
    "for e in range(MAX_EPOCHS):\n",
    "    em = False\n",
    "    init = env.reset()   \n",
    "    q_function_matrix = np.zeros((nS,nA))\n",
    "    if (e%2 == 0):\n",
    "        epsilon = max(epsilon * 0.99, 0.01)\n",
    "    if(e%100 == 0):\n",
    "        em = True\n",
    "    policy = e_greedy(q_function_matrix,epsilon)\n",
    "    total_reward = monte_carlo_on_policy(init,policy,q_function_matrix,e_value = epsilon,emt=em)  \n",
    "    total_rewards.append(total_reward)\n",
    "\n",
    "\n",
    "print(f\"Total Reward Average over {MAX_EPOCHS} epochs : {total_rewards[-1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f30fd7282e0>]"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2tUlEQVR4nO2deZgUxfn4P+/uct/HcoPLfSrXyqkoiIKioiYqilc8iFeiMYnBoCYxmpgYjxgT/WHU6Nc7GsWIiGC8FXSVGwRWQA4Rlvs+lq3fH9MzzMzOPd3TPTPv53nm2d7q7uq3uqvqrXqr6i0xxqAoiqIoAAVuC6AoiqJ4B1UKiqIoSgBVCoqiKEoAVQqKoihKAFUKiqIoSoAitwVIl+bNm5uSkhK3xVAURckqvvzyyy3GmOLw8KxXCiUlJZSVlbkthqIoSlYhIt9GClfzkaIoihJAlYKiKIoSQJWCoiiKEkCVgqIoihJAlYKiKIoSwHNKQUTGishyESkXkcluy6MoipJPeEopiEgh8HfgdKAXcJGI9HJXKkVRlPzBU0oBGASUG2NWGWMOAS8C412WKWGMMfznq/XsP3QEgIOVR/h32Tr87smPVBle/mIdlUeq3BTTM3xSvoU1W/bGvGbfoUqOVFV3777nYGXI/wcrj3CoUt+rHRypMuw7VBn/QiUn8ZpSaAusC/p/vRUWgohMEpEyESmrqKhwXKhYlY0xhuue/ZJPyrcwd/U2bnl5AXe9uQSAB95ZwS9fWci7yzYD8NIX67j11YU88fFq254fjR37DvHfBd+x5Lud7DpwmEXrdxJv74zl3+9m8YadST/r9tcXMW3+hqTvm/jPuZz8l/cBOHykiqqwyr+qytDrzpncMW1xSHjZmm30+c1M3l22KRDW/fa3GfLHd5OWIRV27j/MBY99xvrt+0LC31u+ma17Dtr6rMUbdrL8+928ufA7bvvPQip2H2RVxR5ufWVBSu88Ee6Ytphed86MqIxTYd7a7RyO0hAyxkQ9l2mOVBn+77M17A1qcCxav5P9h45EzJ/GmGplc/GGnVz6xFwOVVax68BhHpy1gp37D6cl14Yd+5mzamtacSSD15RCQhhjphpjSo0xpcXF1VZpp8Xeg5Vc80wZG3fu5+Uv1vHC52vpdvsMpn74DUP/+C4lk6eHtKIOVlYxY/H3TPznXB59/xsA1m/fD8DGnQd8cVrXb993CIA/zvg6rhxHqgz3zviaU+5/n263z6B8857AudVb9rLrQOyMdu2zX/KTF+Yx7uGPOe6373DWIx/zxMermbd2O7OWbuKWl+eHZPIF63Yw5qEPOfNvH/PQ7BWs27Yvatybdx3geyttAM/OWctNL86PKc/r8zZQMnl6oIA8/uGqkPNdp8xg0B9CK/UjlhJ7fu5aSiZPp2TydJZ/v5t5a3cA8Ok3oQVl295DMWWwizcXfsfna7ZxxVNf8Pbijcxauon9h47wo6e+4PKnPg+5dvPuA9z04jwWrt9RLZ6DlUdYtnEXf377a2Yu+R7wVTQPv7uStVt97//Mv33MmIc+5Mbn5/HC5+s4/p7ZjLr/A14uW89NL87nUGVVtcr7vwu+44MVsRtLf3+vnNVWL+2Zz9bwzGdrALh3xtc8P3ctAFUJbMD10xfmMfze/2GM4dZXFjBv7faQ88u/3825//iUP751NM8vXL8jkL/um7mcrlNmcODwkbjPCuf5uWv58lvf84wxTH51IV+s2Rbx2s++2Votfxw4fIQ/vLUsoAT+OnsFd0xbwoDfzwJg656DnPXIx/zilQV0nTKDG57/inXb9vHwuysxxvCvT9fQ7fYZTJu/gT+8tYzKI1VM/s9CPlq5heXf7+bnLy/gr++upO/v3gl57sL1OwJ5OZj9h46wclNoGMDIv7zPhKlzkn4/qeI1NxcbgPZB/7ezwhzjy2+3sX77fsb3a8uRKsOLX6xj1tJNzFq6KeS6PwRl6ic+Ws39s1aw9K4xzF19NBP6C+JHK7dw57TFvLHgO4BAhfnsnKOryi99Yi4frdzC89cMZljn5vzzo1XcPX0ZC35zGrv2H+aXryxgzqqjcY9+4AM++OXJHNOsHiP/8j49WjXgkYsHMPqBDwAQgdV/HBe4fs2W6pX66/M3cPf0ZYH/fzmmO6sq9jJv7Xb+8s6KQPhDs1fy0OyV/OtHxzPgmCbsPVhJxe6DrNy0hx8MbBeovNfcOy4k/ttfX8Td5xxb7bm3vDyf/3zl+4zrtu2jUdtG3PPWUTlKJk8HYMueg3y1djs9WjWgbs0ipFpMMOahDwPHh49U8cLna6lbszAQZoxh14FKGtWpAfgK2j/eL+f4kqbsPVjJ8R2bcu+Mr/m0fAsHKqt44ZohLN24k3+XrWdEN18D49qTOrPvUCVFBQWUrdlGcYNadG3ZAICPVlbw5oKNAJRv3sO1z34FwJ1n+oa+Fm/YFSLvoHt872ra/O84qVsxHZvX4+oTO3LCn96rlrbFvxvD9r2HeGDWCh6YtYJBHZtGeAOhdLt9Bi0a1OLzKaMBn7ntJy/MA+DK4R257Ywe1CgswBhD+eY9dG3ZgG17D3HfzOU8P3ctn0wexZ3TfD3by4aW8NgH3wTinrH4e376wjw+//UptGhYmw079rNkw05O690qcI0/j+/aX8nLZet5ff53/H58by4obY+IULHb13P6+nvfe/lwRQWXPelTnGvuHccLn/sU0N6Dlew+UElhge+rv/f1Zir2HKR5/VoUFkBx/dqc0LU5q7fsZaTVu/Tzq7E9KFuzjXe/3sy/v1zP/DtP5dx/fMpDF/ajT9tGVFUZLnp8Dj1bN2TGTSeyqmIP67bvZ3XFHqZ+uIoCESaf3oNFVi/5oNX632eZgacv3Bh4HzMW+5T38C7NeH1+aPmeGtTQmbt6a0gdUr55D7WKCpi55HsqrN7k2Y98TN2ahXx1x6lM/XAV//lqA8stpfD178dSu0Zhtd7IkSrDy2XrOLtvGwoLhNo1juZ9u/CaUvgC6CoiHfEpgwnAxU4+8AePfgbA+H5tuWf6Mp78JL5p5/5Zvgp0xaY9vGhl6nCe+SzUrUh4S/qjlVsAuPjxuTx39eCAwghvVQRz0n3vc0KX5gB8/f1uxj38UeCcMfDpN1uoPGIChS4cCatmjfGZcKLxi38voFZRIRt27A+E/WBgu6jXPztnLYvW76Rv+8acP7A9ew9V8s+PVjM7yMxz6EgVj/xvZdQ4zvvHpwD860fH09Cq2KMR/o4BXvh8Hb9+bZEvrgFtqVOjkOfmHv1Gwzo3C+lhBCsZf3hRgXD39GUc27ZRoKLwK8BLn4j8bu96c2ng+EiVCVRuwXywooIPVlTwXdD7DKbPb2by/DWDA/9/vjpyqzeczbuPmqyC5Xvyk9V0blGPiYOP4YmPV3P39GVcObwjbZvUAWB/nNb5Ty3lEt6D8/Of64cFjtdaLf9DlVX86tVF1K1ZxLrt++jZuiHge7dHqgxPBZWvj1aG9maOv2d2THnuGt87onnzT28fbbAdqTI8OGsl5Zv3cN6jn/LoxAHMtsy3yy3FNOp+X0PqF6d1A2DXgcM889ka3lueuCnaX29E476Zy0P+H/3ABxzTrC7fbt1HjUJf3jhYWcXByio27NhfzXqwY99hnvxkBUM7NQsJv/rpL3hveQW3/ceXx8MbZnbgKaVgjKkUkRuBmUAh8KQxZolTzwu2SQNMX/RdUvdPfHwOew8l3+2tFk+Mijmcj8u3BI4PhtkzL348djzhNvB4xoEte6qbY/ytej8rwrq7C9bvZMH6nRErbIBXvlwfME/E4oqnvoh7TST8CgEI9E6CCTc5RcLfm1oUVAGVTJ5Ov/aNE5Jh3trtlJZEb+W/E9YLDSbeN4zG6/M28PznR80pfqa8tpi+7RqzYL0vLcGNnm17D7ElaAzkd/9Nrqj5FTjAWY98HHLO31s5b8DRIcHt+w7xbZBZ8o7XF7N9n8+cOPDu2AoBCPRo4uFP46HKKq56OtRZ5r1Bla+/d5xIfkyW8LIJ8K1lEjx8JLTkHThc/dpDlVVM/XBVSO9j3bZ9SSmuVPGUUgAwxrwFvOX0c977enNIhpny2iI27UpukNAOhZBJ/AXQT7yB53iEK4hE2J9l7yyY+et2JHTdDx/7jE8mj6Jt4zrOChTEzS/Nj3ru1lcW0qm4XsRzpUGV8VOfrLFZqlDFbAysqjg622zN1ujjVk5QZQgxj8Xi5hfnURCht+cE4eMwABOmVu+JTHl9cbUwJ/CcUsgU4QX8OQdaC17nwxVb4l9kM6/Nc3SIyDPsPnAYyJxSiMXSjbtYunFX/AsdprLKG7OMEsE/XpAJIk0K+C5oIoefD+NMHrCLrJx9ZAdFGWoFeJlgU4tiL5+UZ24KYbbwz4+Sm4qdL7xpDWR7hbxVCtMWZK4loOQfv39zaUrmtVzm9TzpJWY7easUguf9K4riPFsztI5ESY+8VQqKoihKdVQpKIqiZCnpziCMhCoFRVGULOV9B2YkqVJQFEXJUg44sO5HlYKiKEqWYr/xSJWCoihK1uLAkIIqBUVRlGzFONBXUKWgKIqSpWhPQVEURQmgYwqKoihKAF2noCiKogRQ85GiKIoSQAeaFUVRlADaU1AURVECqFJQFEVRAujsI0VRFMVRHFMKInKfiHwtIgtF5DURaWyFl4jIfhGZb/0eC7pnoIgsEpFyEXlYRHTPTEVRlChk25TUWUAfY8xxwArgtqBz3xhj+lm/a4PCHwWuAbpav7EOyqcoiqKE4ZhSMMa8Y4yptP6dA7SLdb2ItAYaGmPmGJ/6ewY4xyn5GtYucipqRVGUjJDNYwpXAjOC/u8oIvNE5AMROdEKawusD7pmvRVWDRGZJCJlIlJWUWH/JhOKoij5SlrNZRGZDbSKcGqKMWaadc0UoBJ4zjq3EehgjNkqIgOB10WkdzLPNcZMBaYClJaWOqEsFUVR8pK0lIIxZnSs8yJyBXAmcIplEsIYcxA4aB1/KSLfAN2ADYSamNpZYYqiKEqGcHL20VjgVuBsY8y+oPBiESm0jjvhG1BeZYzZCOwSkSHWrKPLgGkOyudU1IqiKJnBATuJk6OtjwC1gFlWBTzHmmk0ArhLRA4DVcC1xpht1j3XA/8C6uAbg5gRHqmiKIriHI4pBWNMlyjhrwKvRjlXBvRxSiZFURQlNrqiWVEURQmgSkFRFEUJoEpBURQlS9H9FBRFURRHUaWgKIqSpeh+CoqiKIqj5K1S0LVriqIo1clfpeC2AIqiKB4kf5WCdhUURclystl1tucoLFCloCiKEk7eKoUiVQqKomQ5OvvIRi4e1MFtERRFUTxH3iqFkT1auC2CoiiK58hbpaAo4Qzo0NhtERTFdVQpKIrFL8Z0d1sERUkK9X2k5A1PXF6a8WcO69w8489UFK+hSkHxJKf0bOm2CHlJx+b1XH3+HWf2cvX5iioFV9B1c4pXcXumdp0ahe4KoKhScIMGtZzcGltRUsfNlf5v/uQE156drWTVOgUR+a2IbBCR+dbvjKBzt4lIuYgsF5ExQeFjrbByEZnslGyKt7l4sK4hyUf6tG2kvWgP4HRP4UFjTD/r9xaAiPQCJgC9gbHAP0SkUEQKgb8DpwO9gIusa3OS4V2a2RrfiV1zZ5C0UGsGRUmIXPF9NB540Rhz0BizGigHBlm/cmPMKmPMIeBF61pXcaKyNUDrRnVsjzdXqKfmNdcwTtgjlKzCaaVwo4gsFJEnRaSJFdYWWBd0zXorLFq4q4zt08r2OHcfqLQ9zmwsy3eN7x0x/ObRXTMsSW4z+fQeCV+r3oPjM+OmE90WIUCzejVtjzMtpSAis0VkcYTfeOBRoDPQD9gI3J++uIHnThKRMhEpq6iosCvayM/Kkp0XnFjE4jTN6tWKGF5bZ6DYyrUndXZbhJzCS3rz2LaNbI8zLaVgjBltjOkT4TfNGLPJGHPEGFMFPI7PPASwAWgfFE07KyxaeKTnTjXGlBpjSouLi9NJQlziZQCvzJgQhNN6Zdfc/oHHNIl/kaKkwHHt7K8s8wUnZx+1Dvr3XGCxdfwGMEFEaolIR6Ar8DnwBdBVRDqKSE18g9FvOCVfosQyy4zu2YI+bRtR3yM28Psv6Ou2CEnRqlFt1tw7zm0xFA9hVyO8S3F9m2LKP5wcU/iziCwSkYXASOBnAMaYJcDLwFLgbeAGq0dRCdwIzASWAS9b13oYD/UjgRqFubns5PqT1fyhJIm3imZW4VgT1xhzaYxz9wD3RAh/C3jLKZmyiQKBqiSGCbJxTCFROjSt67YIOU/jujXYse+w22LYRkGGDf9dW9Rn5eY9tsQ1uGNT5q7eZktcqZCbTUsbiZW3zi9tlzlB4mBM6gNgubTGwcuka+duUreGTZIodvPcNYNti+uc/olPunRC96lSSINhnX0L0FL5LnZPIW3VqLa9ETpAIhl4ooOrmYsbHJ3t9PSVgyJec94A52ZBR5sIkKhH2J6tG9opTgj+/KhWl9Swc1zx+BJ3J2CoUoiDW/P/k31sw9o1PD99NhHprj6xk2PPH3fs0bkPXVtEHohsWtf+ed9+oq0BUI+wR7Gr5et0SahZ5GzV6aa32rxVConaHJ0yTXpprnOmSFW/pvqubjoldBFctrtlzodNgLJlEebsn53E+QPjm48vH3pMSvEnurI8qxzieZ2erRs4/owbRkafNRPvYybysXuFmRMyrWjaNnbPVceDF8affjuiW+hYSWECfqGdfIfpupAY0CE/13XcPq5n0vc4XRY6NKvLab3jezv47dm9mX3LiKTjd1M35q1SsHU5fx62+uvUKKSo0L2Ej+quJpdcJLxYPnf1YC4ZknxrOxOm1Hq14q+8F5GMz4RKl7xVCokS63Omo1jszici0WV1YkpnIvKnsmFKpGgF4aVJQ0LCGtWtEXfhW6yGeTT5nbQVJ5Jf3N75LFfIRD08tJO9no6DcdOMpkohjJ+G2aET+TbR8l+sD2vHR080ind+NoLZt5yU8nNO6dGCkd2ruxOJV+6OaZa8MoqWpsFpFsBBJU1DnxPlQdef3IVrTuzIfT88jtE9W6T1zHDUA2l82rhokozFC9cMqRYmIgETaqyeSTLrjfwkuu5IxxQyQO82oXb6Pm3izy1vFGX+eCarAEGitkRr1yikQe3UpsxdNvQYnrjieIpSWC0dLs/xYRWz0wQ//r7zj0vonnq1ipgyrhfnl7ZnRDd7/WrZ7YH0TgcGzt22dJzY1Z53bnc6hnaO3Ch56cdDuPe8Y6lTM3qv+EgcrVC98STaU/AS4XmpVo34r+j5q6u3ItwgVjmIlsn6tm8cMwOOSWAwLfozQyMeFqVgOUXw47PFBUgyddlFg+xb0+HlXkxqomVGu7VrUpcJcb7D4SNVScYaPcGn9WpJz9YNaWq5zHbCk0F2lBSP075pXYoizGzJloooFVItcr9MZVpl2MMenTggxad7n2SKuNutei/jpXfTqE7sleiRlF40RdijdUNm3HRiYLGcmo9cING89X9XHV3mfrq1MY8TG2DEIlZBSLdFYVcZu2Fkl7Tj6Noy+enEma4kyu85vVpYpNb4f29MzPV6uFkzVVo2jLyHhR8PdxiSIvhzx0uz07RPeqKHuxpNlUIcEi0jwTbHhrV9LYNMVkTxnpVqYfdHG357Ivbx8Gu8vuI6nHSkjTQGU1gQGja4Y1OOTdAf0h/POzYNaeLjVV1gx8y5ujXtcUHRPYXGSCJEKkpRp7tmQGurUggj09sRpmMXDm55CrFld6PQh7eMveTJ1Q3zQvP6qfcc7TJFCsJPRnWxdTzCbt79+Um894uTWXDnabRvWjelfJPq93XjvUSq55+84viY9/jT50SJUqUQRnhecrruGNChscNPiE+0mRV2cJ2LeyHEKjCJNLhaNrTXyaCI8Nr1w7j/fPc2QzIYfn5a96g9Dy+o7c7F9enYvF7UWX2JkGqv9Gcx9gfPVKNGxDeAHfMaB5+vSiGMTC10TjWD1atZyFs/TX7j8FizS647qTNv3Dg8JXniUS9oql6qBdV/V6di/8KuzBROJ7YL7d+hCa0b26NsnLAkeHkWUqI0q1cz4zPd4lEriUWRyXwCJ76XKoUsw3DUBbRdezIXFEh0P0ZWjRy+0jeR6j3YVTVkfrBbQo6TV0hOV49emiGTKM3rJzZoa+c2q7WKCikqkIR9IN15Vi8KgmYDJvKaR/WIv1AxnTGxdBaPBuPPk06auVUphJHMu453aaxKJZ0MVtygFl/dcSo3j+52NL40B5qjZTK/nJcMTs7/TOtGtXl4Qv+k7kmURBtHJuTY2Sr+Aps2XLKz5de/Q2P+OqFfWPxxnh8nTjfMgYUFQvkfzkjKrXqypeuxSway4DenJXlX4iQyAykRh42ZQJVCGE71nsN33Uq3kmpar2ZIa8gpBnf0rUKu5hMozqPPH9iOJmFTcnN59tGff9i3WgXsNq9eO4zx/XybBvl7glGnBCeYHePNufcKwY2cBy/sF/f6mkUFtqXtp6PSm3adTMNUB5pdIJ1eWvCtraIMWtrVDYwXT9MU10ykq3jsSF+1qa0pROlFhRStAfLipCE8e5V92zsCNLQqvLjjJNk/pBDC6J4t6Nu+cUafmWye9++aWCMJr8OBqeIOfC/79pALQ0ReAvzLVxsDO4wx/USkBFgGLLfOzTHGXGvdMxD4F1AHeAu4yWR45MspU91lQ0t4Z+mmauGZSl69CNsFdmlRP2CnDU727eN6sv/QEdZu2xczzkxP3/Vjxytz054fT0EN6dSM9dtjv/uEnhP0mESTm2M6wdZK0ykT5GOXDOTDFRUs3biLqR+uokmM3f8C6XEw/zqmFIwxF/qPReR+YGfQ6W+MMf0i3PYocA0wF59SGAvMcErGSNjRooyUeU7o2jzClSnEbWO+DB788lcgDWsXObolZjqko4TCV7Um8h7trgLsLsd2VlLZrAyGdmrGZ6u2Bv4XOVqK46Xr4Yv608DG/ZVToWm9mpzTvy1n9W3DNSd2SrJXn4Wzj8RXki8AXohzXWugoTFmjtU7eAY4x2n53Mat1nY4ySpDIXIv52dBg992kW5vqvSYJo6+50h+r2Lh5KB3NHflsZJ/xbCSwHE2TkmN5AIk0c99dt82jExg5hEkXkZSzWqFBVJtxl50WZwjE2MKJwKbjDErg8I6isg8EflARPyT7tsC64OuWW+FVUNEJolImYiUVVRUOCN1hki2EEarULygWu774XExK7xUC8tZfdukKFFm8NuEkyVdPRWvkoqkCCNltzZB6yacUgntmji3T0KscS83lFwW6tUQ0lIKIjJbRBZH+I0PuuwiQnsJG4EOxpj+wC3A8yKSlLcvY8xUY0ypMaa0uDh9/+t1Y/hC90Z1mznSyc9nHNv66D8JVkiJUDuFHdyc5MQgU+DI7sX8+9qhtj8jkZ5NMj2ORBVQtlVofzm/L7UjrKFx0g2Em4R/c895STXGjDbG9InwmwYgIkXAecBLQfccNMZstY6/BL4BugEbgODJ3u2sMMdZetdYW+Pzt+AS2rXNLrNGutHEMcJGctDlFdNXpgn2iDvuuDa0bhTaCh7dM/VFhb0T2NQpEcJ7V4lWHlEXMaaJU1nl7L5tPFfxZ6JY+MteNk5JHQ18bYwJmIVEpFhECq3jTkBXYJUxZiOwS0SGWOMQlwHTHJbPEdxw/JbuHOualsO1nlFcNPdoFRoeVyFEqIXsKiyJvN3HLhnoWqs3ds8zNn9J0y+SP83Xp7jI7P4L+vLIxf0z7vY9VTLZLsn0FpkzboruziabxxQmUH2AeQSwUETmA68A1xpjtlnnrgf+CZTj60FkdOYREPNtJ5sBo11+xbASrjupM+2a1EloeX0iXJPmjKE6NQt5+cdDefyy0pTjMDi7HiCZmNs3zcxev5Fs1tHqhPD8E6nyiLWtYzgFMTJk+Kn4OtwnTIPaRZx5XJuI1z92SeTNjZLx62M3kZIlEtRbz2DDIHx/93Tp2Tq0Ida2cR0mHB/qxdVz5qN4GGOuMMY8Fhb2qjGmtzGmnzFmgDHmv0HnyizzU2djzI2ZXKPQPwFvpT8YEOrKILzgvHb9MD7+1cjA/9H2Nf7NWb3o2rIBH/9qFE3r1eSELs2rtS6jtRKivRE7XCsP6tg04R6HG5Yjv2/92kWJVZyJyvij4SVMvXRgqmIljR2K89mrBkcca4lb+afRiz2mWb2I4U/FcfPsNOFlQhBHmtJOzz6KxyeTRwXcZThZ/tydoOsyUy8dGHCP/PSVg1i7dR8Vew6GXJPMy+/fIXS1aLRJEeGml2evHsw/3i/nz28vD4SFtxKi8feLB9CxeeTCmmmMMWlVOj8+qRMlUaZUAjw4oR+flm+lQ4xrQuVJ7Lm/Oat3Wve7QbLrXuxQRNHKQvumdRnTuyUzl1RfnOnHqXcZz4wZ/NgLStvxctn6qNcqPvLazcVpvVsFlsA3rF2DPm3tGeQLZkxvezyZ+gl3CTzuuNb0smmrxlR55OL+TBrRKbBvLBCoQZJRqred3jNmIW9YuwZjra1Ok8HJVlUqg+2JKE6nRI5WOdexdieLl55oLqkz5Ubkicurmzcjvc+jbiB859bcO467xvcJueaGkfY69/NbG/pl0K2GE+OXed1TSITm9dLb3/WRiwew92ClLbLMvuUkx2aHpEOv1g0587j4awmapbHzWCr4e2pFBam1fRKp79OpCpPyyOtwnfvyj4fwztJNoYo9Am0a12Hcca2ZvnBjSHimJleE+24SoLIq9Nlpvas07h3ZvQVlt49O2L14Ojg5ZpLXPYV4rLzn9LR2fwKfrb9xDF8mfhL5uF1a1E9qINJJgstORNHDEtS2cR0uOj5zWx0aAwM6NGHSiE48cKF7O53ZRcLuwlOoJAShU3F9rj0ptZZzQsrTQaV2uDKxRFcbfI+gAYrr1+LHJ0WetJGI4suEQgBn36cqhRjYtS9uurx63bCoMz/cJOLMjyhNrbP6tom58tQJL+AFBcKvz+hZbR2B00RLih09gzvP7MUzVw5KQIbEPctGquz+OqF/wG26n2izndwcexGB0b1CZ/D5Fq+ltvhPRLjlVPtdtWQTaj7KApzYFjJdohW6VMwIc247Je1pjV1b1Gfl5j2B/+1oSbnhNC8eV57QMeb5+LOPEmN4l+YM79KcksnTAfjlmO50bVEfgImDO1QzHyVC/VpF7LHJlBrMsM5HB93P7d+WUT1bMHeVb5a7nQorm9yvp4M3msJZwpXDO7ruUdELtG1ch7vP6RO7ggmrnWIpi1aNalfbkCdbSGfWdCZb2KlWZ6/fMJzfntWLG0Z2CTQEhnVuHrLdpqtuyMMe/uCF/ahVVBjk5iL5l+zFyj+TqFIII1Z26NWmIYt+N6b6LmR5xieTR4X4OYr1zrxewFqn6MwuFvGqITffSbJKrF/7xlwxPHbvJFEymep0BmKzqXw7McCfPanPIlIa7PN23RkTD0/nTwunvklKrde0fVvFWP1sxxqGCHG8/4uT+f05fSJc7TzZXJ4SIeD7SM1H2YXXWskf3TqSSSPs20AnkdT5/ft38sgCu2RIpMCl4xQw1q2p5h0vLbgraV6Pk7oe9WLshmheeh92ks2+j3KaeAXXDcd4sWjftG5gBbfjWKXxjGNb8+p1Q7mgtH1mnpthkimcyVT0yezXG/FZSdxuRz7NVF5P2NWE9ddrZdAuzuzrM986UZ5VKSTA45eV8uKkIY4+w64Wzee/PoXzB7aLev7SIcfY86AkGHhM04y72c72FmKz+rVCpiHblR43X0tGc4B/oDlGgt3MI+f0S2/jqOtO6syS341JeKe2ZFClkACn9mrJkE6Rl/fHwg3zUYuGtbkvhvtlOwfR/CuUa0RaMZwlRt3ETEQw99enOC9MGGP7tI5/URg3juoC+FYeB+OFr2F3HdynbXT3LsnsaeIW/do35q8T+qV0r4hQz6GZkDq/UkmZf0wcyOxlmxJ2UJetFNevFbWbHqnScbMCHt+vTUxTXSQlmG7jJdFeoJ3vpU6NQt78SYz9BiI8LJl0nty9mPeXO7vV7+s3DHc0/lTRnkIYdpg5vNw6sZPiBrW4aFDmXFc4QazP3bx+Ta4/uTPPXT046jVeW6cQbdWxkx23VN7BeQMibr9uPym+4ycuP57ld9u7I2MwXt61UJWCk3j3u+c80crcZUOP4aYEN0MREW4d24OuLRsk9WwnGgXR0nPz6KNpqVezMKprFv8eFE64E4HEWuHB7+WBC/rZvilNqDz+Z6b2NQoLhFoJ7tuRa6j5SLEVv0Ow4gx7RE0Uv/vk73bstyU+p6akJsrNo7txaq+WjHv4YzpE2QQH4KEJ/Xh2zrcR3TqnK4cXW72JzOPPlx59sqhScIB8W7wWzMWDOtCwTg3OPDb5QdJsJKkpqU4thksgv7VsWJufn9Y94rlLHJyRFqul7qTb8aNuLpRkUfORR8iGKZT3n9+Xt34afXAPfJ5Jz47jEdUJjrc8enppw/mv7jjVtrgSmiWVYtyRtvVMlnhmmkzPxAvfZCfWNW6QwZ2Gk0aVQgrEy0y50uoP5wcD27m+y1s0fntWb2bfMoIW1iwhJ/cfSJSmHlJQbpPpRWSFVqMk2j7poL2IaKStFETkfBFZIiJVIlIadu42ESkXkeUiMiYofKwVVi4ik4PCO4rIXCv8JRHxZKmKl5m81AiYODi7ZwclSs2iArq0aKBj+x4jVg/BycZT33aNue7kzimvA8hn7OgpLAbOAz4MDhSRXsAEoDcwFviHiBSKSCHwd+B0oBdwkXUtwJ+AB40xXYDtwFU2yOcYce2amREjJh7ST54k3YrJS+sU3PrWiabXzvcSr+FVUCD8amyPjG+wlCjJDs7/7aL+DklSnbSVgjFmmTFmeYRT44EXjTEHjTGrgXJgkPUrN8asMsYcAl4ExovvLY0CXrHufxo4J135ksULFbmSPWRqnYKXTZKGxMYMiix/Tr0tE2RKG9y7+B4uPN63KLBViv6Gzjwu9ckXZ/VNzy1GMjg5ptAWWBf0/3orLFp4M2CHMaYyLLwaIjJJRMpEpKyiwtlVh6ngJSdcHq5LHMELb97rLXa38LtYeeRin0+nk7u34PMpqbkPcUNJXnlCR9bcOy6hPdcj8cjFA3jgAu/vF57QlFQRmQ20inBqijFmmr0ixccYMxWYClBaWuqFeiAiXpy/nS9k6tU7vU6he8sGLN+0O+VnxGLi4A5s2nUw7XhiJSO4geTvGdUJmu3UokGGvPZS/X17aezPSySkFIwxo1OIewMQ7ISlnRVGlPCtQGMRKbJ6C8HX5zyqP7KTZD5b1xbJrYx2mnvOPdaWeOrWPFrJn9i1ORMHV1/3EGxe0rzubZw0H70BTBCRWiLSEegKfA58AXS1ZhrVxDcY/YbxGWffA35o3X85kPFeiFtoqyW3+fL20WlN5/Vy/gg2p1w8qANj+0QyKhzFy2mxk44xNpby8jqFtFc0i8i5wN+AYmC6iMw3xowxxiwRkZeBpUAlcIMx5oh1z43ATKAQeNIYs8SK7lfAiyJyNzAPeCJd+ZT8xGtlrll9+/3eewkvjaN5gdm3nERxln7ztJWCMeY14LUo5+4B7okQ/hbwVoTwVfhmJylKSuSqZSKWyaVj83oUSKhzPK8RrDTywXzUpUX9mOe9PN6ovo9SIO7n9FCj6ZZTu7ktQk7jhU9dr1YRq/44zlUZok1JDQ638115t0rNflQpOIibGXfaDcPZc7Ay580WbhPpG8f77m2sBVXDOje3XZ5sQCt0b6NKIQW80DqMR99UFgYpCdOrdUOWbtxF/drVi1C8/FHSvB4f/2pkQDnkMl4ea/CybG6iDvHC0GyS3WTq+/n3QE611duuSd2Me5J1E0G4YlgJAA1q13BXGCUm2lMIw46pYqpY3MfD43h5yw0ju3DDyC62xGVHGcu0O+9sQXsKSlYzaUQnRnYvTvq+hnV8rdWM7RWcIsO7+MYdmqToWiHXSEfZh9/qhvmokZXvWjTw7lif9hQcQNsfmePXZ/SMGB6vw1e/VhHL7hpLrSJvt4t+fUYPfjS8hJYpOmHLFOf0b8vbS76nT9tGGXletpaxUT1a8OCFfTnDwzsTqlIIo6gg/UpCzUfukUxlUaem9zdmLyosoH3Tum6LEZexfVqx5l53p8UmS0MXxjZEhHP7t8v4c5PB280kF/CP/Q3t1CztuNSurSg+vDjT55oTO7ktgifRnkIKaF2vKImRyGDuf288IeBWO1NcMqRDxp+ZLahScAAvO7tSlEySSA/h2HaZGYdQEkOVgoN41Xw0+5YRaH8nMzSuW4Md+w67LYbr6PTP1HjmykEUZ3imkioFj5BJBdLFY379c5n//fxkduw75LYYOYeXHcrZyYhuyU+3ThdVCinghHHoimEl/PntSFtdK8ngNcNd03o1aVovN9cYvDRpSMZbsYrz6EhLGG5VKnVrFlF+z+kuPV1Rkmdwp2Z0Ko7tIlrJPlQpRMGN3mlRoX6OdMkPo4KSCsEmJ50LEh2thRxA85uiOIvO8HMOVQoOojMucpfx/doA0LN16vsu5wP1a/mGLY8vaWpLfFqinEcHmqPgVkPkhWuG8M7S7915uJIwZ/Vtw1l929gW339vPIFdB3Jv6mqz+rV452cjOKaZva468mX2kRuoUggjkawW75p0FMrQzs0Y2jl9Fxte5o0bhzNn1Va3xfAUubyAq1tL702BPvM4+xR6rpGW+UhEzheRJSJSJSKlQeGnisiXIrLI+jsq6Nz7IrJcROZbvxZWeC0ReUlEykVkroiUpCObF9DGTGSOa9eYSSM6uy2GksfkesMrHdLtKSwGzgP+X1j4FuAsY8x3ItIHmAkEO66faIwpC7vnKmC7MaaLiEwA/gRcmKZ8rqJjYYqiZBtp9RSMMcuMMdVWXBlj5hljvrP+XQLUEZF4q1zGA09bx68Ap4gaDpUk8Q8At2rk7f0HFMWrZGL20Q+Ar4wxB4PCnrJMR3cEVfxtgXUAxphKYCcQsY8nIpNEpExEyioqKpyUPS1UpWWeSSM68fXvx9K8vq60zUW08+08cc1HIjIbaBXh1BRjzLQ49/bGZwY6LSh4ojFmg4g0AF4FLgWeSVxkMMZMBaYClJaWZjyfaMb0LiJC7Rre3zxHSY9Mtbd+P7533rnyiKsUjDGjU4lYRNoBrwGXGWO+CYpvg/V3t4g8DwzCpxQ2AO2B9SJSBDQCMj5FxY4KX5WGoniPVBTJpUNL7BbD8zhiPhKRxsB0YLIx5pOg8CIRaW4d1wDOxDdYDfAGcLl1/EPgf8bFZYtq+lGSpaG1KXuxmq6ULCat2Ucici7wN6AYmC4i840xY4AbgS7AnSJyp3X5acBeYKalEAqB2cDj1vkngP8TkXJgGzAhHdkUJdOc3K2Y+8/vy7jjvLspu6LEIy2lYIx5DZ+JKDz8buDuKLcNjBLXAeD8dOTJFPE6ETUKfVcUaHcjLZ6/ejAfl29xW4yEERF+MNDbm7LnCmqidQ5d0ewAd4zrRbN6NRnbJ9L4vJIow7o0Z1iX5m6LoSh5hSoFB2hSryZTxvVyWwxFyVm0D+4c6iU1CroaWVGUfESVQhjaAlEUJZ9RpaAoiqIE0DEFRUmQf/3oeFZv2eu2GIriKKoUUkCHG/KTk7u34OTubkuR36Qz1qczxBNDzUdhaIWvKFmAVvCOoUohCtqqUBQlH1GlkAKqLxRFyVVUKSiKoigBVCkoipJ96OCfY6hSUBRFUQKoUlAUJfvQgT3HUKWgKIqiBFClEAV1iKcoSj6iSiEM7ZUqSm4iuvgoIVQppMCY3r7NcwoLNJMpipJbqO+jMBKxGv3l/L5MGdeTGoWqUxUlo6hZ13HSqtVE5HwRWSIiVSJSGhReIiL7RWS+9Xss6NxAEVkkIuUi8rBYfToRaSois0RkpfW3STqypUusnmbNogJaNqydOWEURQlB++jOkW5TdzFwHvBhhHPfGGP6Wb9rg8IfBa4Bulq/sVb4ZOBdY0xX4F3rf0VRFCWDpKUUjDHLjDHLE71eRFoDDY0xc4wxBngGOMc6PR542jp+OihcURRFyRBOGsU7isg8EflARE60wtoC64OuWW+FAbQ0xmy0jr8HWkaLWEQmiUiZiJRVVFTYLriiKEq+EnegWURmA60inJpijJkW5baNQAdjzFYRGQi8LiK9ExXKGGNEJOqQkjFmKjAVoLS0VIeeFCXP0ELvHHGVgjFmdLKRGmMOAget4y9F5BugG7ABaBd0aTsrDGCTiLQ2xmy0zEybk32uoiiKkh6OmI9EpFhECq3jTvgGlFdZ5qFdIjLEmnV0GeDvbbwBXG4dXx4UriiKEoLOPnKOtNYpiMi5wN+AYmC6iMw3xowBRgB3ichhoAq41hizzbrteuBfQB1ghvUDuBd4WUSuAr4FLkhHNkVRlHBuGNk5sPhUiUxaSsEY8xrwWoTwV4FXo9xTBvSJEL4VOCUdeexEfR8pSu7xyzE93BbB8+iS3DC0W6ooSj6jSiEM7SAoipLPqFKIgjpUVBTvYbTZ5jiqFBRFyTrUDbZzqFJQFEVRAqjr7Azyu7N707ZxHbfFUBRFiYoqhQxy+bASt0VQFEWJiZqPFEVRlACqFBRFUZQAqhQURVGUAKoUFEXJCVo2rOW2CDmBDjQripL1zLx5BMUNVCnYgSqFMPp3aEyPVg2YfLo6zlKUbKF7qwZui5AzqFIIo27NIt6+eYTbYiiKEgH1Xuw8OqagKErWoV4unEOVgqIoihJAlYKiKIoSQMcUPMwlQzqw7+ARt8VQFCWPUKXgYe4+51i3RVAUJc9Q85GiKFmDf4C5dlGhu4LkMGkpBRE5X0SWiEiViJQGhU8UkflBvyoR6Wede19Elgeda2GF1xKRl0SkXETmikhJOrIpipJ7NK5bk1vHdueFSUPcFiVnSdd8tBg4D/h/wYHGmOeA5wBE5FjgdWPM/KBLJhpjysLiugrYbozpIiITgD8BF6Ypn6IoOcb1J3dxW4ScJq2egjFmmTFmeZzLLgJeTCC68cDT1vErwCmie+4piqJklEyMKVwIvBAW9pRlOrojqOJvC6wDMMZUAjuBZpEiFJFJIlImImUVFRVOya0oipJ3xFUKIjJbRBZH+I1P4N7BwD5jzOKg4InGmGOBE63fpckKbYyZaowpNcaUFhcXJ3u7oiiKEoW4YwrGmNFpxD+BsF6CMWaD9Xe3iDwPDAKeATYA7YH1IlIENAK2pvFsRVEUJUkcMx+JSAFwAUHjCSJSJCLNreMawJn4BqsB3gAut45/CPzPGHV/pSiKkknSmn0kIucCfwOKgekiMt8YM8Y6PQJYZ4xZFXRLLWCmpRAKgdnA49a5J4D/E5FyYBu+XoaiKIqSQSTbG+OlpaWmrCx8dquiKIoSCxH50hhTGh6uK5oVRVGUAFnfUxCRCuDbFG9vDmyxUZxsQNOcH2ia84N00nyMMaba9M2sVwrpICJlkbpPuYymOT/QNOcHTqRZzUeKoihKAFUKiqIoSoB8VwpT3RbABTTN+YGmOT+wPc15PaagKIqihJLvPQVFURQlCFUKiqIoSoC8VQoiMtbaAa5cRCa7LU86iMiTIrJZRBYHhTUVkVkistL628QKFxF52Er3QhEZEHTP5db1K0Xk8kjP8gIi0l5E3hORpdbOfzdZ4bmc5toi8rmILLDS/DsrvKO1U2G5tXNhTSs86k6GInKbFb5cRMZEeaRnEJFCEZknIm9a/+d0mkVkjYgssrYXKLPCMpe3jTF598Pnd+kboBNQE1gA9HJbrjTSMwIYACwOCvszMNk6ngz8yTo+A5gBCDAEmGuFNwVWWX+bWMdN3E5blPS2BgZYxw2AFUCvHE+zAPWt4xrAXCstLwMTrPDHgOus4+uBx6zjCcBL1nEvK7/XAjpa5aDQ7fTFSfstwPPAm9b/OZ1mYA3QPCwsY3k7X3sKg4ByY8wqY8whfJ5c4+4P4VWMMR/icyIYTPBOdk8D5wSFP2N8zAEai0hrYAwwyxizzRizHZgFjHVc+BQwxmw0xnxlHe8GluHbpCmX02yMMXusf2tYPwOMwrdTIVRPc6SdDMcDLxpjDhpjVgPl+MqDJxGRdsA44J/W/0KOpzkKGcvb+aoUAru8Way3wnKJlsaYjdbx90BL6zha2rPynVgmgv74Ws45nWbLjDIf2IyvkH8D7DC+nQohVP5oOxlmVZqBh4BbgSrr/2bkfpoN8I6IfCkik6ywjOXttFxnK9mBMcaISM7NPRaR+sCrwM3GmF0StKV3LqbZGHME6CcijYHXgB7uSuQsInImsNkY86WInOyyOJnkBGPMBhFpAcwSka+DTzqdt/O1p+Df5c1POyssl9hkdSOx/m62wqOlPaveifj25HgVeM4Y8x8rOKfT7McYswN4DxiKz1zgb9wFyx9Im4TuZJhNaR4OnC0ia/CZeEcBfyW304w5ujvlZnzKfxAZzNv5qhS+ALpasxhq4huUesNlmewmeCe7y4FpQeGXWbMWhgA7rW7pTOA0EWlizWw4zQrzHJad+AlgmTHmgaBTuZzmYquHgIjUAU7FN5byHr6dCqF6miPtZPgGMMGaqdMR6Ap8npFEJIkx5jZjTDtjTAm+Mvo/Y8xEcjjNIlJPRBr4j/HlycVkMm+7PdLu1g/fqP0KfHbZKW7Lk2ZaXgA2Aofx2Q6vwmdLfRdYiW+Hu6bWtQL83Ur3IqA0KJ4r8Q3ClQM/cjtdMdJ7Aj6760JgvvU7I8fTfBwwz0rzYuBOK7wTvgquHPg3UMsKr239X26d7xQU1xTrXSwHTnc7bQmm/2SOzj7K2TRbaVtg/Zb466ZM5m11c6EoiqIEyFfzkaIoihIBVQqKoihKAFUKiqIoSgBVCoqiKEoAVQqKoihKAFUKiqIoSgBVCoqiKEqA/w+k6Fw5gD4ncQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(total_rewards)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}