project_cnn_run_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import h5py\n",
    "from data_utils import get_data\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (2408, 22, 1000) \n",
      "y_train: (2408,) \n",
      "X_val: (100, 22, 1000) \n",
      "y_val: (100,) \n",
      "X_test: (50, 22, 1000) \n",
      "y_test: (50,) \n"
     ]
    }
   ],
   "source": [
    "# Load data from all .mat files, combine them, eliminate EOG signals, shuffle and \n",
    "# seperate training data, validation data and testing data.\n",
    "# Also do mean subtraction on x.\n",
    "\n",
    "data = get_data('../project_datasets',num_validation=100, num_test=50)\n",
    "for k in data.keys():\n",
    "    print('{}: {} '.format(k, data[k].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class flatten to connect to FC layer\n",
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        N, C, H = x.size() # read in N, C, H\n",
    "        return x.view(N, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn x and y into torch type tensor\n",
    "\n",
    "N_train, C_train, H_train = data.get('X_train').shape\n",
    "N_val, C_val, H_val = data.get('X_val').shape\n",
    "N_test, C_test, H_test = data.get('X_test').shape\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "\n",
    "X_train = Variable(torch.Tensor(data.get('X_train'))).type(dtype)\n",
    "y_train = Variable(torch.Tensor(data.get('y_train'))).type(torch.LongTensor)\n",
    "X_val = Variable(torch.Tensor(data.get('X_val'))).type(dtype)\n",
    "y_val = Variable(torch.Tensor(data.get('y_val'))).type(torch.LongTensor)\n",
    "X_test = Variable(torch.Tensor(data.get('X_test'))).type(dtype)\n",
    "y_test = Variable(torch.Tensor(data.get('y_test'))).type(torch.LongTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up sequential model\n",
    "\n",
    "model = nn.Sequential(\n",
    "                      nn.Conv1d(22, 10, kernel_size=12, stride=4),\n",
    "                      nn.ReLU(inplace=True),\n",
    "                      nn.Dropout(p=0.5),\n",
    "                      nn.BatchNorm1d(num_features=10),\n",
    "                      nn.MaxPool1d(kernel_size=4, stride=4),\n",
    "                      Flatten(),\n",
    "                      nn.Linear(620, 20),\n",
    "                      nn.Dropout(p=0.5),\n",
    "                      nn.ReLU(inplace=True),\n",
    "                      nn.Linear(20, 4)\n",
    ")\n",
    "\n",
    "model.type(dtype)\n",
    "loss_fn = nn.CrossEntropyLoss().type(dtype)\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0 , loss is  [ 1.40513206]\n",
      "Training accuracy 0.1875\n",
      "Validation accuracy 0.26 \n",
      "\n",
      "Epoch  1 , loss is  [ 1.36399293]\n",
      "Training accuracy 0.3375\n",
      "Validation accuracy 0.3 \n",
      "\n",
      "Epoch  2 , loss is  [ 1.37248492]\n",
      "Training accuracy 0.325\n",
      "Validation accuracy 0.27 \n",
      "\n",
      "Epoch  3 , loss is  [ 1.37209678]\n",
      "Training accuracy 0.275\n",
      "Validation accuracy 0.28 \n",
      "\n",
      "Epoch  4 , loss is  [ 1.37721419]\n",
      "Training accuracy 0.2875\n",
      "Validation accuracy 0.31 \n",
      "\n",
      "Epoch  5 , loss is  [ 1.37488699]\n",
      "Training accuracy 0.35\n",
      "Validation accuracy 0.22 \n",
      "\n",
      "Epoch  6 , loss is  [ 1.39181626]\n",
      "Training accuracy 0.1875\n",
      "Validation accuracy 0.25 \n",
      "\n",
      "Epoch  7 , loss is  [ 1.36790729]\n",
      "Training accuracy 0.2875\n",
      "Validation accuracy 0.27 \n",
      "\n",
      "Epoch  8 , loss is  [ 1.3433392]\n",
      "Training accuracy 0.325\n",
      "Validation accuracy 0.29 \n",
      "\n",
      "Epoch  9 , loss is  [ 1.38277161]\n",
      "Training accuracy 0.2625\n",
      "Validation accuracy 0.25 \n",
      "\n",
      "Epoch  10 , loss is  [ 1.35818553]\n",
      "Training accuracy 0.35\n",
      "Validation accuracy 0.27 \n",
      "\n",
      "Epoch  11 , loss is  [ 1.3316226]\n",
      "Training accuracy 0.35\n",
      "Validation accuracy 0.22 \n",
      "\n",
      "Epoch  12 , loss is  [ 1.31109011]\n",
      "Training accuracy 0.3375\n",
      "Validation accuracy 0.25 \n",
      "\n",
      "Epoch  13 , loss is  [ 1.34583485]\n",
      "Training accuracy 0.3125\n",
      "Validation accuracy 0.27 \n",
      "\n",
      "Epoch  14 , loss is  [ 1.3180021]\n",
      "Training accuracy 0.325\n",
      "Validation accuracy 0.26 \n",
      "\n",
      "Epoch  15 , loss is  [ 1.32852447]\n",
      "Training accuracy 0.325\n",
      "Validation accuracy 0.26 \n",
      "\n",
      "Epoch  16 , loss is  [ 1.28838122]\n",
      "Training accuracy 0.325\n",
      "Validation accuracy 0.29 \n",
      "\n",
      "Epoch  17 , loss is  [ 1.2820574]\n",
      "Training accuracy 0.4125\n",
      "Validation accuracy 0.3 \n",
      "\n",
      "Epoch  18 , loss is  [ 1.30784237]\n",
      "Training accuracy 0.4125\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "Epoch  19 , loss is  [ 1.29321313]\n",
      "Training accuracy 0.4375\n",
      "Validation accuracy 0.31 \n",
      "\n",
      "Epoch  20 , loss is  [ 1.26818323]\n",
      "Training accuracy 0.4375\n",
      "Validation accuracy 0.28 \n",
      "\n",
      "Epoch  21 , loss is  [ 1.27541935]\n",
      "Training accuracy 0.4125\n",
      "Validation accuracy 0.35 \n",
      "\n",
      "Epoch  22 , loss is  [ 1.31758845]\n",
      "Training accuracy 0.375\n",
      "Validation accuracy 0.25 \n",
      "\n",
      "Epoch  23 , loss is  [ 1.30058503]\n",
      "Training accuracy 0.35\n",
      "Validation accuracy 0.29 \n",
      "\n",
      "Epoch  24 , loss is  [ 1.2763586]\n",
      "Training accuracy 0.425\n",
      "Validation accuracy 0.32 \n",
      "\n",
      "Epoch  25 , loss is  [ 1.23504758]\n",
      "Training accuracy 0.425\n",
      "Validation accuracy 0.33 \n",
      "\n",
      "Epoch  26 , loss is  [ 1.22543073]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.36 \n",
      "\n",
      "Epoch  27 , loss is  [ 1.23575354]\n",
      "Training accuracy 0.425\n",
      "Validation accuracy 0.27 \n",
      "\n",
      "Epoch  28 , loss is  [ 1.28175378]\n",
      "Training accuracy 0.3625\n",
      "Validation accuracy 0.34 \n",
      "\n",
      "Epoch  29 , loss is  [ 1.18945038]\n",
      "Training accuracy 0.4625\n",
      "Validation accuracy 0.28 \n",
      "\n",
      "Epoch  30 , loss is  [ 1.23302186]\n",
      "Training accuracy 0.45\n",
      "Validation accuracy 0.31 \n",
      "\n",
      "Epoch  31 , loss is  [ 1.21989226]\n",
      "Training accuracy 0.375\n",
      "Validation accuracy 0.35 \n",
      "\n",
      "Epoch  32 , loss is  [ 1.20291603]\n",
      "Training accuracy 0.5\n",
      "Validation accuracy 0.33 \n",
      "\n",
      "Epoch  33 , loss is  [ 1.24380684]\n",
      "Training accuracy 0.375\n",
      "Validation accuracy 0.38 \n",
      "\n",
      "Epoch  34 , loss is  [ 1.1900667]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.33 \n",
      "\n",
      "Epoch  35 , loss is  [ 1.17393231]\n",
      "Training accuracy 0.475\n",
      "Validation accuracy 0.36 \n",
      "\n",
      "Epoch  36 , loss is  [ 1.12345123]\n",
      "Training accuracy 0.5375\n",
      "Validation accuracy 0.35 \n",
      "\n",
      "Epoch  37 , loss is  [ 1.1758002]\n",
      "Training accuracy 0.45\n",
      "Validation accuracy 0.33 \n",
      "\n",
      "Epoch  38 , loss is  [ 1.1389426]\n",
      "Training accuracy 0.475\n",
      "Validation accuracy 0.33 \n",
      "\n",
      "Epoch  39 , loss is  [ 1.22996402]\n",
      "Training accuracy 0.425\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  40 , loss is  [ 1.17361426]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "Epoch  41 , loss is  [ 1.14444792]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.36 \n",
      "\n",
      "Epoch  42 , loss is  [ 1.1174705]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.34 \n",
      "\n",
      "Epoch  43 , loss is  [ 1.15802658]\n",
      "Training accuracy 0.525\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  44 , loss is  [ 1.1195122]\n",
      "Training accuracy 0.525\n",
      "Validation accuracy 0.34 \n",
      "\n",
      "Epoch  45 , loss is  [ 1.10310698]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  46 , loss is  [ 1.06435037]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.29 \n",
      "\n",
      "Epoch  47 , loss is  [ 1.13408947]\n",
      "Training accuracy 0.5\n",
      "Validation accuracy 0.35 \n",
      "\n",
      "Epoch  48 , loss is  [ 1.06673837]\n",
      "Training accuracy 0.4875\n",
      "Validation accuracy 0.4 \n",
      "\n",
      "Epoch  49 , loss is  [ 1.16557801]\n",
      "Training accuracy 0.5375\n",
      "Validation accuracy 0.37 \n",
      "\n",
      "Epoch  50 , loss is  [ 1.14834499]\n",
      "Training accuracy 0.45\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  51 , loss is  [ 1.11310363]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  52 , loss is  [ 1.11278152]\n",
      "Training accuracy 0.5375\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  53 , loss is  [ 0.99410737]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.36 \n",
      "\n",
      "Epoch  54 , loss is  [ 1.08257985]\n",
      "Training accuracy 0.525\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  55 , loss is  [ 1.00271344]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.32 \n",
      "\n",
      "Epoch  56 , loss is  [ 1.05348039]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  57 , loss is  [ 1.08347344]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  58 , loss is  [ 1.1270889]\n",
      "Training accuracy 0.5125\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  59 , loss is  [ 1.00658309]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  60 , loss is  [ 1.04098678]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.4 \n",
      "\n",
      "Epoch  61 , loss is  [ 1.01331949]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  62 , loss is  [ 1.05872262]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  63 , loss is  [ 1.00404799]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  64 , loss is  [ 1.02888608]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  65 , loss is  [ 1.02759886]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  66 , loss is  [ 0.94584781]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  67 , loss is  [ 0.96531582]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  68 , loss is  [ 0.9846409]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  69 , loss is  [ 1.03448939]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  70 , loss is  [ 1.00978887]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  71 , loss is  [ 0.96390259]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  72 , loss is  [ 1.04844141]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  73 , loss is  [ 1.02713418]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  74 , loss is  [ 0.9912799]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  75 , loss is  [ 0.95969927]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  76 , loss is  [ 0.94129503]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  77 , loss is  [ 1.00973344]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  78 , loss is  [ 1.03745508]\n",
      "Training accuracy 0.55\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  79 , loss is  [ 1.0142082]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  80 , loss is  [ 0.93826514]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  81 , loss is  [ 0.9632678]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  82 , loss is  [ 0.90470159]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  83 , loss is  [ 0.92345679]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  84 , loss is  [ 0.95419347]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  85 , loss is  [ 0.95159465]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  86 , loss is  [ 1.03337348]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  87 , loss is  [ 0.88387614]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  88 , loss is  [ 0.8298794]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  89 , loss is  [ 0.96171248]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  90 , loss is  [ 0.94489086]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  91 , loss is  [ 0.9749527]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  92 , loss is  [ 0.83475268]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  93 , loss is  [ 0.94240171]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  94 , loss is  [ 0.9415015]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  95 , loss is  [ 0.83543903]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.41 \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  96 , loss is  [ 0.96786773]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  97 , loss is  [ 0.84523833]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  98 , loss is  [ 0.84952259]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  99 , loss is  [ 0.91274959]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  100 , loss is  [ 0.90270454]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  101 , loss is  [ 0.80598849]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  102 , loss is  [ 0.92124605]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  103 , loss is  [ 0.83964634]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  104 , loss is  [ 0.9125616]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  105 , loss is  [ 0.94573784]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  106 , loss is  [ 0.84153879]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  107 , loss is  [ 0.88358563]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  108 , loss is  [ 0.93062288]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  109 , loss is  [ 0.90615749]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  110 , loss is  [ 0.81001455]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  111 , loss is  [ 0.94004852]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  112 , loss is  [ 0.76073229]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  113 , loss is  [ 0.95073336]\n",
      "Training accuracy 0.5625\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  114 , loss is  [ 0.89388245]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  115 , loss is  [ 0.69509411]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  116 , loss is  [ 0.85065699]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  117 , loss is  [ 0.84703147]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  118 , loss is  [ 0.94512969]\n",
      "Training accuracy 0.575\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  119 , loss is  [ 0.80878222]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  120 , loss is  [ 0.76262504]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  121 , loss is  [ 0.85175401]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  122 , loss is  [ 0.82923126]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  123 , loss is  [ 0.80948371]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  124 , loss is  [ 0.82758921]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  125 , loss is  [ 0.77957428]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  126 , loss is  [ 0.78649849]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  127 , loss is  [ 0.89697218]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  128 , loss is  [ 0.9244777]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.42 \n",
      "\n",
      "Epoch  129 , loss is  [ 0.82122087]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  130 , loss is  [ 0.7962324]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  131 , loss is  [ 0.7874583]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  132 , loss is  [ 0.80534065]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  133 , loss is  [ 0.85221082]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  134 , loss is  [ 0.82918805]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  135 , loss is  [ 0.85047626]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  136 , loss is  [ 0.84630936]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  137 , loss is  [ 0.76854134]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  138 , loss is  [ 0.88122761]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  139 , loss is  [ 0.84849358]\n",
      "Training accuracy 0.6125\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  140 , loss is  [ 0.85296249]\n",
      "Training accuracy 0.5875\n",
      "Validation accuracy 0.43 \n",
      "\n",
      "Epoch  141 , loss is  [ 0.72810185]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  142 , loss is  [ 0.70308882]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  143 , loss is  [ 0.84987986]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  144 , loss is  [ 0.79795593]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  145 , loss is  [ 0.75605774]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  146 , loss is  [ 0.8139798]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  147 , loss is  [ 0.76902205]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  148 , loss is  [ 0.83749545]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  149 , loss is  [ 0.90037203]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  150 , loss is  [ 0.65582079]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  151 , loss is  [ 0.90655977]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  152 , loss is  [ 0.70967895]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  153 , loss is  [ 0.81658542]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  154 , loss is  [ 0.77054304]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  155 , loss is  [ 0.73823291]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  156 , loss is  [ 0.86710376]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  157 , loss is  [ 0.76438093]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  158 , loss is  [ 0.81826448]\n",
      "Training accuracy 0.625\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  159 , loss is  [ 0.76417756]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.45 \n",
      "\n",
      "Epoch  160 , loss is  [ 0.63713819]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  161 , loss is  [ 0.68657464]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  162 , loss is  [ 0.7645272]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  163 , loss is  [ 0.6857177]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.39 \n",
      "\n",
      "Epoch  164 , loss is  [ 0.74204731]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  165 , loss is  [ 0.87311256]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  166 , loss is  [ 0.85374546]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  167 , loss is  [ 0.71579182]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  168 , loss is  [ 0.73298258]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  169 , loss is  [ 0.75694835]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  170 , loss is  [ 0.82809991]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  171 , loss is  [ 0.67969626]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  172 , loss is  [ 0.7753225]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  173 , loss is  [ 0.69799352]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.56 \n",
      "\n",
      "Epoch  174 , loss is  [ 0.65974724]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  175 , loss is  [ 0.78067398]\n",
      "Training accuracy 0.6\n",
      "Validation accuracy 0.62 \n",
      "\n",
      "Epoch  176 , loss is  [ 0.65847206]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  177 , loss is  [ 0.74355131]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  178 , loss is  [ 0.70943594]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  179 , loss is  [ 0.77557933]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  180 , loss is  [ 0.70935172]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  181 , loss is  [ 0.72948205]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  182 , loss is  [ 0.78083187]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  183 , loss is  [ 0.75239432]\n",
      "Training accuracy 0.6375\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  184 , loss is  [ 0.70145619]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.59 \n",
      "\n",
      "Epoch  185 , loss is  [ 0.6741811]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  186 , loss is  [ 0.72990942]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.61 \n",
      "\n",
      "Epoch  187 , loss is  [ 0.81707013]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  188 , loss is  [ 0.60890567]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  189 , loss is  [ 0.77326101]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  190 , loss is  [ 0.73377061]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.49 \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  191 , loss is  [ 0.67832756]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  192 , loss is  [ 0.74126863]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  193 , loss is  [ 0.73519284]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.41 \n",
      "\n",
      "Epoch  194 , loss is  [ 0.73236686]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  195 , loss is  [ 0.76081604]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  196 , loss is  [ 0.71159947]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  197 , loss is  [ 0.62789732]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  198 , loss is  [ 0.64987856]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  199 , loss is  [ 0.67192996]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.56 \n",
      "\n",
      "Epoch  200 , loss is  [ 0.73467851]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  201 , loss is  [ 0.63136828]\n",
      "Training accuracy 0.8125\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  202 , loss is  [ 0.69683099]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  203 , loss is  [ 0.61540115]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  204 , loss is  [ 0.85079211]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  205 , loss is  [ 0.7068007]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  206 , loss is  [ 0.74399388]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  207 , loss is  [ 0.7674948]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  208 , loss is  [ 0.74667919]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  209 , loss is  [ 0.6709978]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  210 , loss is  [ 0.69849432]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  211 , loss is  [ 0.75418174]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  212 , loss is  [ 0.64166462]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.59 \n",
      "\n",
      "Epoch  213 , loss is  [ 0.6610254]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  214 , loss is  [ 0.6881904]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  215 , loss is  [ 0.68243295]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  216 , loss is  [ 0.72121334]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  217 , loss is  [ 0.75172353]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  218 , loss is  [ 0.65848249]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  219 , loss is  [ 0.63547933]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  220 , loss is  [ 0.5859099]\n",
      "Training accuracy 0.8125\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  221 , loss is  [ 0.648018]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  222 , loss is  [ 0.66402578]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  223 , loss is  [ 0.7362535]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  224 , loss is  [ 0.78195387]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  225 , loss is  [ 0.61518061]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  226 , loss is  [ 0.58794284]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  227 , loss is  [ 0.62918079]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  228 , loss is  [ 0.64277309]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  229 , loss is  [ 0.71774149]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  230 , loss is  [ 0.73574209]\n",
      "Training accuracy 0.6625\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  231 , loss is  [ 0.6772691]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  232 , loss is  [ 0.62942863]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  233 , loss is  [ 0.66674244]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  234 , loss is  [ 0.64592278]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  235 , loss is  [ 0.60336453]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  236 , loss is  [ 0.66870844]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  237 , loss is  [ 0.73375362]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  238 , loss is  [ 0.59898388]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  239 , loss is  [ 0.65126187]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  240 , loss is  [ 0.67602718]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  241 , loss is  [ 0.65991771]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  242 , loss is  [ 0.5939523]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  243 , loss is  [ 0.73876816]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  244 , loss is  [ 0.69280124]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  245 , loss is  [ 0.62168986]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  246 , loss is  [ 0.63864148]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  247 , loss is  [ 0.68114233]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  248 , loss is  [ 0.56388682]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  249 , loss is  [ 0.66101599]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  250 , loss is  [ 0.55451524]\n",
      "Training accuracy 0.8375\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  251 , loss is  [ 0.63692713]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.61 \n",
      "\n",
      "Epoch  252 , loss is  [ 0.559039]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  253 , loss is  [ 0.58272487]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  254 , loss is  [ 0.62555116]\n",
      "Training accuracy 0.8125\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  255 , loss is  [ 0.59016669]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  256 , loss is  [ 0.69198555]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  257 , loss is  [ 0.61171091]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  258 , loss is  [ 0.62966341]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  259 , loss is  [ 0.68637568]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  260 , loss is  [ 0.69626451]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  261 , loss is  [ 0.65597445]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  262 , loss is  [ 0.77452874]\n",
      "Training accuracy 0.65\n",
      "Validation accuracy 0.55 \n",
      "\n",
      "Epoch  263 , loss is  [ 0.63311064]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  264 , loss is  [ 0.58797514]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.46 \n",
      "\n",
      "Epoch  265 , loss is  [ 0.63194954]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  266 , loss is  [ 0.71211153]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  267 , loss is  [ 0.59110129]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  268 , loss is  [ 0.6750586]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  269 , loss is  [ 0.57134593]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  270 , loss is  [ 0.67452514]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  271 , loss is  [ 0.68721712]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.6 \n",
      "\n",
      "Epoch  272 , loss is  [ 0.70804489]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.58 \n",
      "\n",
      "Epoch  273 , loss is  [ 0.601349]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  274 , loss is  [ 0.7019822]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  275 , loss is  [ 0.73763299]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  276 , loss is  [ 0.54763794]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  277 , loss is  [ 0.71677709]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  278 , loss is  [ 0.73112446]\n",
      "Training accuracy 0.6875\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  279 , loss is  [ 0.60885382]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  280 , loss is  [ 0.60685313]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.44 \n",
      "\n",
      "Epoch  281 , loss is  [ 0.65270042]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  282 , loss is  [ 0.73451895]\n",
      "Training accuracy 0.675\n",
      "Validation accuracy 0.53 \n",
      "\n",
      "Epoch  283 , loss is  [ 0.5726456]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  284 , loss is  [ 0.5002138]\n",
      "Training accuracy 0.825\n",
      "Validation accuracy 0.54 \n",
      "\n",
      "Epoch  285 , loss is  [ 0.64280075]\n",
      "Training accuracy 0.7375\n",
      "Validation accuracy 0.56 \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  286 , loss is  [ 0.67552817]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.62 \n",
      "\n",
      "Epoch  287 , loss is  [ 0.55674016]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.48 \n",
      "\n",
      "Epoch  288 , loss is  [ 0.63483882]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  289 , loss is  [ 0.54818189]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.6 \n",
      "\n",
      "Epoch  290 , loss is  [ 0.68116647]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.51 \n",
      "\n",
      "Epoch  291 , loss is  [ 0.561086]\n",
      "Training accuracy 0.725\n",
      "Validation accuracy 0.57 \n",
      "\n",
      "Epoch  292 , loss is  [ 0.54269588]\n",
      "Training accuracy 0.75\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  293 , loss is  [ 0.61370122]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  294 , loss is  [ 0.72172457]\n",
      "Training accuracy 0.7\n",
      "Validation accuracy 0.5 \n",
      "\n",
      "Epoch  295 , loss is  [ 0.48375201]\n",
      "Training accuracy 0.8125\n",
      "Validation accuracy 0.47 \n",
      "\n",
      "Epoch  296 , loss is  [ 0.58468753]\n",
      "Training accuracy 0.7875\n",
      "Validation accuracy 0.49 \n",
      "\n",
      "Epoch  297 , loss is  [ 0.6508466]\n",
      "Training accuracy 0.7125\n",
      "Validation accuracy 0.52 \n",
      "\n",
      "Epoch  298 , loss is  [ 0.58613813]\n",
      "Training accuracy 0.775\n",
      "Validation accuracy 0.56 \n",
      "\n",
      "Epoch  299 , loss is  [ 0.58546048]\n",
      "Training accuracy 0.7625\n",
      "Validation accuracy 0.56 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train through several iterations\n",
    "num_epoch = 300\n",
    "batch_size = 80\n",
    "step = np.arange(0,N_train,batch_size)\n",
    "#step = np.append(step,N_train) #discard some data\n",
    "\n",
    "loss_his = []\n",
    "train_accu_his = []\n",
    "val_accu_his = []\n",
    "\n",
    "for epoch in range(num_epoch):\n",
    "    for t in range(step.shape[0]-1):\n",
    "\n",
    "        # calculate training loss\n",
    "        y_train_pred = model(X_train[step[t]:step[t+1],:,:])\n",
    "        loss = loss_fn(y_train_pred, y_train[step[t]:step[t+1]])\n",
    "\n",
    "        # backpropagation\n",
    "        model.zero_grad()\n",
    "        loss.backward()\n",
    "\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "    # calculate predicted value for validation\n",
    "    y_val_pred = model(X_val)\n",
    "\n",
    "    # training loss\n",
    "    print('Epoch ', epoch, ', loss is ', loss.data.numpy())\n",
    "    _, y_pred = torch.max(y_train_pred,1)\n",
    "    loss_his.append(loss.data.numpy())\n",
    "    \n",
    "    # training accuracy\n",
    "    train_accu = np.mean(y_pred.data.numpy() == \n",
    "                                       y_train.data[step[t]:step[t+1]].numpy())\n",
    "    print('Training accuracy', train_accu)\n",
    "    train_accu_his.append(train_accu)\n",
    "\n",
    "    # validation accuracy    \n",
    "    _, y_pred = torch.max(y_val_pred,1)\n",
    "    val_accu = np.mean(y_pred.data.numpy() ==  y_val.data.numpy())\n",
    "    print('Validation accuracy', val_accu, '\\n')   \n",
    "    val_accu_his.append(val_accu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztvXmcHGWd+P/+dE9PpifHTELCkUligsuGK5CBAaNhXYGVcEgYORIPXHVV3FVWQQ2G1YXAuks0X8H1J4qouB4sJByGKGi4oi4oR0Iuwn0mmQA5J9d0Mj0zz++Pquqp7q6qrr6mu2c+79drXtNdXcfzVHU/n+f5nGKMQVEURVEAIpVugKIoilI9qFBQFEVRUqhQUBRFUVKoUFAURVFSqFBQFEVRUqhQUBRFUVKoUFAURVFSqFBQFEVRUqhQUBRFUVLUVboB+TJ27FgzefLkSjdDURSlpli1atV2Y8y4XPvVnFCYPHkyK1eurHQzFEVRagoReTPMfjUnFAph6eoOFi1/kS2dCcY3x5k3ayrtrS2VbpaiKErVMeiFwtLVHVx973oSyV4AOjoTXH3v+tTnKiwURVH6kVrLktrW1mbyUR/NXPgoHZ2JrO3N8RgHe/pSwgJAAAO0qIBQFGWQISKrjDFtufYb9CuFLR4CAaAzkcza5ohH92pCBYOiKEOJQe+SOr45XtBxiWQvC5ZtKHFrFEVRqptBLxTmzZpKLCIFHduZSLJ0dUeJW6QoilK9DHqh0N7awoiGwrVkX12ylinz72fmwkdVQCiKMugZ9EIBoLMr234Qll5jMFh2hisWr6H1+gdVOCiKMmgZ9IZmsOwKXh5IhbCrK5lmhNYYCEVRBhOD3iUVsmMVSkVzPMb+7h6Svf33MB6LcsOF01QwKIpSVYR1SR0S6qP21hZuuHAaLc1xBCsO4Xtzp/PGwvNoKdA7CSxDtFsggHotKYpS2wwJ9RFYgsFr9j5v1tSSryIcryVdLSiKUmsMGaHghzNwL1r+Ih2diVRUc7Fcfe86tTUoilJzDHmhAOmriKWrO/jqkrX0FmlrSST7UsZtjZBWFKVWGBI2hXxob23hu3NOJB6LlvS8iWQvi5a/WNJzKoqilBpdKXjgVilt6UwQESl65QD+eZgURVGqhSHhklosS1d3cOXiNUXbGprjMYYPq1M7g6IoA45mSS0h7a0trHxzJ7c/sbEowdCZSKays2pdB0VRqhFdKeSBE71cquho8K7roAFwiqKUmrArBRUKBbJ0dQcLlm3wrMtQClqa4zw+/4yynFtRlKGHqo/KTGYwnF+Ft0JRo7SiKJVAXVJLxLxZUymkasPwem/X10KLAymKohSDCoUS0d7awsdnTMpbMOzvzk6vIcDpR48rSbsURVHyQYVCCflW+zRumju9qCR7YKXZuGdVR1bdhqWrO5i58FEt+qMoStlQQ3OZmDL//pLkUGqxXVSBrMR96qWkKEpY1NBcYUpV2MeJZ2iIRbIyuTppulUoKIpSKlR9VCbmzZpasvxJiWQvu3xKijppuhVFUUqBCoUy4VfY59ICjNG5cCfaU7uDoijFoOqjMuJV2Md5X2zKDDcdnYnU4O+2O2jKbkVR8kWFQgVY8cK2kgkEh3l3r2XEsDpPu8Oi5S+qUFAUJRSqPqoA5YhWTvYaX7uDRkcrihIWFQoVYKCjlTU6WlGUsKhQqAB+nkmRUluggVhUUnEOiqIouVCbQgXIrOw2PiBArViG19epPUFRlNCoUKgQXp5JDqWs2dCZSDJz4aNawEdRlFCoUKgy3MKi9foHfY3HANGQtaMdAZPLRdUpIqQCRFGGLioUqpjOAIEA0GsM8Vg0L3VTpouqu5qcQMpVNp9yoSpMFGXwoEKhismVP8lJlnfF4jV5nbejM0Hr9Q9y3glHcM+qjpRQyVxzJJK9XH3vOkB8A+KWru7QgDlFGUSo91EVE5Q/KR6LpmbkhaTq3tWV5NdPbMy5ykgk+3wT8YG1gvALmFMUpfYom1AQkdtEZKuIPOvzuYjI90XkFRFZJyInlasttYo7fxJYNgSwVgjulNmnHz2u5PmUcuEk4vMLjNOAOUWpTcqpPvof4AfAL30+Pwc4yv57D/Aj+7/iIshLaenqDhYs20BnItj2UC4WLX/RV8WlAXOKUpuEWimIyJdFZJQ9u/+ZiDwjImcFHWOM+TOwM2CXC4BfGosngGYROSJ804c2ji6/UgIBrNWAl4rLUW0pilJ7hF0p/JMx5r9FZBYwGvgE8CvgwSKu3QJscr3fbG97K3NHEbkMuAxg0qRJRVxy8OClyx9oxjfHAwPxMuMjvPZTY7SiVBdhhYKjsj4X+JUxZoOIDJga2xhzK3ArWOU4B+q61Uw16Oy7untYurojTcXlpdLq6Eww7661IFbiPmebeikpSvURViisEpEHgSnA1SIyEugr8todwETX+wn2NiUEpSr3WQy7upJcsXhNyiW2MRYh2WdSA7+bZF/2tkLSemtMhKKUl7BC4TPAdOA1Y0yXiIwBPl3ktZcBl4vInVgG5t3GmCzVkeLNvFlTPfMkjW6Mcd4JR7DihW2pgfP0o8el3iMQIgjaE3dwmxddyfznCZkrHr9B328FcvW961n55s60/qqgUJTCERNihBCRmcAaY8x+EbkUOAn4b2PMmwHH3AF8ABgLvANcC8QAjDG32OqnHwBnA13Ap40xK3O1pa2tzaxcmXO3IUG+s+alqzvyDnQrNy3NcR6ffwZAViAc9AuiIIGU+Vk8Fk1z2VUUBURklTGmLed+IYXCOuBE4AQsV9OfAnOMMX9fZDvzRoVC4cxc+GjFVU5unMEbSpsEENKFjRtVPylDlbBCIaz6qMcYY0TkAuAHxpifichnimuiMtAEGadjUfG0BZSThliEKxavyamWKgSvvpY7JYcKHGUwEDaiea+IXI3linq/iESwVUFK7eAXUNYcj7Ho4hNpaY4jWLPs4fXe6TVKhUAqA2wxAsHPBc6rr9f9dkNZUnIsXd3B9Ose5IrFa+joTGDoFzhLV6vvhFJbhBUKc4GDWPEKb2N5Ci0qW6uUsuAXaLZg9nG0t7bw+PwzeH3heTw+/wz+88PTfPMuBXHpjEmhjivVyiAeixCLSsa27OC5pas7ylLDOiiIUHNAKbVIKKFgC4LbgSYR+RBwwBjjl75CqVLcuZScFYGfQdZr30tnTErlX/IiKsLtT2xkWF2E0Y0xBGsV0hjr/5qNbiztArMr2QeG1PX8+hQ0OHutKpau7mDmwkeZMv9+Zi581HfGnyuIsBriSRQlH0LZFERkDtbK4I9YK/b/T0TmGWPuLmPblDIQlEspzL63P7HRd3+n4E9nIkk8FuWmudM9r1Vqg3eyz9BYX8fqa/wzrwQNzl6rirC2h1yDflM8lrPyndoilGoirProG8ApxphPGmP+ETgV+PfyNUupVsImugtSncybNbXkWV1zDc5B9hSvVUVY20PQ/YhFhP3dPYF2BkcAqS1CqRbCuqSuN8ZMc72PAGvd2wYKdUmtLF6xBH4I8PrC87KO98vsGosIIxrqAkuQBnkqeQXuuYPfMtvt5RKbyxNqdGOMzq5kWj4nvyBCwLMvbndZv1VTczzGmmsDc04OGLqSGRyU2iX1DyKyHLjDfj8XeKDQxim1i1cCvK7uHs/Bz606aYrH6O7p9Y16boxFGBaL0tmVRHyirqMifPQ9E9OqxblxCgc5eKl9vBL3uQf1XFMkp5/OuS86uYVhdZHU8aMbY1x7vmW4nzL/fs9zuFc1fiscp15FpQdfraw39Ai1UgAQkYuAmfbb/zPG/KZsrQpAVwrVh9/qIRoRej1yHnmRa4YeiwqLLj4xNev/6pK1KRtGznPbJ/ea5RZr3wiKpvY7d5iVAlhCsM+Yis7Ow/RBqQ1KGtFcTahQqE6+uXQ9tz+xseRBaA6Z6pQp8+8v6FrxWJSLTm5JqZjK0d7meIzhw+p81VHu1UTY1COFpu4oVvXjd5+9VINKdRNWKAQamkVkr4js8fjbKyJ7StdcpdZZ8cK2sgkEsNQpbvfQQiu7JZK93P7ExpRhtxx0JpKp2bXXNXZ1JVPG5PbWllBuuoXEPJTCiO13n7Wy3uAlUCgYY0YaY0Z5/I00xowaqEYq1c9A+OO7B7bTjx5HLFKYD1M1rI3dg/y15x8XKuAv33ucjxeVH1pZb+gR1iVVUQIZyJljItnLihe2seiSE2mOlzfbijvwLlbiX4szyGcGCvoFCOZ7j/2ESD7CJZ+AR2VwoDYFpSR4GZsdF9POriRN8RgipLlzXrl4TaB7aWdXMjBdtqPTLtS+EISXDj+MUXp0YyzQpdZNUCZXL8O920PLq8Rp5j328wqLivDdOSdWbGCvZhfXam5bsaihWRlw8v1BFeOdE9aDpxAEaM6IR3BcTIN+Lc3xGAtmHxc6jmN4fZSu7l7PexUUz+EQi0haidN8Pq+U4TooXqTSg281t60UqFBQqp4wP0K/WXOmB0/YgTgMmWnEnTblqvnguM2ufHNn3p5Y7n47A28pBF1zPMbeAz2e7rst9qAedpDPtRos1USgUlRz20pBqYPXFKXk+AWUuQcV53XmrNnx4HH28RqIC6nTIB4za8c4O2/WVObdtdaz3jRYxzlG3Hyv6zYAl1LAdSaSvilFHKN92MA0r9TjyT6TFdDn4PVcS2HnKBfV3LaBRFcKSk2Qaxbn93lUJHSQWzwW9R2MHRtG6/UP5kzDAYV7OA2vj7K/uzQCAfpVYWHtHNAfZ5EZ+R22lGtzPMbBnj7PFeB1v93gm/oj16olqH53KewAha4Ugq5fTTYKVR8pg4pcQVRB+v7MwV6Aj8+YRNu7xmT9YP3UNk50ca5fS4vtIVRNZU+LJR6L0hCL5CVYvGiOx9h/sCdrpRWLCnNPyU5fkqlS81I1XnRyS+Bx+VCITSGornhzPMb+7h5PVWSuWurlECSqPlIGFeOb454DreOm6fd5rhmo14/NS30TZrXh9t8P8qwqlOZ4LNDwXC4Syd6SqLP82j68vo77173lG1PR3triG3Nxx5Obsp6N+7h88FJnnn70OBYtf5ErF6/xHKC92uW0JqjwUlghU4lcUyoUlJpg3qypnrM4ZxAO+jzfGhLQPzBEQqqfHMO3c2yuI/Id4L9n16YotadVqYlFxNfm4kfQfXD0+X56fb9nU6gdwP1dCRqggYKdAYLaFhRwqEJBUVxkDtaOT/6Vi9ekjMCOh1Cxy273wOCX6dShJUO3nctIXIgrbTwWSc1Wm+IxIgJ5jrsDQlC68CDbRpDdJ9dK0O/YphxBjZkqmtOPHpeVct1vgF6wbEOWzSQfgoIQq8HYrTYFpeYYSH/yoIE70wCZa5D3qt8QhkwX2Wol1yrhe3Onez63oMHVWSEF2RQWP7XJ97otPgbrXMI7V7sKxcvlOkwa+lK4xZYkIZ6iVCOlyOkTlqAcP5mzt6DZnJMeAkglqQtLqQRCLCJpaTtKTZBAaGmO+6bMaPGZObuzfWQe2xyP0RCLcPsTG+kJmNh6JQHMVVcbrO9TUD3yfHDXD7/oZMs+MmX+/Uy/7kHm3b02LWHhvgM9xKLp1x3oXFOqPlJqjoFcYre3tvi6UWaqAYKM3W6VUTlmoGE4dcpontm4O21brqjoUiBYwjXIq8Zr5m6M5QZ73W83pFXUa8r06snR9ESyl68uWQsQGCuRSa8xRa0YHC+3b7Vbk4HMFYqXLSXZZzxdggfSjVWFglJz5PJEKjXXnp+duiJz9rZ0dQf7D/ZkHZu5X5gBSSC0gTsfnnhtV9Y53YNQuQzYBiv40D2Qe3nV+BVOyqyoV4gHVq8xqev5fX8ycdKW5FXQCau/LS47xZT596dUQ2EEzO5EMlU7xBGkft5P5UDVR0rNMdDpnHNlCnVmgJmD1ejGWJadI5fgammO8/rC8+jLQyA0xiJpKgo//Aa23YlkKH11McqUzkTSN1LcGfhKLQQzcUemh0lVvr/bEvJhn0VLc5yb5k7njYXnMW/WVO5Z1ZGmGgob5+F8R0pRD6MQdKWg1Bxh0mOU45p+5/fTUTfW12Udc/rR43zzIrkFW9jZLMDo4cNCGbz9PHXGN8dZurojMC2IO9dUKd1iM1NtlBun3WFyWSV7TagobgFusg3iDmHsFl64vwOVck9VoaDUJPnEHpSbsDaOpas7uGdVh+fAm+kl4xV3Efb6fjEbftG/82ZN5brfbghUzR9I9gWev1BEGHAbixNY2ByPFe3Z5dgNMr+Lhdi33II36Bzldk9VoaAoRRLWxuE3e/RyN3SvhnLNyjOv4xfTcfsTG2myvXYyazLkUm1kGmvDti0XlfCID4o4DoOz4vJyd3XIZ6Xn0FhvDcczFz4aGDhZ7oJWKhQUpUhyRVs75Dvzc1ZDQX71frYUv2M7E0nisWiaumPmwkdD9dNtrHXO76dKisciJFyri8GCAK/ecC7gbwReurqDnfsP5n3uTFWal0AYCPdUNTQrSpGELVnpN8PLNfNznx/6y3WGKY0ZJqYjH3VE5rF+Rv8bLjwhFeE8mIiIsHR1h68R+JtL1zPvrrWeAjGao6Z4VMRT8EdFBrQUqkY0K8oAUYnKXrmyy0L+levcx0JwSuuBNCL7ka/dINf+QVljg1J2uF1/M436ueIh3Ptn2h7CohHNilJlhF1RlJIwqxOv2X4sIvhNbL1sGI/PP4PXF57H4/PPSMtAe9HJLVmurEHz5VhUuHTGpNQ9KgYBLp0xiUUXn0hzjlxIDs3xGHNPmRh47USy19cGE+RW69gwvjd3OjfNnR46shvSBciuriTz7l5bNtdUtSkoygAy0F5TYewdfi6+kB1pnK9Oe8UL27JWKkFz9uH1dakIYCi8/nazy7i+4oVtJHtz2zfisSgLZh8XKsutH7mKOjlqphsunJbmXOAX/OiHU+WvHN8lFQqKMogJG9ORKw6j0HiQfN0nd2d4BBXi/ppZ+S2MUImKpFZtV4asMJeJEK7uhuPJdcXiNSkhUkjp2HK5pqpQUJRBTjGrk2JXNvmmvA7rXtvZlczOgYQ128839iHTrlOIO2m+g7rTd+d/ISuTcrmmqlBQFKVsFBJIl0mQYPIycucz0/cy2uazOgkqv1rO0qyxqJTNNVWFgqIoZSNIfeVVI7uQEpqZx4QNqmuOx1h9zVmh2nz60eN8hZifENrSmeAmjxoS+eClXirU+ygsZXVJFZGzgf8GosBPjTELMz7/FLAIcMzoPzDG/DTonOqSqihKEGGL6OTr+eXneutnDHci1d3H5Zv9NtP9txjCuqSWbaUgIlHgZuCDwGbgaRFZZox5LmPXxcaYy8vVDkVRhhbOQL9g2QbPVBaFzrT91Fi5PLwy04LkY38od0oLL8qpPjoVeMUY8xqAiNwJXABkCgVFUZSS4k7zUe5surk8vDJXLoZ+w7SjHmr2MZoPZMU1h7Kpj0TkYuBsY8xn7fefAN7jXhXY6qMbgG3AS8CVxphNQedV9ZGiKLVELvWSg1uAub2sSiXMaiWi+bfAZGPMCcBDwC+8dhKRy0RkpYis3LZt24A2UFEUpRjCJkJ0IsNvmjudgz197OpKDmhxHYdyCoUOYKLr/QT6DcoAGGN2GGOcdII/BU72OpEx5lZjTJsxpm3cuHFlaayiKEo5yDcRYpgkhuWknELhaeAoEZkiIvXAR4Bl7h1E5AjX29nA82Vsj6IoyoCTb/nYShXXcSibodkY0yMilwPLsVxSbzPGbBCR64GVxphlwJdEZDbQA+wEPlWu9iiKolSCfMvHhi3aVC40dbaiKEoVUa4U6xWPU1AURVHyJ9+VRampuZWCiGwD3izw8LHA9hI2p5JoX6oT7Ut1on2Bdxljcnrq1JxQKAYRWRlm+VQLaF+qE+1LdaJ9CU+l4xQURVGUKkKFgqIoipJiqAmFWyvdgBKifalOtC/VifYlJEPKpqAoiqIEM9RWCoqiKEoAKhQURVGUFENGKIjI2SLyooi8IiLzK92efBGRN0RkvYisEZGV9rYxIvKQiLxs/x9d6XZ6ISK3ichWEXnWtc2z7WLxffs5rRORkyrX8mx8+rJARDrsZ7NGRM51fXa13ZcXRWRWZVqdjYhMFJEVIvKciGwQkS/b22vuuQT0pRafS4OIPCUia+2+XGdvnyIiT9ptXmznk0NEhtnvX7E/n1x0I4wxg/4PK/fSq8CRQD2wFji20u3Ksw9vAGMztn0HmG+/ng98u9Lt9Gn7+4GTgGdztR04F/g9Vh2SGcCTlW5/iL4sAL7mse+x9ndtGDDF/g5GK90Hu21HACfZr0di1TM5thafS0BfavG5CDDCfh0DnrTv9xLgI/b2W4B/sV9/AbjFfv0RrEqWRbVhqKwUUlXgjDHdgFMFrta5gP4aFL8A2ivYFl+MMX/GSnjoxq/tFwC/NBZPAM0Z2XQrik9f/LgAuNMYc9AY8zrwCtZ3seIYY94yxjxjv96LlaG4hRp8LgF98aOan4sxxuyz38bsPwOcAdxtb898Ls7zuhs4U0SkmDYMFaHQArgrum0m+EtTjRjgQRFZJSKX2dsOM8a8Zb9+GzisMk0rCL+21+qzutxWq9zmUuPVRF9slUMr1qy0pp9LRl+gBp+LiERFZA2wFav42KtApzGmx97F3d5UX+zPdwOHFHP9oSIUBgOnGWNOAs4Bvigi73d/aKz1Y036F9dy221+BLwbmA68BXy3ss0Jj4iMAO4BrjDG7HF/VmvPxaMvNflcjDG9xpjpWIXJTgWOHsjrDxWhkLMKXLVjjOmw/28FfoP1ZXnHWcLb/7dWroV549f2mntWxph37B9yH/AT+lURVd0XEYlhDaK3G2PutTfX5HPx6kutPhcHY0wnsAJ4L5a6zslq7W5vqi/2503AjmKuO1SEQs4qcNWMiAwXkZHOa+As4FmsPnzS3u2TwH2VaWFB+LV9GfCPtrfLDGC3S51RlWTo1j+M9WzA6stHbA+RKcBRwFMD3T4vbL3zz4DnjTE3uj6quefi15cafS7jRKTZfh0HPohlI1kBXGzvlvlcnOd1MfCovcIrnEpb2wfqD8t74iUs/dw3Kt2ePNt+JJa3xFpgg9N+LN3hI8DLwMPAmEq31af9d2At35NY+tDP+LUdy/viZvs5rQfaKt3+EH35ld3WdfaP9AjX/t+w+/IicE6l2+9q12lYqqF1wBr779xafC4BfanF53ICsNpu87PANfb2I7EE1yvAXcAwe3uD/f4V+/Mji22DprlQFEVRUgwV9ZGiKIoSAhUKiqIoSgoVCoqiKEqKuty7VBdjx441kydPrnQzFEVRaopVq1ZtNyFqNNecUJg8eTIrV66sdDMURakBlq7uYNHyF9nSmWB8c5x5s6bS3lo1wcsDioi8GWa/mhMKiqIoYVi6uoOr711PItkLQEdngqvvXQ8wZAVDGFQoKIoyKFm0/MWUQHBIJHtZtPzFARcKtbRiUaGgKMqgZEtnIq/t5aLWViyDQigkk0k2b97MgQMHKt2UstLQ0MCECROIxWKVbooyyCnlzLZSs+TxzXE6PARARIQp8+8fsLYUsmKp5MqirEJBRM4G/huryM1PjTELMz6fhJULvNneZ74x5oF8r7N582ZGjhzJ5MmTKTKVeNVijGHHjh1s3ryZKVOmVLo5yiCmlDPbSs6S582aytfvWcfBnr607b12FoeBaku+K5ZKryzKFqcgIlGsXCnnYFU6+qiIHJux2zeBJcaYVqwkdT8s5FoHDhzgkEMOGbQCAUBEOOSQQwb9akipPEEz20qeK1/aW1u4/PS/Sb2PeowPA9GW8c3xvLZX8p5BeVcKqWpnACLiVDt7zrWPAUbZr5uALYVebDALBIeh0MfBTj5qgVKpEPI9Tyl18ZXW6584sRmAk981mmfe3FW2tmTe49OPHseKF7axpTNBQyx77h2PRZk3a2pe7fFShZWDckY0h6lutAC4VEQ2Aw8A/1rG9pSNzs5OfvjD/Bc55557Lp2dnWVokVKNOGqBjs4Ehn61wNLV2an889m3VNd0yHdmG0Qpz1UIu7q6Adh7IFm2tnjd418/sTH1PpHsyzrmopNbfAWzX3vEvla5qXSai48C/2OMmYCd6lZEstokIpeJyEoRWblt27aiL7p0dQczFz7KlPn3M3Pho0XfaD+h0NPT47F3Pw888ADNzc1FXVupHfJRC5RKhVDIeebNmko8Y3YbNLMNwjpX1PNchfwO8z2msysJwN4DPcybNZVhdeH6lc91vO5xLla8sM3zGktXd7D/oPe4YYCvLllbdsFQTvVRmOpGnwHOBjDG/FVEGoCxZFQQM8bcCtwK0NbWVlSu73IYcebPn8+rr77K9OnTicViNDQ0MHr0aF544QVeeukl2tvb2bRpEwcOHODLX/4yl11mlVh2orP37dvHOeecw2mnncZf/vIXWlpauO+++4jHB2Y2pQwM+ahSSqV2KeQ87a0tJHv7mHf3OgBailBdOcd89a619PYZmuIxrpt9HEDev8NCfrv9K4Ue2ltb2LhzPzc+9DIA45sauOrso7OOzfc6haifnHO6rzHvrrUgkOz1H+J6jSm70bmcQiFV7QxLGHwE+FjGPhuBM4H/EZFjsApGFLUUuO63G3huyx7fz1dv7KS7N305l0j2ctXd67jjqY2exxw7fhTXnn+c7zkXLlzIs88+y5o1a/jjH//Ieeedx7PPPpvyErrtttsYM2YMiUSCU045hYsuuohDDkmvrf3yyy9zxx138JOf/IQ5c+Zwzz33cOmll4bttlJhwujt/VwkvdQFufYNayfI55puzjj60NRrYwxXLl7DouUv5hQOXu26YPp4rrpnHb19ho+eOon21hZmLnzUcwWzYNkGAM++hXHtzLz+keOGA7DvYA/vW/gIWzr7HTVu/9wM1m7qZObCR9Ou5XedBcs2eLbL7x7nIvMayb5w812nLeUSCmVTHxljeoDLgeVY5eSWGGM2iMj1IjLb3u2rwOdEZC1WRatPmTJX/ckUCLm2F8Kpp56a5jb6/e9/nxNPPJEZM2awadMmXn755axjpkyZwvTp0wE4+eSTeeONN0rWHqW8hNXbB6lSMsmldglrJ8hHZeKmq7t/wNqy+0Aoe4Rfu+54aiPdtluoM3j6za47E0nm3b3Ws29B6uwaAAAgAElEQVS5Vj1e1//LKztc+6V77v3n/c95ttdvgO9MJD3b5fWsyk1nIlk2NVJZ4xTsmIMHMrZd43r9HDCzlNcMmtEDzFz4qOdDb2mOs/jz7y1JG4YPH556/cc//pGHH36Yv/71rzQ2NvKBD3zA06102LBhqdfRaJREYmCjLgcD5Qr4yXXefGaWN1w4ja/dtZaePsPYEcP45nnHeLbR2fZvv1lPV3cvw+uj1NdFuHLxGiIiKV979/W8gqHaW1vYsGU3P/m/14FgVZC7n+NGDsv63LnOV5esTWujc5zX7yqR7OXffvNs6v1v127hmTd30dwYY5et788kU32SSPZyxeI1nvuCFYzmtCHzOWTeJzcPP781a1si2UvU4/564W5XczxGsrePnj7D6MYY551wBHc+tYmePkM8JvQZyYqXKJZypesYFBHN+TBv1tQ0XR4UbkRzGDlyJHv37vX8bPfu3YwePZrGxkZeeOEFnnjiiYKvU2lyDY4DGYXp5QJ4z6qONB3tlYvXcMXiNb4DYZj2htEvB816OxPJtONuuHAaY4bXs3XvQb7VfhxnH3+E57HO+R967h3uX/8W3b197Ldn734Dll87jh1veX0fP34Uu7qSnqqgzH5u3XvQt129xnCFfW8FywCaDx2dCWKR0rlX9xrDvLvWhla/hDlfQyzCAQ+vIT86E0mcHv3TzCn865lH8du1b7E7keSjp05m694DPPnaDrbt6y5JG6F8br1DTig4P4JSDl6HHHIIM2fO5Pjjjycej3PYYYelPjv77LO55ZZbOOaYY5g6dSozZswoug+VINfgOJBRmF7Xuv2JjVmDk/Peqy1h2xtGjx1Wp2wd90LKI+adPf4Dr8NuW6gEGR8d/OwEu+3rPffWHpxxM7O/hXjQQP4CwSHZZxCBUimLSyUQwFpNXXRyC99/5BUADhleT58xvisbB6cFW3Yn2HewJ/XstnQmeH37Prbvz18gxCLi27dyufVKmVX4Jaetrc1k1lN4/vnnOeaYYyrUooGlUn0NUrs9Pv8M38+jIvQZEzgTz1dA+10rF05bw/THYcr8+z0HPgFeX3heqg+Zq08/3DPrL57+bubNOjr1mde9+Nljr7O+Y3fO88YiwoiGOjq7kjTFY4jArq5kTlVIVITvzjmRKxevKXiALyURoLRKlvwQ4H3vHsOGLXtTqzyw1EP7u3tCCWeAiEAJ5VTWiiwei3LDhdPymnCJyCpjTFuu/Sodp6DUCLmMfH6f9xrja6QsNECr0GWz+7iwrpphAp7aW1v4rw8fH6oNh47q19Vvda0U/O7Fls6unOesiwC2EDBYqgxnVptLN+64ODY3DnySxSOaGgAY1WApLGIRiEYrG7VvgMdf3ZkmEMC6p315jPKlEgjOvZkydjgtzXEEa+KSr0DIhyGnPlIKI5drYxgVijtoKsgwmcuA5netXPpt90Ae1lVz3qypzL93XZp+2csGddZxhwNrA65u4fbquWvVZu5atdl330SyN3D14cxGRST0DNbvOgcKUB0VQywiqSCtnt4+BLBucTWsV7wp4hYXzLSWUTz+6k5e276fluY4N82dXvakeLpSUEKRy7XRKwrWi1xufxCcPdJR+3jNJ2MBs8zMgXzerKlZ+8ciQld3T1qEaXtrC/PP6VfxHNHU4DlL23sgOHo93/3C4MxGixEIDgM93iX7DHvse9GV7KtiUVBZHn91Z+p1oalO8kWFghKK9tYWvvLBo1LvM5ew7a0tqUhV8M5I6WzPpXv3Utm41SvgPYh1uwZHAepsD5fhw7L1r+2tLZwyeXTqfXM8lqaCcf8AT/ub/lrnv/rMezxnansPBBshFaUUDES2VBUKSmje++6xAJw6ZQyPzz8ja3A8/WjL6+rIscP57pwTPfPn5NJx+7kH5+sdY4Aeeyr9N+OGZ7nOTr/uQf5iz8IEEI/0Ao5P/u/Xv5Xa9g83/skzF86eEq4AlNIw0AFlA0W5M8yqUFBC4+jD9yS8Z8WOjnjvQSvPzL9/qL98xnhb7dIS4EZ3uI9qBor7Iazv2JMaxJeu7mDeXWvTDIkGfN0Ne43hew+/lLbNaxnvtVIYO6K+4DZXC45xs5L4rTpzHRP0fSthmETJCNvPcmeYVaFQAUaMGDEg1yl1Nlhnpr7bTyh0W0LBEQ7OygLgjstm0N7a4mmbcPjfz74nFfOQ2e5ifgh9Bq5cvIbJ8+/nisVr8vZp91LZO9GsTvu8bAVfOvOo7ANriLqI8Pj8M/jL1Wfk3hl8nyuQt2ARge/Nnc4bC8+zV53hZ/3xWJTvzjkx9X3zShfS3jo+9X58cwON9eVfVQQJIqfNQZMmZ79iAm3DMDSFwrolcNPxsKDZ+r9uSaVbVHJKlY/fTaI7h1A4aH3e1d1Lb59JZaiE/gjZ9tYWPnNaf16oEcP6f4y77XwuXu0+/ehxRakDymXIdNr355ez8zi+unVfma4ajuZ4cW6mRzRbLqOjG60VT1TwUAlGUgP+N8472nNl0dIc5+MzJgVGMddnGP0/8Lfj0uxVF50czuNmdGMsy9blrBgcd86LTm7hgfVvp47Z0nkgzSusXPjNRdxt9hJizp0ptyuqw9BzSV23BH77JUja6ojdm6z3ACfMKeiU8+fPZ+LEiXzxi18EYMGCBdTV1bFixQp27dpFMpnkW9/6FhdccEEpehCKQoqF58L54XR195Ls7SMWTR8gnJUCWFkpO11C4Xdrt/C5X6zM8v/ed7C/jbsTSd923/HkpjR7xLgRw9i5/yDDYlG6unsZVicIwoES55cJQyLZy10rs11LX922P+exLc1xurp7ckbLAjTWRzlsVAOvb99PxIkGDogKLrZQ36adCWYufJTTj7YM7b0GhtVFaYhF6exKptKL/O+TGzEGbvnTa1w1y0pFfeZ3/8ir2/bznYtPYE6blUH//nVv+fZz3MgGOwXNOhLJPp7ZuCvl/QVW/YEg6iLC/7vkRN88Uu7tMxc+mlcKi1ITFNBZjowL+TL4hMLv58Pb6/0/3/w09GakF0gm4L7LYdUvvI85fBqcs9D3lHPnzuWKK65ICYUlS5awfPlyvvSlLzFq1Ci2b9/OjBkzmD179oCV1CxHGcSEa9Bf/PRGfvTH19K+uHWu2d7+gz2pdA4Av/jrmznP//lfrfJNGpZpoL7w5BZ+/KfX+OzfHcn/PrmRfzjmUBpiUe548k16DPSWMpy0QP7y6vbAz92qgDAR0V3dvSR7rH2OOnQky698f2A0dRhBk0lEIBrpj3twqog5dCaSxGNRbpo7PdVu51Zv6TzA1feuZ+WbO9m40wq6+84fXqA+GqG9tSXt+5BJf30B6/nvTvSkpeHI9b3t6TMpr5xcA2g+v4FoREr+XeozJhUN70WmEBtoBp9QyEWmQMi1PQStra1s3bqVLVu2sG3bNkaPHs3hhx/OlVdeyZ///GcikQgdHR288847HH744QVfJx8KzaMfhHuJ/R+/ez41gDs/6NnT+5O77TvYw59fyq80Rj5ZJBc/ZVV6bYrHaGluoKMzwaEjGzhkxDCOGT+KRzwyYBZKQ51woMdQF4F8FiJBY4lXkj5nduiVBdVhy24rw26TrRpKFbFZsjZUZs9cGHLHPbjdIr1Wde48VNv3dacG96AARy9XZffKNkxwZNh8W/nUP4gAoxpjaSujFS9so6MzkUoh0mynFXGnGOnsSvo+x4EqRVoooYSCiNwL/Az4vTGmkqlJchMwowcsG8LuTdnbmybCp+8v+LKXXHIJd999N2+//TZz587l9ttvZ9u2baxatYpYLMbkyZM9U2aXi3Jkg3WfK3MATyR7+cOz/XravQd6SjowZ+KooV7ZupcX39nLgWQfDXUR6uuEp1/fmePo8MRjUU77m0N49MVtDItF6DlYvO554YXT+Mipk9K2uWeHQbN/R9CMctkL2ltbuDIgtXQ+hJUrQbPtzFM4g/u8WVM9s5nGov4R2c51vL7PXhHsYVSkXufyI9lnaKyvY/U1Z+XcNxOv5zgQhuJiCWto/iFW1bSXRWShiFR3r4I48xqIZUjqWNzaXgRz587lzjvv5O677+aSSy5h9+7dHHroocRiMVasWMGbb+ZWn5QSx8DmeFVkGuAKIZHDGLc70a9e2n+wh70+tWZLyT2rNqf0wwd6+thzoLdkMQMC3HDhNCaMaaQxFk0Z0osl10zReXZBjIqnz+eCzjnazmvkuDwGuT7m4xaZz4x3S2fCysZ6yYlpBvDRjTEWXezvdeNcw8tg7Ce/cqmHMs8Vpu2F4NXmgTAUF0uolYIx5mHgYRFpAj5qv94E/AT4tTGmdsI5HWPyI9fD7s3QNMESCAUamR2OO+449u7dS0tLC0cccQQf//jHOf/885k2bRptbW0cffTRuU9SYtpbW3jwubd5YP3bfOUsf2OVOzune/mbaeTK5aExYlgd+2xB8C+/Whm4b6noLiLFQzwWpSEW8dS9j2qoY8+BHs4+/nD++uoOGodFiUYky1BeCF+7ay3/dq53cR0HJ521n5qjKcOzaN6sqb6ZTjNnuk6shteMfe4pE9PqUngRZAvxyz/lHtz9+p1rVu1lMC5UReo+V66su8WoeyptHyiE0C6pInII8Cngs8Bq4L+Bk4CHytKycnLCHLjyWVjQaf0vUiA4rF+/nhUrVgAwduxY/vrXv7J+/Xp+/vOf8/zzzzN58mQA9u0bOFfFPfbs3S8NQ6YLqJNh08uN9YW3/Gtfx2NRTpzQlHq/v4LeHV4Iloumk/qivi7CDRdO49rzj/P0Yz/neMv2s2N/N13JXobX13HxyRNK0patew+Gcg/OdE+cHXmMx4d9ideGfYwrn70wzZW6vbUl9Mw5aMb+rfZpWbPbS2dM8pztes2EPz5jUuiSo5ltyndWnU950yCCymmWXN1TA+7wYW0KvwGmAr8CzjfGOHH/i0VkYKaESmjcM3/HI8gruGrp6o6cBkq3jvbZLf55/RPJXp5+I7wu3zG0Bs2GS4W7RsK8u9Zy16rNfPDYwzyNvM7qaPiwOhav3Ez7zY+zbe9BYlGrXkGpCKP7drsntu15iIX1PyOO5RAx6uDbWa7ULRkG1NmRx7iqbgnjIzvgpvQVcdAMNp/Zrde+be8aU5BLZb6z6tDum+uWBGoG3OdxG5CDSpcWRBnc4ctBqCI7InK6MWZF3icXORtrRREFfmqMybICi8gcYAHWqnOtMeZjQefUIjv9ffUqygLe7o0z3z2G2z/XX4M6n8IwYEWXBtXJzRenSE2+7ciXzGIk//yrlfxhwztAcL3iz/9yJcufeydtW1T6o5sLKUOZibtQT06CHCSutGogu+/l7MhjLIz9lEZxVfuKxeH871fVAFR2MgdiKP4+5BAyvvg9Q7CeYwnU2EGELbITdupzrIisNsZ02icfDXzUGPPDgAZEgZuBDwKbgadFZJkx5jnXPkcBVwMzjTG7ROTQkO0Z8viVk2yIRTwH2Gc2ps/y800wd/W966mLSCrJXLE4etpCy0D6MX3CKLbtS3rOHJeu7uCRF/o9ovxcGJeu7sgSCJCe7sKOG8NAyiXRXeksTPH3fHTVZvdmT6Ooe7t7xntV15J0gQDWwPjI9UNLKDxyfbpAgOLuQzGz/d3+tTOqadUQVih8zhhzs/PGHsA/h+WV5MepwCvGmNcARORO4ALgOdc+nwNuNsbsss9bsA+jMWbAAsMqhXtV5xf56zfAOtud1UW+Khvn+FLMkN162nw9O5xI3szShCOGRdm2r5uLTp7IJ9472fPYRctf9MyEmqnGCZua2JBdvtNN0CooX131O4zlcLLjPqzt/bS3ttAefRzu9QmcCxqYCqXQmfNA4NffQu9DMUKmaYL/SiGf85SZsIbmqLhGXHsVkCsFZAvgvgOb7W1u/hb4WxF5XESesNVNWYjIZSKyUkRWbtuW/cNoaGhgx44d1Fq96XwwxrBjxw4aGqx8NPkOpvXRSFZNgoLaQXb+mzCMbox5GhDzmS2Pboxx45zpHHPEyNQ253xTDx8FpPvvZxI2yjvMvZ0deYzH6r/E/yU+7GswdBtPod/dsxDXxBu6L6HLpP/kukw9N3Rfkr6jM5P1o6k0xvKs6+3eBJj+GW8YA2rK6NoE142x/rvvZbFG2XVLQHy+q4XeB18hEzDYO3i5w4c9/wASdqXwByyj8o/t95+3t5Xi+kcBHwAmAH8WkWmOmsrBGHMrcCtYNoXMk0yYMIHNmzfjJTAGEw0NDUyYYH2Z84nKBOju7SuZTSBfkRA0mw4TSJR5/CMvbOW5t/Zy2fuP5N/OPYalqzt4ZuMuAK7/7XMY4x3RGjbKO9e9zdLXByz9S+WSuHLUB5m/B/4z9nNGSoI9Js43k59m1agPpu/oNZN1CIrHKXS2X+jMOVMNY+zn79zLjU/A2v9NV9Pcexnc+zlL/37UWfDyg/7tdc5vPL5X+cYlue+NRLzPCfDtKXDch7PbBf3Hx0f7Px/rRlgCsIKrrbBC4etYguBf7PcPAT/NcUwHMNH1foK9zc1m4Ek7zuF1EXkJS0g8HbJdAMRiMaZMmZJ7xyrHy3DsN6DkE5VZavJxN82lJsn0/MhUT2Uev3R1Bw9tsCKnFz+9ka7unjS/+h37u31THYSN8s51b6+qG3h9vdWmbmb0Ps/H6lawrPd9PBT9e27IvLdBM00/42o59OS5ZrxBwiuZgFX/4zH4mv72rfyZ61oe7fU7v0TzMzL7CS8vEjuz27X0C1Zmwt7u/n0cIjHo83AVr7B9IWzwWh/wI/svLE8DR4nIFCxh8BGsqGg3S7GC4X4uImOx1Emv5XGNQYOf4Ri8Z73OtlJ6BJUap9BJrplyZooHP8Ho3CMnE+ruRE9anh0HP3fPsC6MufIJjZcB1NcDrFtC+x+v54LoZg7YWtuW+i5uON/j3vrprZsm+g8wfrP93/yz9TpoYPK9Xg71TK57FTT4epEplP3Ob/ryG2iDhFcYvAZ9h8OnwfaXoNsjbsndn3VL4Pdf7xco8TFwzrfLJjDCxikcBdwAHAs0ONuNMUf6HWOM6RGRy4HlWC6ptxljNojI9cBKY8wy+7OzROQ5oBeYZ4zZUXBvqoB8Zvtu/AzHC5Zt8D3fudOOqGqh4BQ6yYcgdYtzj1L+97KdLWYs3+mZw7K+09L29bMLhFXntEcf56zma2joepst5pDUNeKxKAcaj6Ax8Vb2QYXoqXOpbVwzVYFUnMLpo7eDVz/OvAbu+2L/zNRh9yZ/tYTvANprqWw2PgEfutF7nzOvyXb5RCz1ThC5jK4SsQbwfHD64dgSvASL1zPKHHSdazdNDGcrKJS318HhJ8CWZ7w/373ZatvSL6QLl8RO6xlDWQRDWPXwz7FWCT3A6cAvgV/nOsgY84Ax5m+NMe82xvynve0aWyBgLL5ijDnWGDPNGHNnYd2oDoopbOM3iHUmkr7ne2fPwCXYy5fmeKzk4f1bOhMpff6EyHYiAhMi21kY+ymzI4+l7VtUJkp7IG5MvEVETOoanxrxlJVP6pzrS5M/K4yR1m+muut173OeMAeOv8j7Mz8jcKAwM7DyNn8j7wlz4MRMBYCx7AFBhuEgo2skRv612rD6EWRLAPjA/PT3zqDrVus4wmj3psLaEZa+Hnhng//nTROs5++12ujttj4rA2GD11YZY04WkfXGmGnubWVpVQBewWvVgl8OFS9Da+aKImyhFYfmeIzePlPypHOlcjn1VRsV4b44c+GjLO76HBMi2eqbHX0jSNDAeNnOW4xly8lXccrsz+c+qVd7Hrk+Z6AYT/0EHvha/3avfuTqa4iANBY04/tEFvhEmP9pEaz4lm+X087vtHPZ5dATkD7e65hcxtf4GKgfHrwKyrxuQ7N1PvcgHZa2z8CG3wQf+5XnYVR/Kc7AgDI/6hqg54C/TSCTSBT68lSHOQF2916G/y9SrFQ9IQkbvBZ2pXBQRCJYWVIvF5EPAwNTaLiGCOvy6LWi2Hegh1g0/KykM5EsmUBwl/vLJRAcV8zXhn2Mx+q/xKdGPOWbGyeLYtwXsYyt48Vbuzgmsi+1emiR7Zyy/trc5/3dV6wfXWZ7/AYJt5rlXTOt/w1N3vmzvPp67+csD5V1S6w/3+tsstwzvz3F8lbx46bj+s/ldt3c+GRwv93XdQb3IIHgHON2FXX3zW9WntiZ3f//Gt9/nhPmwCEZdazf88+Q2BXcFj+e+WVuYdLl+jzoGfhRF4f3z7Neh42LChQIHudomthvDA9axZXavdhpUciVwinA80Az8B/AKGCRMeaJsrQqgMGwUvDbb+SwKHtLlJ45F375XYIyRl5XdxufiD6cXoA8n5QBYWbG4JqFbrK8RUxvajbe9ftrvPX5fgTN4gNnYQHncmbIzrHX7IJIxvzq21P8B6hILN0jpRg8zxVivdf2GZg0w8MeMEA4evtM28Epn4Vn7/W/d7F4ce2NDiuqoBYAM6+Ax79X3Dn8OPwESyg6q6sxR8Lrf8reL1oPF9ycl02hZGku7EC1ucaYrwH7gE+HbsUQY96sqXz9nnVpBWi8XCr9Bt1yCITMcoKBqh383TFnRx7LFgiQnytmGPfFHP7rjSd+LN1/Pec1fdz7HrmevARCLG4ZT70G0dW/hJM/ld6HoBlrGJVDWDzPFaJfK2+zVC2VEAjQLwgyjckdq+HgXv/jmibBzlctfXwhFCsQwFqRlIu315Pmeus1iaofDh/6Xtm8j3Kqj4wxvcBpufZTLM+Wy8/4m9T7TFWKozYaSCL4RxOnYasg2u87jlUjruBTI55KS538b/V3ZQsEh7CumH7LXff23389t//6iR+DSB4ZSx33SndkbL7uo3Vx/0H0d1emn7tMBsDSYgrT24emQAPtO8/mEJoGDj+xsHOXirLetxACPT6mrPELYX9Zq0VkGXAXsN/ZaIy5tyytqmHa3jUGgFMmj+auf35fanuYNNXlIFQ5wYzZeWPiLRbEfsyCjx0HJ9hZPNcEFKEPq9s86qz04B7on4GHNfiZXmulIFEsZ7iQZEbMxkfn9+MO2tftrVIpdUw1kGZ4LfB7nmsmv/0l+9lDadwiapAyp8IIa2huAHYAZwDn238fKlejahmnmM0eV2lKZ4Uw0ALBIWcun6BUBQ6+A7+Ec8Vct8QazDOZcKq1PR+DXzJRnBrA6Ws+q418zp0atIYYpVSLBZEybNeoQPDLxxSWMhmYHcJGNKsdISROMRt3pbNc6aEj0l+QvRzk9NkPo+v3DIoSaPunwnPkALzxWP7Rq6UgsQsmvgc2P21f31F3lOBBmF5L3dRTJSuG2HBI7s+9n1IgYkUnv73OXoF2wrAmOJjpLmr/XibNgKX/UphdJFpfdD35XIQSWSLycxG5LfOvrC2rURxh4K50lmum/qFph/uWAwyDM5w1x2NZbq0X1/+Fh+QLwZkm/WYeEuk/DtKDoqL1cOGt/pGukO4q6bcSKJVAkCjWnRAYd4zlZWJ94L1/fLQVSWp6oX6ktWqINea+zrCRufcBiPpnax1Q4mPgG1us//kwrCn3PopFfDRst9OtR+qs38W8l9P3aZqY/nuJFPj9qB9R9nxIYdfPv3O9bgA+DGwpfXNqH0cY7Ovuoa/PEIlIzqybK17cnraScNxFm+Mx9hxIpq0iMlM8/LT+Uqafd5ln7qBPjniKb5qfUpewI5/9PHHOvAbu+wL0Ziz/3Xp4t/vm+Fbo7rJe33S8d3CSV8WrsmKsH90frobON/vVSyMOg67t2bOyg7v7/ce7bW+XMOqPIM+YtP3setb1w6G7grP07n22N1Qevv+RWPWscipChq1C6sAEzOq79/WvoPdv6/+N1Y+wPjtqFnw8I8aj0PtbaAxHHoSKU8g6yApke8wY876cO5eYao5TAPivB57n1j9bOf1uuPB4fvDoq57ZP/2IRYVFF5+YNsg7+Y3yLrEYNi4A4K5Pwwbbb8CJDfBDotbAERH/MoeFRIoWS3yM/aNx32mB4YfC/uxKakOGJjtZcZjn4awoSulhExtuCdxC4zKi9aWJ6ZCoKzbC5/sdi1vebS/8DvbZ35kP3QSP/If3PfHL0dQ0ERDYvRFaL7ViCiDH7yLEKOH12w1JqSOaMzkK0NKZHrhtCQuWPZdaIYQVvclek1b5q721hdGN1lLTN2VzprulQ1BBkMziJq88ZH126LG5E5GZXug94G2cvvdz1vkGWiCA/aPNvNOmNgRCtKH/dWx4ac+9e3NuPXTTRLjwJ/D114ufjbpVT9PmWOqrC2723z8XF9zso/4S6/saFtNnpYX48C3eeZfiY6xJzYduhDO+2b/9/2606iR45bvy+63s3gyNdjT6iMPSt3si1krXEeB++5TZngDhbQp7RWSP8wf8FqvGgpLBHpctwR3Elg+ZNohrzz+OeCzqn7LZ9OKZNsI3RYL0D9rOjMlRi+x6Izi1wlAi8AdaAA0BevrT51u5jC78CZDje+Pn3eS3vWmCtXrzsyuMODw9VUcx3i3xMfBlV+beo8+1/p8wp7D76aT8/vrr1r1xZuBNEy2j7Y5X8zjXhP62nP/99HM5AjGVqvqq/uN2b7I85E78WPoxqXP4XMu5326hEBSrc8IcexXgF+NhBqS+Qljvo5DWNcVtYC6UTG8hR5X09n1jGU9AvAD0rxzu/VzATgHrlmSX9VcNxMdYutcwdolIzDICl0ztYc/Kfj8fEiXK5n7AJ4EdWPpnyJ2/30m1kWmvcdQemdHe7uyt53zb287z/qvS33umww5BLG5d45WHSalCHrjKslWdMCf/82Zmnj1hTnZCwbCuybnO5cbPRfvlB71VN17P4qiz+l2w//Rta6Lldw8y2xZUE2MACLtS+LCINLneN4tIe/maVd0sXd3B9OseZPL8+5k8/35ar38wlc56TyLJyIbi/N+9KpW1t7Zwa92lWXV6PamEi2epcQaYoNmYm9ZLrf1Dxx7kirg11uDQ9qmQ5yuSeltlFBSY5AweXkJNP1YAABNiSURBVDNdR+3htd0Z/NKOc5HZR7/9gnBUL2AbWu2Jx/6t/avXzHbHx1j2gjTs55LZdi+C7lUkZs/UPe5DLvKpJuf1LDKFc9cO/3vg1TavtOKFpGYvkLAJ8dYYY6ZnbFttjGktW8t8qLSheenqDubdtZZkRmBBRKChLkJXHqUqHTI9it5q8077PGX+/ZwfeYwbYz+iTgzGhE/UWBOMOAz2bc0vzbRD41g49gJY87/hPDuiw6wUyA2jgg1/H7wOHhqAH+OcX1rt9+unRC1deKnUB2v+1/KVj4+x1CZBhEmT7RhA83FuyDx3nqnUy3av8u1DqY+H4u6LDyVLiGfjtaIoQzho9bNo+YtZAgGs4LN8BMLho4bx9p6DWR5FE2Q7h6z6Jk9DlmAY3xyHPRCxZ2GDSiCA9WO+8FbvL/+Z1wRnNe3anp1CI4jeg9bf+OnpGU/dxEfDihvCn7MQHO8VZ6Xgp17IZ6YbBkffPXxs7n3dqpYFzd77OLPofGs2B6lxclGuexVGxRNEoXWr3RRzX4okrPfRShG5UUTebf/dCKwqZ8OqkaAMp/ny5X84ings6ulRFJduJj6zKOuYebOm8vXYEv/EdLVCfLRLNeHqzN4t/vUVTpgDI4/of1+qVBKv/xmmvJ8sdZKzfA9adaRUFAUSi0PjOPu1LRTCqBdKgeNM0BhCKLjJldQwTNLDUlGue1XseQfyHpSBsELhX4FuYDFwJ3AA+GKug0TkbBF5UUReEZH5AftdJCJGRHIubSqFozYqFXPaJnHDhdN8PYoONdnb21tbfIvM1Ba2EbdpIlkz9MycS26aJ1n/33t5/vV7fTFWuuKUO6BrEAhyzWyaCO0/tFQvbZ/J/jwWt7ZnuT5m6Myb7Iy19S43VMcLZUGndwGfUrDJLsSz8S/+ke5e5NJ3D7Q+vFz3qpjzVtgmUCxhvY/2A76Duhd2HYabgQ8Cm4GnRWSZMea5jP1GAl8GcpSLqix+aqNCGF4fJRoR2ltbePu+cRzOtqx9tspYDndvsHWMMlBJwCQKsQafaNwis1MmdgZ7ofgtsx2XzlHjcxd+d0iVhAzY1/FYytT3hinL6ZnoTyxj44dutPLcBOmGV//K+l9f4tiEINYtgRX/1f/eL9LdC+dzvz7l+nwoUOP3IJRQEJGHgEuMMZ32+9HAncaYWQGHnQq8Yox5zT7mTuAC4LmM/f4D+DYwL8+2Dyg5M40GEIsKyd7+QXSEyztp00nzaFr1TeIuFVK3iTK6PmkFgaWiiwc4TbDptYyx0cxI1BK1w8km6mm09Fhmr1sCb/yf9fr/vgvHtucutuN4MDm+50E2Ca9CQWF0y54upMZyX4Rg3fC6JbDpaev1/3zIMmoPxMDxyPXZarF8iiXl0ndXUB9eNdTwPQirPhrrCAQAY8wuckc0twDuadZme1sKETkJmGiMuT9kOypGzkyjAXz6fZMBUpHJw4f1C4VTZn+e51qtQcYY6GQk0UiEYUn7dlcyTXBip9Uot2tfKdthesMts1N5lOz4ia4d3sFEbZ8Jdsds+yf/toR1N8zULRdqVMzMgRNkTyk1pTCEKoOWsB5EfSIyyRizEUBEJlPk6GDnT7oR+FSIfS8DLgOYNGlSMZctmHmzpvKVJWtypri++OQW/t8l07nxoZf4/iMvc0RTAyPjljD4uwMruKp+CeP3bOftBePYdNI8Tpn9eU4+/UJY802kcTTN9SMqkyLCj76kpdpw3BaDXEObJlqCJGwCuMyax37L7HyDifz40I1W9TSvADc/I2CuGZ9voFEOo2JQDYtyzzALbbMyJAi7UvgG8JiI/EpEfg38Cbg6xzEdgDv6ZYK9zWEkcDzwRxF5A5gBLPMyNhtjbjXGtBlj2saNGxeyyaWlvbWFC04cn7Yt5nH37l/3FktXd9DSbOWyEeDmR19JuZ5OiGwnInA42zh+1Td5etmP+wep5IH8Zmvx0cV54Ug03PGZdRW8ZvcX/sQaoD/0vezPI7HsICV3IFYug14pZ7bnfLu0RsBCjYqVnK3XuCFUKS+hhIIx5g9AG/AicAfwVSCXkv1p4CgRmSIi9cBHgGWuc+42xow1xkw2xkwGngBmG2OqNgXq+NFxohHhtf86lzcWnseho7JVSh/s/TMz7vt75tx/Ao/Vf4m2vQ9zoKfP1/W09Zn58KydnbQnkV9VpgN7CL1gi8TSE63Fx1gBPh++JbdbpXsGmUul4vV5+w+tpGbV4OJXajfGQs9XSbfFgXJ7VWqSsIbmz2J5CE0A1mDN6v+KVZ7TE2NMj4hcDiwHosBtxpgNInI9sNIYs8zv2Gpl5/5uxgyvJ2IHCmQan1OBaNiBaJHtLIz9FJL4up7W0Qd//UH/hnxSVATtm8oFtCu390NQDQSvGWShhsZSBykVOrMttRGwkPOVuk/5UsOGUKW8hLUpfBk4BXjCGHO6iBwN/FeOYzDGPAA8kLHN81tvjPlAyLaUBHcxmvHN8VS+oQXLNtCZsNJfj26Mce35x6US0m3f180hw/vVIJ8c8RSf7f51Kj1FoxzIWg00SjfX1v2SPiJE/LJfliJXvBtHV5/vj75aXemqtV3FMBj7pAwKwuY+etoYc4qIrAHeY4w5KCIbjDHHlb+J6ZQi99HS1R1cfe/6tGpnsYhV7SzTkOwuenPRj/5CQyzC7Z+dAeuW0HPfv1LXeyC1r18uogHPUbQgIBunoihDklLnPtosIs3AUuAhEdkFvFlMAwca98ogYpe7dOMXmJbsNXx1iRXJvGPfQaZNsHO/PHJ9mkAA/4F/QAXCsFEDeDFFUQYbYSOaP2y/XCAiK4Am4A9la1WJyVwZZAqEXPQakyqJ+caOLh57eRvP9G32TL48oKsCrzKFx5w/QBdXFGUwknc5TmPMn4wxy4wxJVaEl49Fy19MUxUVy66uJFvMIZ6fDZhAaJro8uiBVE6dl5YPTACUoiiDkkJrNNcUXl5Cj9V/ideGfYzH6r/E7MhjeZ1vduQxGsyBSsQYW7h9/FM+53ZrurYPXGSsoiiDjiEhFNwpKjKDyBy30bCCwTn+kMi+lPpoQIVDpk95UGSsoihKngyJQjnzZk1lxV0/YF7dElpke5aKp1G6uapuCcu6Tws8z+zIY9wYu4U6SXctLbnGyCm84iSMc97P+RUcOzt9X81joyhKCRkSK4X26OMsGvYzJkSyBYJDi2xPqZPao9mrhuvqbuN7sR9mCYSy0PZPllvptTut9BFOlPMDX8tWC9V4QQ9FUaqLISEUeOR66s3BwF1EcKmTfsanRjyV+mx25DE+EX249BXP4mPSU084/O3Z1n8nwrivx3q/751se4HmsVEUpYQMCfVRvqqUBg5yVWwxi2PvJZHs5aq6EpbA9Kshm0zAf9pldZxiMmEyaWpkrKIoJWRoCIWAKl1+cQWNibf43ZG/Yfibj3CYR2nMwrArcnkN2HUN/a8doRDWXqB5bBRFKRFDQ3101Fm+HwXFFbz7zTs5nG0ljD1wVeTyakis0XrtRCWrvUBRlAFm8AsFzxq6FSRIleXYBpyVgtoLFEUZYAa/UPCsoVtBgmb5sUa77oEtCDTvvaIoA8zgtykMuL9+QGH7oFn+uiWw9y3L0+h70/qNxWovUBRlABn8K4UB178bPMPZ4mP8Z/mZrqe7N2mqCkVRKsLgFwpeevmyY9JVPhf+xCp87zfj11QViqJUCYNffWQPxD0PXUdkTweJxsMZfs718PuvQ2Jnea7ZNNEqQh8WTVWhKEqVMPhXCgAnzOGVj/+VIw/ezp/OXWFtO7g3v3PE4rkL3Dv75esdpK6niqJUCWUVCiJytoi8KCKviMh8j8+/IiLPicg6EXlERN5VrrbsP2jVUxg+rM5Sy/Qlwx8sUcsecM63s1VRkZgtLIrwDlLXU0VRqoSyqY9EJArcDHwQ2Aw8LSLLjDHPuXZbDbQZY7pE5F+A7wBzy9Ge/QctI+7w+mj+ahnTlz7QlzqlhKaqUBSlSiinTeFU4BVjzGsAInIncAGQEgrGmBWu/Z8ALi1HQ5au7uD6324A4Au3P8Mf44fTmHgra78+iRAxHllQ3WqccrmIquupoihVQDnVRy2AO+HQZnubH58Bfl/qRjj1mXd2WeqirXsPcs3+i+iWYWn7dZl67ug9k55oQ/oJVI2jKMoQoiq8j0TkUqAN+Hufzy8DLgOYNGlSXuf2qs98d/f76IkavhZdzHjZwRZzCN/pmcOyvtN4uf44FjTdo2ocRVGGJOUUCh3ARNf7Cfa2NETkH4BvAH9vjHfRA2PMrcCtAG1tbXlVv8ysz+ywtHcmS3tnZm3/xb5TWfDN6/K5hKIoyqChnOqjp4GjRGSKiNQDHwGWuXcQkVbgx8BsY8zWcjTCXZ/ZTdQn9anf/oqiKEOBsgkFY0wPcDmwHHgeWGKM2SAi14uIU2h4ETACuEtE1ojIMp/TFcy8WVOJx6Jp2+KxKB99z0TP7fNmTS11ExRFUWqGstoUjDEPAA9kbLvG9fofynl9gPZWy7a9aPmLbOlMML45zrxZU2lvbaHtXWM8tyuKogxVxJi8VPQVp62tzaxcubLSzVAURakpRGSVMaYt5361JhREZBvwZoGHjwVKVVuz0mhfqhPtS3WifYF3GWPG5dqp5oRCMYjIyjCSshbQvlQn2pfqRPsSnqGREE9RFEUJhQoFRVEUJcVQEwq3VroBJUT7Up1oX6oT7UtIhpRNQVEURQlmqK0UFEVRlACGjFDIVfCn2hGRN0RkvR35vdLeNkZEHhKRl+3/oyvdTi9E5DYR2Soiz7q2ebZdLL5vP6d1InJS5VqejU9fFohIh/1s1ojIua7Prrb78qKIzKpMq7MRkYkissIucrVBRL5sb6+55xLQl1p8Lg0i8pSIrLX7cp29fYqIPGm3ebGdOggRGWa/f8X+fHLRjTDGDPo/IAq8ChwJ1ANrgWMr3a48+/AGMDZj23eA+fbr+cC3K91On7a/HzgJeDZX24FzsVKoCzADeLLS7Q/RlwXA1zz2Pdb+rg0DptjfwWil+2C37QjgJPv1SOAlu70191wC+lKLz0WAEfbrGPCkfb+XAB+xt98C/Iv9+gvALfbrjwCLi23DUFkppAr+GGO6AafgT61zAfAL+/UvgPYKtsUXY8yfgZ0Zm/3afgHwS2PxBNAsIkcMTEtz49MXPy4A7jTGHDTGvA68gvVdrDjGmLeMMc/Yr/di5SdroQafS0Bf/Kjm52KMMfvstzH7zwBnAHfb2zOfi/O87gbOFPHJ9hmSoSIU8i34U40Y4EERWWXXlwA4zBjjlJB7GzisMk0rCL+21+qzutxWq9zmUuPVRF9slUMr1qy0pp9LRl+gBp+LiERFZA2wFXgIayXTaawko5De3lRf7M93A4cUc/2hIhQGA6cZY04CzgG+KCLvd39orPVjTbqS1XLbbX4EvBuYDrwFfLeyzQmPiIwA7gGuMMbscX9Wa8/Foy81+VyMMb3GmOlYNWhOBY4eyOsPFaEQquBPNWOM6bD/bwV+g/VlecdZwtv/y1KTokz4tb3mnpUx5h37h9wH/IR+VURV90VEYliD6O3GmHvtzTX5XLz6UqvPxcEY0wmsAN6Lpa5zslq725vqi/15E7CjmOsOFaGQs+BPNSMiw0VkpPMaOAt4FqsPn7R3+yRwX2VaWBB+bV8G/KPt7TID2O1SZ1QlGbr1D2M9G7D68hHbQ2QKcBTw1EC3zwtb7/wz4HljzI2uj2ruufj1pUafyzgRabZfx4EPYtlIVgAX27tlPhfneV0MPGqv8Aqn0tb2gfrD8p54CUs/941KtyfPth+J5S2xFtjgtB9Ld/gI8DLwMDCm0m31af8dWMv3JJY+9DN+bcfyvrjZfk7rgbZKtz9EX35lt3Wd/SM9wrX/N+y+vAicU+n2u9p1GpZqaB2wxv47txafS0BfavG5nACsttv8LHCNvf1ILMH1CnAXMMze3mC/f8X+/Mhi26ARzYqiKEqKoaI+UhRFUUKgQkFRFEVJoUJBURRFSaFCQVEURUmhQkFRFEVJoUJBUQYQEfmAiPyu0u1QFD9UKCiKoigpVCgoigcicqmd136NiPzYTlK2T0RusvPcPyIi4+x9p4vIE3bitd+4ahD8jYg8bOfGf0ZE3m2ffoSI3C0iL4jI7cVmtVSUUqJCQVEyEJFjgLnATGMlJusFPg4MB1YaY44D/gRcax/yS+DrxpgTsCJone23AzcbY04E3ocVCQ1WFs8rsPL6HwnMLHunFCUkdbl3UZQhx5nAycDT9iQ+jpUYrg9YbO/za+BeEWkCmo0xf7K3/wK4y85V1WKM+Q2AMeYAgH2+p4wxm+33a4DJwGPl75ai5EaFgqJkI8AvjDFXp20U+feM/QrNEXPQ9boX/R0qVYSqjxQlm0eAi0XkUEjVLX4X1u/FyVT5MeAxY8xuYJeI/J29/RPAn4xVAWyziLTb5xgmIo0D2gtFKQCdoShKBsaY50Tkm1iV7iJYGVG/COwHTrU/24pldwArdfEt9qD/GvBpe/sngB+LyPX2OS4ZwG4oSkFollRFCYmI7DPGjKh0OxSlnKj6SFEURUmhKwVFURQlha4UFEVRlBQqFBRFUZQUKhQURVGUFCoUFEVRlBQqFBRFUZQUKhQURVGUFP8/XzyB970SJ28AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe249cd47f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot training and validation history\n",
    "plt.subplot(2, 1, 1)\n",
    "plt.plot(loss_his, 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(train_accu_his, '-o')\n",
    "plt.plot(val_accu_his, '-o')\n",
    "plt.legend(['train','val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy 0.6 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test set\n",
    "y_test_pred = model(X_test)\n",
    "  \n",
    "_, y_pred = torch.max(y_test_pred,1)\n",
    "test_accu = np.mean(y_pred.data.numpy() ==  y_test.data.numpy())\n",
    "print('Test accuracy', test_accu, '\\n')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}