project_rnn_updated_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from data_utils import get_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (2358, 1000, 22) \n",
      "y_train: (2358,) \n",
      "X_val: (100, 1000, 22) \n",
      "y_val: (100,) \n",
      "X_test: (100, 1000, 22) \n",
      "y_test: (100,) \n"
     ]
    }
   ],
   "source": [
    "# Load data from all .mat files, combine them, eliminate EOG signals, shuffle and \n",
    "# seperate training data, validation data and testing data.\n",
    "# Also do mean subtraction on x.\n",
    "\n",
    "data = get_data('project_datasets',num_validation=100, num_test=100, subtract_mean=False, transpose=True)\n",
    "for k in data.keys():\n",
    "    print('{}: {} '.format(k, data[k].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class flatten to connect to FC layer\n",
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        N, C, H = x.size() # read in N, C, H\n",
    "        return x.view(N, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn x and y into torch type tensor\n",
    "N_train, C_train, H_train = data.get('X_train').shape\n",
    "N_val, C_val, H_val = data.get('X_val').shape\n",
    "N_test, C_test, H_test = data.get('X_test').shape\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "\n",
    "X_train = Variable(torch.Tensor(data.get('X_train'))).type(dtype)\n",
    "y_train = Variable(torch.Tensor(data.get('y_train'))).type(dtype)\n",
    "X_val = Variable(torch.Tensor(data.get('X_val'))).type(dtype)\n",
    "y_val = Variable(torch.Tensor(data.get('y_val'))).type(dtype)\n",
    "X_test = Variable(torch.Tensor(data.get('X_test'))).type(dtype)\n",
    "y_test = Variable(torch.Tensor(data.get('y_test'))).type(dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class for RNN model\n",
    "\n",
    "class RNN(nn.Module):\n",
    "    # input_size: number of EGG channels\n",
    "    # output_size: number of classes\n",
    "    # hidden_size: size of hidden layers in RNN module\n",
    "    # num_layers: number of hidden layers in RNN module\n",
    "    # dropout: non-zero value means applying dropout to hidden layers\n",
    "    def __init__(self, input_size, output_size, hidden_size=10, num_layers=1, dropout=0):\n",
    "        super(RNN, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.output_size = output_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        \n",
    "        self.rnn = nn.RNN(input_size, hidden_size, num_layers, dropout=dropout, batch_first=True)\n",
    "        self.output = nn.Linear(hidden_size, output_size)\n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        \n",
    "    def forward(self, x, y, hidden_init, T_length=1):\n",
    "        N, _, _ = x.size()\n",
    "        out_rnn, state_rnn = self.rnn(x, hidden_init)\n",
    "        score = self.output(out_rnn)\n",
    "        \n",
    "        # calculate the total loss at time interval of T_length\n",
    "        loss = 0\n",
    "        for t in np.arange(T_length):\n",
    "            yy = score[:,-(t+1),:].squeeze(1)\n",
    "            loss += self.loss_fn(yy,y)\n",
    "        # score[N,S,C]:\n",
    "        # N: batch_size(number of time series), S: length of time series, C: number of classes\n",
    "        return score[:,-1,:].squeeze(1), loss, state_rnn\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decare a RNN model\n",
    "\n",
    "N_train, S, input_size = X_train.size()\n",
    "output_size = 4\n",
    "hidden_size = 40\n",
    "num_layers = 2\n",
    "dropout = 1\n",
    "\n",
    "model = RNN(input_size, output_size, hidden_size, num_layers, dropout)\n",
    "model.type(dtype)\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [ 13.86315346]\n",
      "1 [ 13.86517906]\n",
      "2 [ 13.86302185]\n",
      "3 [ 13.86305046]\n",
      "4 [ 13.86305714]\n",
      "5 [ 13.86291695]\n",
      "6 [ 13.86301804]\n",
      "7 [ 13.86350536]\n",
      "8 [ 13.86323833]\n",
      "9 [ 13.86331177]\n",
      "10 [ 13.86328793]\n",
      "11 [ 13.8631525]\n",
      "12 [ 13.86309052]\n",
      "13 [ 13.86264801]\n",
      "14 [ 13.86280155]\n",
      "15 [ 13.86287689]\n",
      "16 [ 13.86288548]\n",
      "17 [ 13.86282921]\n",
      "18 [ 13.86289787]\n",
      "19 [ 13.86278057]\n",
      "20 [ 13.86261082]\n",
      "21 [ 13.86283016]\n",
      "22 [ 13.86262131]\n",
      "23 [ 13.8628273]\n",
      "24 [ 13.86283016]\n",
      "25 [ 13.86286163]\n",
      "26 [ 13.86287594]\n",
      "27 [ 13.86289215]\n",
      "28 [ 13.86281872]\n",
      "29 [ 13.86269474]\n",
      "30 [ 13.86260891]\n",
      "31 [ 13.86284637]\n",
      "32 [ 13.8626442]\n",
      "33 [ 13.86257076]\n",
      "34 [ 13.86245346]\n",
      "35 [ 13.86268425]\n",
      "36 [ 13.86283875]\n",
      "37 [ 13.86292553]\n",
      "38 [ 13.86266804]\n",
      "39 [ 13.86281013]\n",
      "40 [ 13.86263752]\n",
      "41 [ 13.86264706]\n",
      "42 [ 13.86266518]\n",
      "43 [ 13.86285686]\n",
      "44 [ 13.86284924]\n",
      "45 [ 13.86254406]\n",
      "46 [ 13.86261082]\n",
      "47 [ 13.86293507]\n",
      "48 [ 13.86262035]\n",
      "49 [ 13.86252403]\n",
      "50 [ 13.86285591]\n",
      "51 [ 13.86285591]\n",
      "52 [ 13.86284924]\n",
      "53 [ 13.86262035]\n",
      "54 [ 13.8626442]\n",
      "55 [ 13.86263943]\n",
      "56 [ 13.86263943]\n",
      "57 [ 13.86263943]\n",
      "58 [ 13.86285782]\n",
      "59 [ 13.86263657]\n",
      "60 [ 13.86253262]\n",
      "61 [ 13.86252499]\n",
      "62 [ 13.86285019]\n",
      "63 [ 13.86252308]\n",
      "64 [ 13.86275291]\n",
      "65 [ 13.86260891]\n",
      "66 [ 13.86284256]\n",
      "67 [ 13.86274147]\n",
      "68 [ 13.86247635]\n",
      "69 [ 13.86270905]\n",
      "70 [ 13.86254406]\n",
      "71 [ 13.86275196]\n",
      "72 [ 13.86278057]\n",
      "73 [ 13.86262989]\n",
      "74 [ 13.86241531]\n",
      "75 [ 13.86279202]\n",
      "76 [ 13.8629446]\n",
      "77 [ 13.86275196]\n",
      "78 [ 13.86285877]\n",
      "79 [ 13.86263561]\n",
      "80 [ 13.86262989]\n",
      "81 [ 13.862854]\n",
      "82 [ 13.86292267]\n",
      "83 [ 13.86284924]\n",
      "84 [ 13.86284733]\n",
      "85 [ 13.86283779]\n",
      "86 [ 13.86285019]\n",
      "87 [ 13.86285686]\n",
      "88 [ 13.86262798]\n",
      "89 [ 13.86262131]\n",
      "90 [ 13.86276054]\n",
      "91 [ 13.86262989]\n",
      "92 [ 13.86263943]\n",
      "93 [ 13.86272526]\n",
      "94 [ 13.86284637]\n",
      "95 [ 13.86286831]\n",
      "96 [ 13.86286354]\n",
      "97 [ 13.86284637]\n",
      "98 [ 13.86284637]\n",
      "99 [ 13.86263847]\n",
      "Epoch  0 , loss is  [ 13.86263943]\n",
      "Training accuracy 0.252332485157\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "0 [ 13.86263943]\n",
      "1 [ 13.86425591]\n",
      "2 [ 13.86267662]\n",
      "3 [ 13.8626318]\n",
      "4 [ 13.86287594]\n",
      "5 [ 13.86263847]\n",
      "6 [ 13.86285973]\n",
      "7 [ 13.86287308]\n",
      "8 [ 13.86254215]\n",
      "9 [ 13.86298275]\n",
      "10 [ 13.86252403]\n",
      "11 [ 13.86281776]\n",
      "12 [ 13.86261082]\n",
      "13 [ 13.86297321]\n",
      "14 [ 13.86263847]\n",
      "15 [ 13.8627615]\n",
      "16 [ 13.86270523]\n",
      "17 [ 13.86284256]\n",
      "18 [ 13.86262989]\n",
      "19 [ 13.86283779]\n",
      "20 [ 13.86263847]\n",
      "21 [ 13.86270618]\n",
      "22 [ 13.8627634]\n",
      "23 [ 13.86263657]\n",
      "24 [ 13.86263943]\n",
      "25 [ 13.86263943]\n",
      "26 [ 13.86286163]\n",
      "27 [ 13.86263084]\n",
      "28 [ 13.86262798]\n",
      "29 [ 13.86283112]\n",
      "30 [ 13.86263275]\n",
      "31 [ 13.86263752]\n",
      "32 [ 13.8629303]\n",
      "33 [ 13.86253452]\n",
      "34 [ 13.86283779]\n",
      "35 [ 13.86293507]\n",
      "36 [ 13.86298275]\n",
      "37 [ 13.86285019]\n",
      "38 [ 13.86263084]\n",
      "39 [ 13.86262989]\n",
      "40 [ 13.86270905]\n",
      "41 [ 13.86284733]\n",
      "42 [ 13.86255169]\n",
      "43 [ 13.86254406]\n",
      "44 [ 13.86277008]\n",
      "45 [ 13.86276627]\n",
      "46 [ 13.86272335]\n",
      "47 [ 13.86283684]\n",
      "48 [ 13.86264896]\n",
      "49 [ 13.86262131]\n",
      "50 [ 13.86262512]\n",
      "51 [ 13.86262989]\n",
      "52 [ 13.86285019]\n",
      "53 [ 13.86279011]\n",
      "54 [ 13.86273384]\n",
      "55 [ 13.86262512]\n",
      "56 [ 13.86262798]\n",
      "57 [ 13.86253452]\n",
      "58 [ 13.86252308]\n",
      "59 [ 13.862854]\n",
      "60 [ 13.86285019]\n",
      "61 [ 13.86283779]\n",
      "62 [ 13.8627634]\n",
      "63 [ 13.86276054]\n",
      "64 [ 13.86285019]\n",
      "65 [ 13.8627615]\n",
      "66 [ 13.86283493]\n",
      "67 [ 13.86294937]\n",
      "68 [ 13.86262894]\n",
      "69 [ 13.86285019]\n",
      "70 [ 13.8628273]\n",
      "71 [ 13.86284924]\n",
      "72 [ 13.86283875]\n",
      "73 [ 13.86282921]\n",
      "74 [ 13.86283684]\n",
      "75 [ 13.86283684]\n",
      "76 [ 13.86282921]\n",
      "77 [ 13.8627634]\n",
      "78 [ 13.86278152]\n",
      "79 [ 13.86275864]\n",
      "80 [ 13.86276245]\n",
      "81 [ 13.86276245]\n",
      "82 [ 13.86278152]\n",
      "83 [ 13.86278057]\n",
      "84 [ 13.86275864]\n",
      "85 [ 13.86284924]\n",
      "86 [ 13.86285019]\n",
      "87 [ 13.86285686]\n",
      "88 [ 13.86276245]\n",
      "89 [ 13.8627615]\n",
      "90 [ 13.86285019]\n",
      "91 [ 13.86287594]\n",
      "92 [ 13.86287594]\n",
      "93 [ 13.86284924]\n",
      "94 [ 13.86275864]\n",
      "95 [ 13.8627615]\n",
      "96 [ 13.8627615]\n",
      "97 [ 13.8627615]\n",
      "98 [ 13.8627615]\n",
      "99 [ 13.8627615]\n",
      "Epoch  1 , loss is  [ 13.8627615]\n",
      "Training accuracy 0.252332485157\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "0 [ 13.8627615]\n",
      "1 [ 13.86413765]\n",
      "2 [ 13.86271763]\n",
      "3 [ 13.86274719]\n",
      "4 [ 13.8628664]\n",
      "5 [ 13.86279202]\n",
      "6 [ 13.86262512]\n",
      "7 [ 13.86263752]\n",
      "8 [ 13.8626194]\n",
      "9 [ 13.86260128]\n",
      "10 [ 13.86283779]\n",
      "11 [ 13.86261654]\n",
      "12 [ 13.86269379]\n",
      "13 [ 13.86272049]\n",
      "14 [ 13.86277771]\n",
      "15 [ 13.86261749]\n",
      "16 [ 13.86275864]\n",
      "17 [ 13.8628664]\n",
      "18 [ 13.86255264]\n",
      "19 [ 13.86263752]\n",
      "20 [ 13.86256123]\n",
      "21 [ 13.86285877]\n",
      "22 [ 13.86283875]\n",
      "23 [ 13.86260891]\n",
      "24 [ 13.862854]\n",
      "25 [ 13.86285973]\n",
      "26 [ 13.86262703]\n",
      "27 [ 13.86263657]\n",
      "28 [ 13.86283875]\n",
      "29 [ 13.86253452]\n",
      "30 [ 13.86261845]\n",
      "31 [ 13.8629837]\n",
      "32 [ 13.86263752]\n",
      "33 [ 13.86263657]\n",
      "34 [ 13.86253262]\n",
      "35 [ 13.86277008]\n",
      "36 [ 13.86254406]\n",
      "37 [ 13.86271572]\n",
      "38 [ 13.86293411]\n",
      "39 [ 13.86277199]\n",
      "40 [ 13.86282063]\n",
      "41 [ 13.86282825]\n",
      "42 [ 13.86262703]\n",
      "43 [ 13.86252975]\n",
      "44 [ 13.86253548]\n",
      "45 [ 13.86260891]\n",
      "46 [ 13.8627634]\n",
      "47 [ 13.86273384]\n",
      "48 [ 13.86240864]\n",
      "49 [ 13.86249638]\n",
      "50 [ 13.86252403]\n",
      "51 [ 13.86253452]\n",
      "52 [ 13.86262989]\n",
      "53 [ 13.86283684]\n",
      "54 [ 13.86274242]\n",
      "55 [ 13.86274147]\n",
      "56 [ 13.86283779]\n",
      "57 [ 13.86285782]\n",
      "58 [ 13.86283779]\n",
      "59 [ 13.86272526]\n",
      "60 [ 13.86270428]\n",
      "61 [ 13.86263084]\n",
      "62 [ 13.86285019]\n",
      "63 [ 13.86283016]\n",
      "64 [ 13.86283779]\n",
      "65 [ 13.86285019]\n",
      "66 [ 13.86275864]\n",
      "67 [ 13.86283493]\n",
      "68 [ 13.86283493]\n",
      "69 [ 13.8627739]\n",
      "70 [ 13.86275387]\n",
      "71 [ 13.86275864]\n",
      "72 [ 13.86276245]\n",
      "73 [ 13.8627615]\n",
      "74 [ 13.86278343]\n",
      "75 [ 13.86284924]\n",
      "76 [ 13.86285019]\n",
      "77 [ 13.86285686]\n",
      "78 [ 13.86283016]\n",
      "79 [ 13.86283493]\n",
      "80 [ 13.86282921]\n",
      "81 [ 13.86285019]\n",
      "82 [ 13.86285019]\n",
      "83 [ 13.86285019]\n",
      "84 [ 13.86284924]\n",
      "85 [ 13.86276627]\n",
      "86 [ 13.8627634]\n",
      "87 [ 13.86285877]\n",
      "88 [ 13.86285019]\n",
      "89 [ 13.8627615]\n",
      "90 [ 13.86276245]\n",
      "91 [ 13.86278152]\n",
      "92 [ 13.86278248]\n",
      "93 [ 13.86278248]\n",
      "94 [ 13.86278152]\n",
      "95 [ 13.86278915]\n",
      "96 [ 13.8627634]\n",
      "97 [ 13.8627615]\n",
      "98 [ 13.8627615]\n",
      "99 [ 13.8627615]\n",
      "Epoch  2 , loss is  [ 13.8627615]\n",
      "Training accuracy 0.252332485157\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "0 [ 13.8627615]\n",
      "1 [ 13.86388302]\n",
      "2 [ 13.86268997]\n",
      "3 [ 13.86265373]\n",
      "4 [ 13.86275196]\n",
      "5 [ 13.86271572]\n",
      "6 [ 13.86248493]\n",
      "7 [ 13.86262989]\n",
      "8 [ 13.86274433]\n",
      "9 [ 13.86283112]\n",
      "10 [ 13.86269665]\n",
      "11 [ 13.86270905]\n",
      "12 [ 13.86262798]\n",
      "13 [ 13.86285877]\n",
      "14 [ 13.86264038]\n",
      "15 [ 13.86284733]\n",
      "16 [ 13.86271477]\n",
      "17 [ 13.86287308]\n",
      "18 [ 13.86254215]\n",
      "19 [ 13.86283684]\n",
      "20 [ 13.8628397]\n",
      "21 [ 13.86262703]\n",
      "22 [ 13.86282921]\n",
      "23 [ 13.86283779]\n",
      "24 [ 13.86261749]\n",
      "25 [ 13.86240101]\n",
      "26 [ 13.86270332]\n",
      "27 [ 13.86260128]\n",
      "28 [ 13.86271381]\n",
      "29 [ 13.86261177]\n",
      "30 [ 13.86253548]\n",
      "31 [ 13.86260891]\n",
      "32 [ 13.86275196]\n",
      "33 [ 13.86261749]\n",
      "34 [ 13.86271477]\n",
      "35 [ 13.86285877]\n",
      "36 [ 13.86293411]\n",
      "37 [ 13.86277199]\n",
      "38 [ 13.86260605]\n",
      "39 [ 13.86251354]\n",
      "40 [ 13.86262703]\n",
      "41 [ 13.86262989]\n",
      "42 [ 13.8627634]\n",
      "43 [ 13.8627634]\n",
      "44 [ 13.86242867]\n",
      "45 [ 13.86270428]\n",
      "46 [ 13.8624115]\n",
      "47 [ 13.86261177]\n",
      "48 [ 13.86261177]\n",
      "49 [ 13.86273479]\n",
      "50 [ 13.86276245]\n",
      "51 [ 13.8628273]\n",
      "52 [ 13.86281109]\n",
      "53 [ 13.8627615]\n",
      "54 [ 13.8626976]\n",
      "55 [ 13.86272335]\n",
      "56 [ 13.86262894]\n",
      "57 [ 13.86264896]\n",
      "58 [ 13.86285019]\n",
      "59 [ 13.86283493]\n",
      "60 [ 13.86285019]\n",
      "61 [ 13.8627634]\n",
      "62 [ 13.86282921]\n",
      "63 [ 13.86278152]\n",
      "64 [ 13.86276245]\n",
      "65 [ 13.86275482]\n",
      "66 [ 13.86275482]\n",
      "67 [ 13.8627615]\n",
      "68 [ 13.86278534]\n",
      "69 [ 13.86285019]\n",
      "70 [ 13.86285782]\n",
      "71 [ 13.86283779]\n",
      "72 [ 13.86292362]\n",
      "73 [ 13.86283493]\n",
      "74 [ 13.86283684]\n",
      "75 [ 13.86285686]\n",
      "76 [ 13.86285019]\n",
      "77 [ 13.86283112]\n",
      "78 [ 13.86276245]\n",
      "79 [ 13.8627634]\n",
      "80 [ 13.86284924]\n",
      "81 [ 13.86284924]\n",
      "82 [ 13.86284924]\n",
      "83 [ 13.8627615]\n",
      "84 [ 13.86278152]\n",
      "85 [ 13.86276054]\n",
      "86 [ 13.86276054]\n",
      "87 [ 13.8627615]\n",
      "88 [ 13.8627615]\n",
      "89 [ 13.8627615]\n",
      "90 [ 13.86275864]\n",
      "91 [ 13.86275864]\n",
      "92 [ 13.8627615]\n",
      "93 [ 13.86275482]\n",
      "94 [ 13.86275864]\n",
      "95 [ 13.86275482]\n",
      "96 [ 13.86275482]\n",
      "97 [ 13.86275864]\n",
      "98 [ 13.8627615]\n",
      "99 [ 13.8627615]\n",
      "Epoch  3 , loss is  [ 13.8627615]\n",
      "Training accuracy 0.252332485157\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "0 [ 13.8627615]\n",
      "1 [ 13.86360073]\n",
      "2 [ 13.86283684]\n",
      "3 [ 13.86286163]\n",
      "4 [ 13.86288738]\n",
      "5 [ 13.86285019]\n",
      "6 [ 13.86277103]\n",
      "7 [ 13.8628397]\n",
      "8 [ 13.86261749]\n",
      "9 [ 13.86273479]\n",
      "10 [ 13.86282063]\n",
      "11 [ 13.86262989]\n",
      "12 [ 13.86253548]\n",
      "13 [ 13.86274242]\n",
      "14 [ 13.86261845]\n",
      "15 [ 13.86264706]\n",
      "16 [ 13.8629446]\n",
      "17 [ 13.86253452]\n",
      "18 [ 13.8627634]\n",
      "19 [ 13.86286163]\n",
      "20 [ 13.86262989]\n",
      "21 [ 13.8627243]\n",
      "22 [ 13.86253548]\n",
      "23 [ 13.86248779]\n",
      "24 [ 13.86262512]\n",
      "25 [ 13.86262798]\n",
      "26 [ 13.86269665]\n",
      "27 [ 13.86263847]\n",
      "28 [ 13.86262798]\n",
      "29 [ 13.86262989]\n",
      "30 [ 13.86277294]\n",
      "31 [ 13.86254215]\n",
      "32 [ 13.86263943]\n",
      "33 [ 13.86285973]\n",
      "34 [ 13.86286736]\n",
      "35 [ 13.86242104]\n",
      "36 [ 13.86255169]\n",
      "37 [ 13.86251354]\n",
      "38 [ 13.86272049]\n",
      "39 [ 13.8627634]\n",
      "40 [ 13.86273479]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41 [ 13.86242867]\n",
      "42 [ 13.86261749]\n",
      "43 [ 13.86263657]\n",
      "44 [ 13.86262989]\n",
      "45 [ 13.86263847]\n",
      "46 [ 13.86273956]\n",
      "47 [ 13.86282921]\n",
      "48 [ 13.86283779]\n",
      "49 [ 13.86275864]\n",
      "50 [ 13.86272049]\n",
      "51 [ 13.86272335]\n",
      "52 [ 13.86262798]\n",
      "53 [ 13.86264896]\n",
      "54 [ 13.86284924]\n",
      "55 [ 13.86283684]\n",
      "56 [ 13.86262894]\n",
      "57 [ 13.8627634]\n",
      "58 [ 13.86285591]\n",
      "59 [ 13.8627615]\n",
      "60 [ 13.86274147]\n",
      "61 [ 13.86283779]\n",
      "62 [ 13.86283016]\n",
      "63 [ 13.86274147]\n",
      "64 [ 13.86275482]\n",
      "65 [ 13.86284924]\n",
      "66 [ 13.86274242]\n",
      "67 [ 13.86283112]\n",
      "68 [ 13.86283779]\n",
      "69 [ 13.86283684]\n",
      "70 [ 13.86285686]\n",
      "71 [ 13.86284733]\n",
      "72 [ 13.86283016]\n",
      "73 [ 13.8627634]\n",
      "74 [ 13.8627634]\n",
      "75 [ 13.86284924]\n",
      "76 [ 13.86284733]\n",
      "77 [ 13.86285019]\n",
      "78 [ 13.86278534]\n",
      "79 [ 13.8627634]\n",
      "80 [ 13.8627634]\n",
      "81 [ 13.8627615]\n",
      "82 [ 13.86278248]\n",
      "83 [ 13.86276245]\n",
      "84 [ 13.86275864]\n",
      "85 [ 13.86275864]\n",
      "86 [ 13.8627634]\n",
      "87 [ 13.86276245]\n",
      "88 [ 13.86275387]\n",
      "89 [ 13.86285019]\n",
      "90 [ 13.86284924]\n",
      "91 [ 13.86275864]\n",
      "92 [ 13.8627615]\n",
      "93 [ 13.8627615]\n",
      "94 [ 13.8627615]\n",
      "95 [ 13.86276245]\n",
      "96 [ 13.8627615]\n",
      "97 [ 13.8627615]\n",
      "98 [ 13.86278152]\n",
      "99 [ 13.86278152]\n",
      "Epoch  4 , loss is  [ 13.86275482]\n",
      "Training accuracy 0.252332485157\n",
      "Validation accuracy 0.24 \n",
      "\n",
      "0 [ 13.86275482]\n",
      "1 [ 13.86345673]\n",
      "2 [ 13.86270714]\n",
      "3 [ 13.8626585]\n",
      "4 [ 13.86253262]\n",
      "5 [ 13.86264706]\n",
      "6 [ 13.86254406]\n",
      "7 [ 13.86273384]\n",
      "8 [ 13.86274242]\n",
      "9 [ 13.86280155]\n",
      "10 [ 13.86261368]\n",
      "11 [ 13.86270332]\n",
      "12 [ 13.8625164]\n",
      "13 [ 13.86263752]\n",
      "14 [ 13.86262512]\n",
      "15 [ 13.86286831]\n",
      "16 [ 13.86262989]\n",
      "17 [ 13.86263847]\n",
      "18 [ 13.86262798]\n",
      "19 [ 13.86253548]\n",
      "20 [ 13.86276436]\n",
      "21 [ 13.86262512]\n",
      "22 [ 13.86253262]\n",
      "23 [ 13.86253357]\n",
      "24 [ 13.86272049]\n",
      "25 [ 13.86282825]\n",
      "26 [ 13.86263657]\n",
      "27 [ 13.86262894]\n",
      "28 [ 13.86274433]\n",
      "29 [ 13.86266518]\n",
      "30 [ 13.86263943]\n",
      "31 [ 13.86285877]\n",
      "32 [ 13.86277294]\n",
      "33 [ 13.86242294]\n",
      "34 [ 13.86252594]\n",
      "35 [ 13.86253166]\n",
      "36 [ 13.86262989]\n",
      "37 [ 13.862854]\n",
      "38 [ 13.86254501]\n",
      "39 [ 13.8624115]\n",
      "40 [ 13.86262989]\n",
      "41 [ 13.86262703]\n",
      "42 [ 13.86262989]\n",
      "43 [ 13.86283779]\n",
      "44 [ 13.86273956]\n",
      "45 [ 13.86283779]\n",
      "46 [ 13.86284924]\n",
      "47 [ 13.86284924]\n",
      "48 [ 13.86270428]\n",
      "49 [ 13.86262798]\n",
      "50 [ 13.86260128]\n",
      "51 [ 13.86285019]\n",
      "52 [ 13.86261177]\n",
      "53 [ 13.86263275]\n",
      "54 [ 13.86285019]\n",
      "55 [ 13.86281109]\n",
      "56 [ 13.8627634]\n",
      "57 [ 13.86283016]\n",
      "58 [ 13.86283684]\n",
      "59 [ 13.86273956]\n",
      "60 [ 13.86278915]\n",
      "61 [ 13.86285019]\n",
      "62 [ 13.86274242]\n",
      "63 [ 13.86283684]\n"
     ]
    }
   ],
   "source": [
    "# train through several iterations\n",
    "num_epoch = 10\n",
    "batch_size = 2358\n",
    "\n",
    "T_num_epoch = 100\n",
    "T_batch_size = int(S/T_num_epoch)\n",
    "step = np.arange(0,N_train+1,batch_size)\n",
    "#step = np.append(step,N_train) #discard some data\n",
    "\n",
    "loss_his = []\n",
    "train_accu_his = []\n",
    "val_accu_his = []\n",
    "\n",
    "for epoch in range(num_epoch):\n",
    "    for t in range(step.shape[0]-1):\n",
    "        N = step[t+1] - step[t]\n",
    "        for T_epoch in range(T_num_epoch):  \n",
    "            print(T_epoch, loss.data.numpy())\n",
    "            if T_epoch == 0: \n",
    "                hidden_init = Variable(torch.zeros(num_layers, int(N), hidden_size))\n",
    "\n",
    "            # calculate loss\n",
    "            X_train_sub = X_train[step[t]:step[t+1],:(T_epoch+1)*T_batch_size,:]    \n",
    "            y_train_pred, loss, _ = model.forward(X_train_sub, y_train[step[t]:step[t+1]].type(torch.LongTensor), \n",
    "                                                      hidden_init, T_length=T_batch_size)\n",
    "            # backpropagation\n",
    "            model.zero_grad()\n",
    "            loss.backward()\n",
    "\n",
    "            # update parameters\n",
    "            optimizer.step()\n",
    "        \n",
    "    # calculate predicted value for validation\n",
    "    hidden_init = Variable(torch.zeros(num_layers, N_val, hidden_size))\n",
    "    y_val_pred,_,_ = model(X_val,y_val.type(torch.LongTensor),hidden_init)\n",
    "\n",
    "    # training loss\n",
    "    print('Epoch ', epoch, ', loss is ', loss.data.numpy())\n",
    "    _, y_pred = torch.max(y_train_pred,1)\n",
    "    loss_his.append(loss.data.numpy())\n",
    "    \n",
    "    # training accuracy\n",
    "    train_accu = np.mean(y_pred.data.numpy() == \n",
    "                                       y_train.data[step[t]:step[t+1]].numpy())\n",
    "    print('Training accuracy', train_accu)\n",
    "    train_accu_his.append(train_accu)\n",
    "\n",
    "    # validation accuracy    \n",
    "    _, y_pred = torch.max(y_val_pred,1)\n",
    "    val_accu = np.mean(y_pred.data.numpy() ==  y_val.data.numpy())\n",
    "    print('Validation accuracy', val_accu, '\\n')   \n",
    "    val_accu_his.append(val_accu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(threshold=np.nan)\n",
    "y_temp = y_pred.data.numpy()\n",
    "#print(y_temp)\n",
    "np.isnan(np.sum(y_temp))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VOXZ//HPRfawJBhAhaChFZFVqNFaeVrXKmpVVIqi2D5qte1Ta22fUsW21tL2p5VWq61arbXLUysiIC5VcUOsVq0gqyKCViRB2SRhCxDC9fvjnEC2SSaZTM4k+b5fr7ySOXPmzDUDyXfu+z7nvs3dERERaakuURcgIiLtm4JEREQSoiAREZGEKEhERCQhChIREUmIgkRERBKiIBERkYQoSEREJCFJCxIzu9/M1pvZshrbfmZmS8xskZk9Y2Z9Yzz2l2a2LPy6oMb2AWb2upmtNLOHzCwzWfWLiEh8LFlXtpvZF4BtwF/dfVi4rYe7bwl/vhoY4u7fqPO4M4FrgNOBLGAecJK7bzGz6cAsd59mZr8HFrv73U3V0qtXLy8qKmrFVyci0vEtWLBgo7v3bmq/9GQV4O4vmVlRnW1batzsCjSUYkOAee6+B9hjZouBMWb2MHAScFG431+AG4Emg6SoqIj58+c39yWIiHRqZrY6nv2SFiSxmNkvgK8A5cCJDeyyGPiJmd0K5Ib7vA0UAGVhwACUAP2SVefshaVMnbOCtWUV9M3PYdJpgxg7KmlPJyLSbrX5YLu7/9Dd+wMPAFc1cP8zwJPAv4AHgVeBPYA1dLhYz2NmV5rZfDObv2HDhmbVOHthKZNnLaW0rAIHSssqmDxrKbMXljbrOCIinUGUZ239HTi/oTvc/RfuPtLdv0gQICuBjUC+mVW3ogqBtbEO7u73unuxuxf37t1kF18tU+esoKKyqta2isoqps5Z0azjiIh0Bm0aJGY2sMbNs4F3GtgnzcwKwp9HACOAZzw4K2AuMC7c9avAo8moc21ZRbO2i4h0ZkkbIzGzB4ETgF5mVgL8BDjDzAYBe4HVwDfCfYuBb7j714AM4J9mBrAFmFhjXORaYJqZ/RxYCPwxGbX3zc+htIHQ6Jufk4ynExFp15J51taEBjY3+Iff3ecDXwt/3klw5lZD+70PHNNaNcYy6bRBTJ61tFb3Vk5GGpNOG5TspxYRaXfa/Kyt9qD67CydtSUi0jQFSQxjR/VTcIiIxEFzbYmISEIUJCIikhAFiYiIJERBIiIiCVGQiIhIQhQkIiKSEAWJiIgkREEiIiIJUZCIiEhCFCQiIpIQBYmIiCREQSIiIglRkIiISEIUJCIikhAFiYiIJERBIiIiCVGQiIhIQhQkIiKSEAWJiIgkREEiIiIJUZCIiEhCFCQiIpIQBYmIiCREQSIiIglRkIiISEIUJCIikhAFiYiIJERBIiIiCVGQiIhIQhQkIiKSEAWJiIgkJD3qAqRxsxeWMnXOCtaWVdA3P4dJpw1i7Kh+UZclIrKPgiSFzV5YyuRZS6morAKgtKyCybOWAihMRCRlxNW1ZWbfMbMeFvijmb1pZqcmu7jObuqcFftCpFpFZRVT56yIqCIRkfriHSO5zN23AKcCvYFLgZuTVpUAsLasolnbRUSiEG+QWPj9DOBP7r64xrbYDzK738zWm9myGtt+ZmZLzGyRmT1jZn1jPPYWM3vLzJab2R1mZuH2CWa2NDzG02bWK87X0O70zc9p1nYRkSjEGyQLzOwZgiCZY2bdgb1xPO7PwJg626a6+wh3Hwk8AdxQ90FmdhwwGhgBDAOOBo43s3TgduBEdx8BLAGuivM1tDuTThtETkZarW05GWlMOm1QRBWJiNQX72D75cBI4H1332FmBxB0bzXK3V8ys6I627bUuNkV8IYeCmQDmQQtnwxgXfizAV3NbBPQA1gV52tod6oH1HXWloiksniD5HPAInffbmYTgc8QtAxaxMx+AXwFKAdOrHu/u79qZnOBjwiC43fuvjx87DeBpcB2YCXwrRjPcSVwJcAhhxzS0lIjN3ZUPwWHiKS0eLu27gZ2mNmRwA+A1cBfW/qk7v5Dd+8PPEADXVNmdhgwGCgE+gEnmdkXzCwD+CYwCuhL0LU1OcZz3Ovuxe5e3Lt375aWKiIiTYg3SPa4uwPnALe7++1A91Z4/r8D5zew/VzgNXff5u7bgKeAYwm613D398J6pgPHtUIdIiLSQvEGyVYzmwxcAvzDzNIIxi2azcwG1rh5NvBOA7t9SDi4HrZCjgeWA6XAEDOrbmJ8MdwuIiIRiXeM5ALgIoLrST42s0OAqU09yMweBE4AeplZCfAT4AwzG0Rw1tdq4BvhvsXAN9z9a8AM4CSCsRAHnnb3x8P9fgq8ZGaV4eP/O87XICIiSWBBD1EcO5odSHAaLsC/3X190qpqZcXFxT5//vyoyxARaVfMbIG7Fze1X7xTpIwH/g18GRgPvG5m4xIrUUREOoJ4u7Z+CBxd3QoJxyieI+iCEhGRTizewfYudbqyNjXjsSIi0oHF2yJ52szmAA+Gty8AnkxOSZKKtC6KiMQSV5C4+yQzO59g/isD7nX3R5JamaQMrYsiIo2Je2Erd58JzExiLZKiGlsXRUEiIo0GiZltpeFJFQ1wd++RlKokpWhdFBFpTKNB4u6tMQ2KtHN983MobSA0tC6KiIDOvJI4aF0UEWlM3GMk0nlpXRQRaYyCROKidVFEJBYFibQbupZFJDUpSKRd0LUsIqlLg+3SLjR2LYuIREtBIu2CrmURSV3q2pJ2IZWuZdFYjUhtapFIu5Aq17JUj9WUllXg7B+rmb2wtE3rEEklChJpF8aO6sdN5w2nX34OBvTLz+Gm84a3eUtAYzUi9alrS9qNVLiWRWM1IvUpSESaQWM1IvWpa0ukGTRWI1KfgkSkGTRWI1KfurZEmkljNSK1KUhE2iGN1UgqUdeWSDuksRpJJQoSkXZIYzWSStS1JdJOaaxGUoVaJCLSYrHGZKIYq5HoKEhEpMVSZawGgvGa0Te/wIDr/sHom1/QOE0bUteWiLRYddda1GdtaeGzaClIRCQhqTBW09igf9S1dQbq2hKRdk+D/tFSkIhIu6dB/2gpSESk3dOgf7Q0RiIi7Z4G/aOlIBGRDkGD/tFJWteWmd1vZuvNbFmNbT8zsyVmtsjMnjGzvjEee4uZvWVmy83sDjOzcHummd1rZu+a2Ttmdn6y6hcRaa7OOuifzDGSPwNj6myb6u4j3H0k8ARwQ90HmdlxwGhgBDAMOBo4Prz7h8B6dz8cGALMS07pIiLN11kH/ZMWJO7+EvBJnW1batzsCnhDDwWygUwgC8gA1oX3XQbcFB5rr7tvbOWyRURaLFUG/dt6wL/Nx0jM7BfAV4By4MS697v7q2Y2F/gIMOB37r7czPLDXX5mZicA7wFXufu6uscQEYlCKgz6RzHgb+4NNQpa6eBmRcAT7j6sgfsmA9nu/pM62w8DbgcuCDc9C1wLvA1sAMa5+0wz+x4wyt0vifHcVwJXAhxyyCFHrV69ulVek4hIKht98wsNLnrWLz+HV647qVnHMrMF7l7c1H5RXkfyd6ChwfJzgdfcfZu7bwOeAo4FNgE7gEfC/R4GPhPr4O5+r7sXu3tx7969W7dyEZEUFcWAf5sGiZkNrHHzbOCdBnb7EDjezNLNLINgoH25B02nx4ETwv1OJmiliIhIKIoB/2Se/vsg8CowyMxKzOxy4GYzW2ZmS4BTge+E+xab2X3hQ2cQjH8sBRYDi9398fC+a4Ebw8dfAvxvsuoXEWmPohjwT+oYSaooLi72+fPnR12GiEibmL2wtFUG/OMdI+kUQWJmG4CWjrb3AnSa8X56P/bTe1Gb3o/aOsL7cai7NznI3CmCJBFmNj+eRO4s9H7sp/eiNr0ftXWm90Oz/4qISEIUJCIikhAFSdPujbqAFKP3Yz+9F7Xp/ait07wfGiMREZGEqEUiIiIJUZCIiEhCFCSNMLMxZrbCzFaZ2XVR1xMVM+tvZnPDhcbeMrPvRF1TKjCzNDNbaGZPRF1L1Mws38xmhAvOLTezz0VdU1TM7Lvh78kyM3vQzLKjrinZFCQxmFkacCdwOsEiWhPMbEi0VUVmD/C/7j6YYALNb3Xi96Km7wDLoy4iRdwOPO3uRwBH0knfFzPrB1wNFIeznqcBF0ZbVfIpSGI7Bljl7u+7+25gGnBOxDVFwt0/cvc3w5+3EvyR6LgLUMfBzAqBM4H7mtq3ozOzHsAXgD8CuPtudy+LtqpIpQM5ZpYO5AJrI64n6RQksfUD1tS4XUIn/+MJ+9aYGQW8Hm0lkfsN8ANgb9SFpIBPEawV9Kewq+8+M+sadVFRcPdS4FcEs5h/BJS7+zPRVpV8CpLYrIFtnfpcaTPrBswErqmzbHKnYmZfAta7+4Koa0kR6QRrA93t7qOA7UCnHFM0s54EPRcDgL5AVzObGG1Vyacgia0E6F/jdiGdoIkaS7g2zEzgAXefFXU9ERsNnG1mHxB0eZ5kZn+LtqRIlQAl7l7dSp1BI4vOdXCnAP9x9w3uXgnMAo6LuKakU5DE9gYw0MwGmFkmwYDZYxHXFAkzM4L+7+XufmvU9UTN3Se7e6G7FxH8v3jB3Tv8p85Y3P1jYI2ZVS940ZkXnfsQONbMcsPfm5PpBCcepEddQKpy9z1mdhUwh+DMi/vd/a2Iy4rKaIKFxJaa2aJw2/Xu/mSENUlq+TbwQPih633g0ojriYS7v25mM4A3Cc52XEgnmCpFU6SIiEhC1LUlIiIJUZCIiEhCFCQiIpKQTjHY3qtXLy8qKoq6DBGRdmXBggUb41mzvVMESVFREfPnz4+6jPZtyXR4fgqUl0BeIZx8A4wYH3VVIpJEZrY6nv06RZBIgpZMh8evhsqK4Hb5muA2KExERGMkEofnp+wPkWqVFcF2Een0FCTStPKS5m0XkU6l03ZtVVZWUlJSws6dO6MuJamys7MpLCwkIyMjgYPkwc4GZgXPK2z5MUWkw+i0QVJSUkL37t0pKioimBKn43F3Nm3aRElJCQMGDGjZQV69MwgRSwOv2r/dusDx17ZOoSLSrnXarq2dO3dSUFDQYUMEwMwoKChoeavrX7+DOdfDkHPgnDshrz9gkFMAvhcWT4Pd21u1ZhFpfzptiwTo0CFSrcWv8V+/hWd+BEPGwvn3QVoGjJyw//6lM2DWFfDAl+Gi6ZDVrXUKFpF2p9O2SKJWVlbGXXfd1ezHnXHGGZSVJXkV01fuqB8idQ0fF9z34WtBmOzaltyaRCRlKUjiNHthKaNvfoEB1/2D0Te/wOyFpQkdL1aQVFVVNbD3fk8++ST5+fkJPXejXrkdnv0xDD0Xzv9jwyFSbdj5QZiseV1hItKJKUjiMHthKZNnLaW0rAIHSssqmDxraUJhct111/Hee+8xcuRIjj76aE488UQuuugihg8fDsDYsWM56qijGDp0KPfeu385g6KiIjZu3MgHH3zA4MGDueKKKxg6dCinnnoqFRUVsZ4uPi//Bp69AYaeB+fdB2lx9HwOOw/G/TEMk3Gwa2tiNYhIu5PUMRIzGwPcTrAw1H3ufnOd+78BfAuoArYBV7r72+F9k4HLw/uudvc58RyzJX76+Fu8vTb2EuQLPyxjd9XeWtsqKqv4wYwlPPjvDxt8zJC+PfjJWUNjHvPmm29m2bJlLFq0iBdffJEzzzyTZcuW7Tu76v777+eAAw6goqKCo48+mvPPP5+CgoJax1i5ciUPPvggf/jDHxg/fjwzZ85k4sQWLtT38m3w3I1BK+Pce+MLkWpDzw2+z7gc/jYOJs6ArO4tq0NE2p2ktUjMLA24EzgdGAJMMLMhdXb7u7sPd/eRwC3AreFjhxAsYToUGAPcZWZpcR6z1dUNkaa2t8QxxxxT6xTdO+64gyOPPJJjjz2WNWvWsHLlynqPGTBgACNHjgTgqKOO4oMPPmjZk//z1paHSLWh5wYtk5I3gjBRy0Sk00hmi+QYYJW7vw9gZtOAc6ixlrO712wGdAWql2s8B5jm7ruA/5jZqvB4NHXMlmis5QAw+uYXKC2r323ULz+Hh77+uUSeep+uXbvu+/nFF1/kueee49VXXyU3N5cTTjihwVN4s7Ky9v2clpbWsq6tf/46mOpk2Dg4956WhUi1oecCBjMug7+dDxfPgOweLT+eiLQLyRwj6QesqXG7JNxWi5l9y8zeI2iRXN3EY+M6ZmubdNogcjLSam3LyUhj0mmDWnzM7t27s3Vrw5/ay8vL6dmzJ7m5ubzzzju89tprLX6eRr30qyBEhn858RCpNnQsfPlPULogCJOdsbsMRaRjSGaQNHQBQ70F4t39Tnf/NHAt8KMmHhvXMQHM7Eozm29m8zds2BBnyQ0bO6ofN503nH75ORhBS+Sm84YzdlTLM6ygoIDRo0czbNgwJk2aVOu+MWPGsGfPHkaMGMGPf/xjjj322ITqb9BLU+GFnwUhMvb3rRMi1YacA+P+BGvfVJiIdALm3uDf4cQPbPY54EZ3Py28PRnA3W+KsX8XYLO759Xd18zmADeGu8Z9zGrFxcVedz2S5cuXM3jw4Ja9uHam3mudNxXm/hyGj4dzfw9d0mI/OKEnfhwe/m/oOwomzgzm7BKRdsPMFrh7cVP7JbNF8gYw0MwGmFkmweD5YzV3MLOBNW6eCVSPKD8GXGhmWWY2ABgI/DueY0oT5t0ShMiIC5IbIgCDz4Iv/xnWLoT/Ow92lifvuUQkMkkLEnffA1wFzAGWA9Pd/S0zm2JmZ4e7XWVmb5nZIuB7wFfDx74FTCcYRH8a+Ja7V8U6ZrJeQ4fz4i9h7i9gxIUw9u7khki1wWfBl/8CHy1SmIh0UEnr2kol6tpazuB1j8KLN8GRE4IJGNsiRGp65x8w/atw8AiYOAtyknh1voi0ilTo2pJUsbM8DJGLogkRgCPOhPF/hY+WwP+dCxVJni9MRNqMgqSj2/JRECQjL4ZzfhdNiFQ74gy44P/g46UKE5EOREHSUbkHIbLtY8jsBmf/NtoQqTbo9BphMhYqNkddkYgkSEHSTnTr1oz1Ptxh68dBiOQcADk9UyNEqg06HS74G6x7C/6qMBFpdUumw23D4Mb84PuS6Ul9OgVJvNr4H6bFaoZI7gGQfwik4gJeg8YEYbL+bYWJSGtaMh0evxrK1wAefH/86qT+zVKQxCMJ/zDXXnttrfVIbrzxRn76059y8skn85nPfIbhw4fz6KOPNu+g7rA17M7KLYC8FA2RaoefBhc8EIbJObDjk6grEmn/np8ClXXm3ausCLYniU7/BXjquqDPPpaSN6BqV/3taVlQeHTDjzloOJwee4b7hQsXcs011zBv3jwAhgwZwtNPP01+fj49evRg48aNHHvssaxcuRIzo1u3bmzb1sjCUftCZF0YIv33hUjKn+r87jPw0MXQ+wj4yqNBS0pEWubGfBqeOcrgxuad4KLTf1tTQyHS2PY4jBo1ivXr17N27VoWL15Mz549Ofjgg7n++usZMWIEp5xyCqWlpaxbt67pgzUSIu3C4afChX+HDe+oZSKSqNyChrfnFSbtKeOaqc/MZgL3A0+5e+stwpEqGmk5AMGYSPma+tvz+sOl/2jx044bN44ZM2bw8ccfc+GFF/LAAw+wYcMGFixYQEZGBkVFRQ1OH19Lew+RagO/GITJtIuDMFHLRKT5Vj4XjjcatVolGTlw8g1Je9p4WyR3AxcBK83sZjM7ImkVpaKTbwj+IWpqhX+YCy+8kGnTpjFjxgzGjRtHeXk5ffr0ISMjg7lz57J69erGD9BRQqRadZhsWAF/PVstE5HmWPkcTLsIDhwKX7o1+HuABd/PugNGjE/aU8fVInH354DnzCwPmAA8a2ZrgD8Af3P3yqRVmAqq/wGenwLlJUET8eQbEv6HGTp0KFu3bqVfv34cfPDBXHzxxZx11lkUFxczcuRIjjiikbx2h61rYdv6jhEi1QaeAhP+Dg9eFITJVx5Ty0SkKSufDUKk5jhj8WVt9vRxD7abWQEwEbgEWAs8APwXMNzdT0hWga2hw821VStEegXB1kiItMvXuuq5IEx6HR78YnSN0e8r0tlVn6zSZzBcMrtVP3i16mC7mc0C/gnkAme5+9nu/pC7fxtoxpVykjB32BJ/iLRbh50CEx6ETSuDlsn2TVFXJJJ63p2TtBBpjnjHSH7n7kPc/SZ3/6jmHfGklbSS6hDZ3sFDpNphJ4dhskphIlLXiqfhoYnQZ0jkJ6fEu77qYDN7093LAMysJzDB3e9q4nGSqB2fBAPqVbvBuoDv7RwhUu3TJ8GEafDghfD7z4OFc4i10jhViyyZ3urjZSLNUh0iBw2DSx4JpkGKULwtkiuqQwTA3TcDVySnpLaT8hdj7vgkOO24andw2/cCBpld4w6RlH+N8fj0ifDZb8LW0qBF1kbTPjQoguknRGpZ8VSNEJkdeYhA/C2SLmZmHv5VMrM0IDN5ZSVfdnY2mzZtoqCgAEvVT/ZbPwrDo6bwlN84mrHuzqZNm8jOzk5OfW1p2Yz62yor4IlrYPUrbVfH0odjTz+hVokk2ztPwvSvBDNnXPJIyiwQF2+QzAGmm9nvCa5y+QbBErjtVmFhISUlJWzYsCHqUmIrK4l93yfxNSazs7MpLEzeFa1tpjzGe7F7e/AJra3s3t7w9lj1ibSWFF5lNN4guRb4OvBNgksmnwHuS1ZRbSEjI4MBAwZEXUbjbhvX8B+ovP7w3WVtX0+U8gpjzy7Qlu9FrFkOMnKCrkhd8yLJUDNELnkEsvOirqiWuD7Wuvted7/b3ce5+/nufo+7VzX1ODMbY2YrzGyVmV3XwP3fM7O3zWyJmT1vZoeG2080s0U1vnaa2djwvj+b2X9q3DeyuS+63Th0dP1tSZ7qIGUlaXaBVqmjS3rQvXXXsUHXg0hrWv5E0J118JEpGSIQ/3UkA81sRvhH//3qryYekwbcCZwODAEmmNmQOrstBIrdfQQwA7gFwN3nuvtIdx8JnATsIGgFVZtUfb+7L4rnNbQ7Wz8Oumz6DGnTqQ5S1ojxwWuP+r1oqI6xd8PX50HXPjBtAsy8QtO7SOtY/jg8/FXoOwoumZWSIQLxd239CfgJcBtwInApQRdXY44BVrn7+wBmNg04B3i7egd3n1tj/9cIrpyvaxzBZJE74qy1Y3j2BtizM1j8qeDTUVeTGkaMT40QjVXHFS/Ay7fCS1Ph/RfhS7fB4C+1eXnSQbz9GMy4NAiRibMgu0fUFcUU7+m/Oe7+PMGUKqvd/UaClkJj+gE1O5NLwm2xXA40NGp6IfBgnW2/CLvDbjOzrCbqaH8+eAWWPATHXa0QaU/SM+GE6+DKF6H7gcEVxzMu14WU0nxvPxqGyGdSPkQg/iDZaWZdCGb/vcrMzgX6NPGYhlosDV7UYGYTgWJgap3tBwPDCc4aqzYZOAI4GjiA4ESAho55pZnNN7P5KX1mVl1VlfDk94PVDT//v1FXIy1x0HC4Yi6c+MPgD8Jdnw0+XYrE4+1H4eFLod9RMHFmyocIxB8k1xDMs3U1cBRBF9RXm3hMCdC/xu1CgskeazGzU4AfAme7e92VosYDj9ScXdjdP/LALoIut2MaenJ3v9fdi929uHfv3k2UmkL+fW+w9OzpN0NmbtTVSEulZcDxPwhaJz36wvRLgj8O2zdGXZmksrdmB/9PCovbTYhAHEESDpqPd/dt7l7i7peGZ2691sRD3wAGmtkAM8sk6KKq9bHMzEYB9xCEyPoGjjGBOt1aYSsFC64iHAt0nPNgt3wEc2+CgafCoDOirkZaw0HD4GvPw0k/CgZO7/xs8MdCpK63HoEZlwXLd0+cCVndo64obk0GSXia71HWzMu/3X0PcBVBt9RyYLq7v2VmU8zs7HC3qQSzBz8cnsq7L2jMrIigRTOvzqEfMLOlwFKgF/Dz5tSV0p75UTAdyum/7BzzaHUWaRnwhUnw9ZeC62Ee/mpwTcC2dtTlKsm1bFYwntb/GJg4o12FCMS5HomZ/RoYCDwM7Lu0191nJa+01tPQeiQp5z8vwV/OguOvhROvj7oaSZaqPfCv2+HFm4M/Fmf8CoadF3VVEqVlM4NTxvsfAxc/nFIh0qrrkRAMam8iOFPrrPBL5zW2lqpK+Mf3If9Q+K/vRl2NJFNaenASxddfCv69Z1wKD10SrC8jnc++EPksXNz+WiLV4l1q99JkF9KpvXY3bFwBEx6qf9W0dEx9BsPlz8Krv4W5/w8+eBnOmArDzle3ZmexdAbMugIO+RxcNB2y2u8agXEFiZn9iQZO3XX3tlsUuKMqLw26OQ4/HQaNiboaaUtp6UEL9PDT4dH/gZmXBwOuZ94aXIci7dLshaVMnbOCtWUV9M3PYdJpgxg7qs4ldB0oRCD+K9ufqPFzNnAuDZzKKy3wzA/Bq4LTfaVz6nMEXPYMvHYnvPCL4LqT06fC8HFqnbQzsxeWMnnWUioqg6kIS8sqmDxrKcD+MFnyMDxyJRxyHFw8PVhfKAl1NBlmrSjerq2ZNW+b2YPAc0mpqDN5b27wCfSE66FnUdTVSJTS0mH0d/a3TmZ9Lfi/8aVboftBUVcncfrl0+/sC5FqFZVVTHnibfJyM+j34eMM/Nf32X7QZ1l7yv1kboHM9Aoy07uQmd6FrPQuZKZ1SWiNpLjCrJXFddZWvQeZDQL+4e6HtX5Jra8lZ20lPdH37Ia7j4O9lfA/r0NGB1h8SlrH3ip47S544eeQng2n3xLM7aXWScoo31HJu+u38u66raxct4131wU/b9y2O+ZjxnZ5mV9n3M3rewdzeeX3qSD273xmWo1gqf6qty2NzLQuZGV0IStt/36z3ixh2676k7P3y8/hleuamtmqtnjP2op3jGQrtcdIPibG1CQdQZsk+mt3wqaVcNHDChGprUsaHPdtOHwMzP6foBvk7dnBJJBqnbSprTsrWbl+GyvXbeXdGoGxbsv+STi6ZqZx2IHdOXFQH555+2PKK/bUO85Xur7GT/f+ni19jqXq+Hu5gyx27ali9569wVdV8H1X+LV/e9W+7TX33bVnL+UVleG2qlqPbyhEANaWVTS4vTXE27XVPs9Ja6Gpc1Y02DyLTU/8AAASNElEQVS96anlnDniYDLS4j1rOobyEph3CxzxJTj81MSOJR1Xr4Fw2dPBWX0v/AzuPCZsnVyg1kkr27F7D6vWb2PFx1tZuX7bvpZGaY0/vtkZXTisTzdGH9aLww/szuEHduPwA7vTNy+HLl2Cf4/RC3vV+hAKcEHmK/y06i5swOfJm/AQn0/y1Eejb36hVt3V+uYn74zQeFsk5wIvuHt5eDsfOMHdO+RcD7GSe92WXRzx46c5qEc2hT1z6H9AbvC9Z+6+2wf2yCatSxO/5HOuB3c47f8loXrpULqkwXFXBa2TR78Fj3w9HDv5DfQ4OOrqUko83dE7K6tYtX4bK9cHLYyV67ayYt1WSjZXUN3Ln5nehU/37kZxUU8uOvCQfaFR2DO3yd/t6uerruOy7q/xo8ogRJjwUJvMnzfptEH1wiwnI41Jpw1K2nPGe2X7onCRqZrbFrr7qKRV1oqaO0YSK9HzczK45HOHUrK5gjWf7KBkcwXrtu6k5luYkWb0zc+pFzDVt3ute4UuD5wXzL30hUlN1tLWZ19ICttbBa/fA89PCaasH3oerHouaOHmFQarN0axXsuS6UFNEdYxe2EpLz9yF9cwjb62kbXei9v8QnKLJ5CXnRG0MNZvY/Wm7ewNf1/Tuxif6t01DIogLAYe2J1DD8glPZFeh33vR7iKRq8j4Mq5bToJa2v93Yh3jCTeIFkSrmJYc9tSdx/e7Moi0NwgqTtGAkGi33Te8Hr/GLv2VLG2bCclm3ew5pOK4Pvmin23N27b35eaSSVzsq4lo4txQ+F9HHxAHoU9c+l/QE7wvWcOB3TN3HfGRnPqkE5k03vwwHj4ZFXt7enZcPKP23bCzxVPUvXsFNL27v9/XtUli7Qv3tCqdbjDjsoqtlRUsnVnJVt37mFLxR627Kxky85K1r4+k+8wjWzbN1E4Oz2DW/aMZ64fTb+e2Qzo1ZWigm4M6JXLp3p1pV/PXDLSWrmLcMWT8PzPgkXpqqXnwNntc2XT1g6S+4EygqVzHfg20NPd/zvBOttElGdtVeyuorQsCJf8+XcwauVv+W3fX/Ls7mGUbK7gk+21z/LIzUyjsGcQLK+/v4ntu1vn7AvpYG4bGrQApH3I6w/fbX8TlbfqWVsEwfFj4KHw9jPAj1pYW7swdlS/VvnUn5OZxmF9unNY5mb4z30w+Gy+fcE3+HZ4/7Zdeyjd11VWuzXTUIhAcBbZs2+v48j+efTprjO+OqXy0tj3nXtPm5Xhj3y94RXsHB7oez07du9hx+6q8Cv4ubHPrlnpXcjNSic3I43crDRyM9PIzUwnNzONruHPOVlpdM1ID+9PJyczjYzHvhlzJT1rw/eDR77e8PYOHvrxnrW1HbguybV0bE9PDs60qTPA3i0rnUEHdWfQQfVPjIs1VgNwxV+DFlbfvGyO7J8ffBXmM7wwj25Z8X4+kHYrr3B/H3yt7f3hyAtb9an2VO1lbdlO/rNpO6s3beeDjTuC75u285e9vSjsUn+xrlLvxb3lx5Cfm0Fefgb5uZnk52TQLzeDvJz9t/Nzg6+8nEzycjLITG/Z2MSOZ6eQW/FRve0VOQeT28rvR6Ne+HmMf5fCtqshAvGetfUs8GV3Lwtv9wSmuftpySyuw1j5LLzzBJz8E8jv3/T+oVhnX0w5Zyif6t2NRWvKWLymjMUlZTy17GMgyKqBfbpxZGEQLiP75zPooO6Jn7KcAnTiQQ0n3wCPXw2VNT5oZOQE21tg9569lGzewepNO/hg0/Za39d8soM9e/c3I3Iy0ji0IJeBfbpzx5YJ3Oj3kGv7u2h3eCb3ZU7kpR+c2OKX11y5p09hz6PfJr1q/9jEnrRsck+f0mY1AK3+79JexPvRtVd1iAC4+2Yza2rNdgGo3AlPToKCgfC5q5r10LqnEtb943nUoT337bt5+24Wl5SxeE05i0vKeP6d9Ty8IGhOZ6V3YVi/vDBc8hjZP59DDshNaBqGthbFtA8pbcR43vhgM/3fnEof38h668Wa4ZM4upEB3Z2VVaz5ZAcfbNrfoqgOjNLNFdTICrplpVPUK5chfXtwxvCDOLSgK0UFXSkqyKV396waJ4QcxA2PONf4NPraJtZ6Ab/hQv7rzCuT/Q7UNmJ88Mesxtlj6VGcxVb9fBGfxdbW4h1sXwCc6+4fhreLgFnu/pmkVtdKIl3Yat4tMPcXcMkj8Om2GyB3d0o2V9RqtSwtLWdn5V4A8nMzarRagpAp6JbVZvU1pbIquHK3bEcl5RW7ufKvC9i0vf70E727ZTH7qtH07pbV4m6R9ijWGX0/PXsoI/rn1eh+Cr6v3rSDteUVtcYnemSnM6BX1zAkcoPvvYLvBTXOHoynFrUUO6bWPmtrDHAv+5e9/QJwpbvPSajKNhJZkGz+IFij+/AxMP4vbf/8deyp2su767axuKSMRR8G4fLuuq37PokW9sxhZNgddmT/fIb1zSMnMw1o+R+LnZVV+wKhbMduyioqKd9RSVnF7mBb3ds7KimvqGTbrvrTTDTlgK6Z9OmeRe/uWRzYI5s+3bOCrx7ZHNgjiz7ds+ndPYvsjLRmHztKdUO1bEcl3394MZt3VDb52AO6ZlJUkEtRQddaQVFUkEt+bmYbVC/tWasGSXjAPsCVwCKCqeTXu/tLCVXZRiILkgcnwPvz4Ko3IC81P6Ft37WHZaXl+7rFFq0p2zfAn9bFOPzA7uTnpDN/9WYqq/b/X8lMM84Z1Y8BvboGQVAjDPYFR8XufS2ghqR3sXCgdf/ga15uBvk5mTUGYYP7vj99MRtqXJNTraBrJt8/bRDrt+xi3dadrN+yiw1bd7J+6y42bN1Vq2+/Wo/s9FrhUi98wu9dY5y0kOqh+tsJoygq6MohBbnk5WQ067EiNbV2i+RrwHeAQoIgORZ41d0b7asJWzK3A2nAfe5+c537vwd8DdgDbAAuc/fV4X1VwNJw1w/d/exw+wBgGsHyv28Cl7h77Ck3iShIVjwND14AX5wSTA/ejqzfupMl4VjLojVlvLxyY/1VzWrITO+y/wycnMwwDDLo2TUzDIL94bDvdm4mXTPTmtV90tyLM/fudT7ZsZv1W3axPgyZ9WHI1A6eXeyuqh943bLS94VMnx7ZHNg9i/Vbd/LUso/bNFR7VodqjTOdrvjr/FoTB1bTNUbSmlo7SJYCRwOvuftIMzsC+Km7X9DIY9KAd4EvAiXAG8AEd3+7xj4nAq+7+w4z+ybB/F0XhPdtc/d6y4aZ2XSC8ZlpZvZ7YLG7391Y/W0eJJUVQZdWejZ84+VgOot2bMB1/2gwSAx4e8oYsjMSWz8hXsnqi3d3yisqWVcrcOqHz7otOxsNAwhCtWedQK0OztYKVdCsB9I2WvuCxJ3uvtPMMLMsd38nXJOkMccAq9z9/bCgacA5wL4gcfe5NfZ/DZjY2AEt+E07Cbgo3PQX4Eag0SBpc6/cDmWr4SuPtfsQgWDW0FiziVaPobSF1rpItC4zC1oBuZkNXs9Tzd351OQnY4bq8p+NabPxl6bO6BNpS/EGSUk44+9s4Fkz20zTS+32A2pemVMCfLaR/S8HnqpxO9vM5hN0e90czjRcAJS5e3WncUn4PKnjk//AP2+FYefDp46PuppWEcVsoqnIzBoN1bYexE9WsIo0V7xXtp8b/nijmc0F8oCnm3hYrBkL6u9oNhEoBmr+5T3E3dea2aeAF8LutS3NOOaVBCcHcMghhzRRaitxh6euhbQMOPUXbfOcbUCffvdTqIrU1+y5NNx9XtN7AUFroeZl3IU00Ioxs1OAHwLHu/u+0UN3Xxt+f9/MXgRGATOBfDNLD1slDR4zfNy9BKcsU1xc3Pz1hFtixVOwck4QIh1srQh9+g0oVEXqS+akTG8AA8OzrEqBC9k/tgGAmY0C7gHGuPv6Gtt7AjvcfZeZ9QJGA7e4u4ctonEEZ259FXg0ia8hfrt3BK2R3oPhszEmbpMOQaEqUlvSLgUOWwxXAXOA5cB0d3/LzKaY2dnhblOBbsDDZrbIzB4Ltw8G5pvZYmAuwRhJ9SD9tcD3zGwVwZjJH5P1Gprl5dug/EM481dB15aISCcR9wWJ7VnST//d9B7cdSwMGQvn/yF5zyMi0obiPf2380xOlCzVA+zp2XDqz6KuRkSkzSlIEvXOP2DVs3Di9dD9oKirERFpcwqSROzeDk9fB32GwtFXRF2NiEgktJReIv7562A1tEufgjS9lSLSOalF0lIbV8Erd8CRE+DQ46KuRkQkMgqSlnCHpyZBRm4wu6+ISCemIGmJ5Y/Bey/AST+CblpxWEQ6NwVJc+3eDk9PhoOGQ/FlUVcjIhI5jRA317xbYEspjPuTBthFRFCLpHk2rIBXfwcjJ8Ihjc2ILyLSeShI4uUOT06CzK5wyo1RVyMikjLUNxOvtx6B/8yDM34F3XpHXY2ISMpQiyQeu7bCnOvh4CM1wC4iUodaJPGYdwts/Qgu+Bt0advlVEVEUp2CJJYl0+H5KVBeAjgc+l9Q2ORsyiIinY66thqyZDo8fnUwj1b1kvClC4LtIiJSi4KkIc9PgcqK2tv2VATbRUSkFgVJQ8pLmrddRKQTU5A0JK+wedtFRDoxBUlDTr4BMnJqb8vICbaLiEgtCpKGjBgPZ90Bef0BC76fdUewXUREatHpv7GMGK/gEBGJg7l71DUknZltAFa38OG9gI2tWE57p/djP70Xten9qK0jvB+HunuTc0J1iiBJhJnNd3ddiRjS+7Gf3ova9H7U1pneD42RiIhIQhQkIiKSEAVJ0+6NuoAUo/djP70Xten9qK3TvB8aIxERkYSoRSIiIglRkDTCzMaY2QozW2Vm10VdT1TMrL+ZzTWz5Wb2lpl9J+qaUoGZpZnZQjN7IupaomZm+WY2w8zeCf+ffC7qmqJiZt8Nf0+WmdmDZpYddU3JpiCJwczSgDuB04EhwAQzGxJtVZHZA/yvuw8GjgW+1Ynfi5q+AyyPuogUcTvwtLsfARxJJ31fzKwfcDVQ7O7DgDTgwmirSj4FSWzHAKvc/X133w1MA86JuKZIuPtH7v5m+PNWgj8S/aKtKlpmVgicCdwXdS1RM7MewBeAPwK4+253L4u2qkilAzlmlg7kAmsjrifpFCSx9QPW1LhdQif/4wlgZkXAKOD1aCuJ3G+AHwB7oy4kBXwK2AD8Kezqu8/MukZdVBTcvRT4FfAh8BFQ7u7PRFtV8ilIYrMGtnXqU9zMrBswE7jG3bdEXU9UzOxLwHp3XxB1LSkiHfgMcLe7jwK2A51yTNHMehL0XAwA+gJdzWxitFUln4IkthKgf43bhXSCJmosZpZBECIPuPusqOuJ2GjgbDP7gKDL8yQz+1u0JUWqBChx9+pW6gyCYOmMTgH+4+4b3L0SmAUcF3FNSacgie0NYKCZDTCzTIIBs8cirikSZmYE/d/L3f3WqOuJmrtPdvdCdy8i+H/xgrt3+E+dsbj7x8AaMxsUbjoZeDvCkqL0IXCsmeWGvzcn0wlOPNA08jG4+x4zuwqYQ3Dmxf3u/lbEZUVlNHAJsNTMFoXbrnf3JyOsSVLLt4EHwg9d7wOXRlxPJNz9dTObAbxJcLbjQjrBFe66sl1ERBKiri0REUmIgkRERBKiIBERkYQoSEREJCEKEhERSYiCRCTFmdkJmmFYUpmCREREEqIgEWklZjbRzP5tZovM7J5wvZJtZvZrM3vTzJ43s97hviPN7DUzW2Jmj4RzNGFmh5nZc2a2OHzMp8PDd6ux3scD4VXTIilBQSLSCsxsMHABMNrdRwJVwMVAV+BNd/8MMA/4SfiQvwLXuvsIYGmN7Q8Ad7r7kQRzNH0Ubh8FXEOwNs6nCGYbEEkJmiJFpHWcDBwFvBE2FnKA9QTTzD8U7vM3YJaZ5QH57j4v3P4X4GEz6w70c/dHANx9J0B4vH+7e0l4exFQBLyc/Jcl0jQFiUjrMOAv7j651kazH9fZr7E5iRrrrtpV4+cq9LsrKURdWyKt43lgnJn1ATCzA8zsUILfsXHhPhcBL7t7ObDZzD4fbr8EmBeu8VJiZmPDY2SZWW6bvgqRFtCnGpFW4O5vm9mPgGfMrAtQCXyLYJGnoWa2ACgnGEcB+Crw+zAoas6Wewlwj5lNCY/x5TZ8GSItotl/RZLIzLa5e7eo6xBJJnVtiYhIQtQiERGRhKhFIiIiCVGQiIhIQhQkIiKSEAWJiIgkREEiIiIJUZCIiEhC/j/fUSafoUljYwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fac9f46c828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot training and validation history\n",
    "plt.subplot(2, 1, 1)\n",
    "plt.plot(loss_his, 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(train_accu_his, '-o')\n",
    "plt.plot(val_accu_his, '-o')\n",
    "plt.legend(['train','val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy 0.17 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test set\n",
    "hidden_init = Variable(torch.zeros(num_layers, N_test, hidden_size))\n",
    "y_test_pred,_,_ = model(X_test,y_test.type(torch.LongTensor),hidden_init)\n",
    "  \n",
    "_, y_pred = torch.max(y_test_pred,1)\n",
    "test_accu = np.mean(y_pred.data.numpy() ==  y_test.data.numpy())\n",
    "print('Test accuracy', test_accu, '\\n')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}