lab-10-1-1-mnist_nn_softmax.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def sigmoid() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.sigmoid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(2):\n",
    "            self.model.add(dense(256, weight_init))\n",
    "            self.model.add(sigmoid())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(2) :\n",
    "        model.add(dense(256, weight_init))\n",
    "        model.add(sigmoid())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_softmax'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.3214, train_loss: 2.23722649, train_accuracy: 0.1641, test_Accuracy: 0.1010\n",
      "Epoch: [ 0] [    1/  468] time: 0.4400, train_loss: 2.28909516, train_accuracy: 0.1172, test_Accuracy: 0.1135\n",
      "Epoch: [ 0] [    2/  468] time: 0.5412, train_loss: 2.30295420, train_accuracy: 0.0859, test_Accuracy: 0.1135\n",
      "Epoch: [ 0] [    3/  468] time: 0.6293, train_loss: 2.26018047, train_accuracy: 0.1484, test_Accuracy: 0.1135\n",
      "Epoch: [ 0] [    4/  468] time: 0.7447, train_loss: 2.28151369, train_accuracy: 0.1172, test_Accuracy: 0.1136\n",
      "Epoch: [ 0] [    5/  468] time: 0.8545, train_loss: 2.26245213, train_accuracy: 0.2109, test_Accuracy: 0.2517\n",
      "Epoch: [ 0] [    6/  468] time: 0.9439, train_loss: 2.22125435, train_accuracy: 0.1719, test_Accuracy: 0.1654\n",
      "Epoch: [ 0] [    7/  468] time: 1.0468, train_loss: 2.25816059, train_accuracy: 0.2031, test_Accuracy: 0.2150\n",
      "Epoch: [ 0] [    8/  468] time: 1.1519, train_loss: 2.23189902, train_accuracy: 0.3438, test_Accuracy: 0.3895\n",
      "Epoch: [ 0] [    9/  468] time: 1.2529, train_loss: 2.24128866, train_accuracy: 0.4062, test_Accuracy: 0.4593\n",
      "Epoch: [ 0] [   10/  468] time: 1.3527, train_loss: 2.21562743, train_accuracy: 0.4453, test_Accuracy: 0.4565\n",
      "Epoch: [ 0] [   11/  468] time: 1.4533, train_loss: 2.18162489, train_accuracy: 0.3828, test_Accuracy: 0.4094\n",
      "Epoch: [ 0] [   12/  468] time: 1.5543, train_loss: 2.17894244, train_accuracy: 0.4531, test_Accuracy: 0.4715\n",
      "Epoch: [ 0] [   13/  468] time: 1.6576, train_loss: 2.14895487, train_accuracy: 0.5703, test_Accuracy: 0.5070\n",
      "Epoch: [ 0] [   14/  468] time: 1.7457, train_loss: 2.14816427, train_accuracy: 0.5078, test_Accuracy: 0.4942\n",
      "Epoch: [ 0] [   15/  468] time: 1.8480, train_loss: 2.15663004, train_accuracy: 0.4453, test_Accuracy: 0.4868\n",
      "Epoch: [ 0] [   16/  468] time: 1.9531, train_loss: 2.11797333, train_accuracy: 0.4297, test_Accuracy: 0.4387\n",
      "Epoch: [ 0] [   17/  468] time: 2.0413, train_loss: 2.11331487, train_accuracy: 0.4609, test_Accuracy: 0.4306\n",
      "Epoch: [ 0] [   18/  468] time: 2.1391, train_loss: 2.06329584, train_accuracy: 0.4922, test_Accuracy: 0.4079\n",
      "Epoch: [ 0] [   19/  468] time: 2.2459, train_loss: 2.08047390, train_accuracy: 0.4062, test_Accuracy: 0.4101\n",
      "Epoch: [ 0] [   20/  468] time: 2.3307, train_loss: 2.00329399, train_accuracy: 0.5234, test_Accuracy: 0.4339\n",
      "Epoch: [ 0] [   21/  468] time: 2.4176, train_loss: 2.06611729, train_accuracy: 0.5000, test_Accuracy: 0.4814\n",
      "Epoch: [ 0] [   22/  468] time: 2.5233, train_loss: 2.01769280, train_accuracy: 0.5078, test_Accuracy: 0.4983\n",
      "Epoch: [ 0] [   23/  468] time: 2.6100, train_loss: 2.01936626, train_accuracy: 0.5078, test_Accuracy: 0.4880\n",
      "Epoch: [ 0] [   24/  468] time: 2.7073, train_loss: 1.95456338, train_accuracy: 0.5469, test_Accuracy: 0.4740\n",
      "Epoch: [ 0] [   25/  468] time: 2.8051, train_loss: 2.01162434, train_accuracy: 0.4453, test_Accuracy: 0.4741\n",
      "Epoch: [ 0] [   26/  468] time: 2.9140, train_loss: 1.98020816, train_accuracy: 0.4219, test_Accuracy: 0.4782\n",
      "Epoch: [ 0] [   27/  468] time: 3.0180, train_loss: 1.91924524, train_accuracy: 0.4844, test_Accuracy: 0.4852\n",
      "Epoch: [ 0] [   28/  468] time: 3.1047, train_loss: 1.86071646, train_accuracy: 0.6016, test_Accuracy: 0.4890\n",
      "Epoch: [ 0] [   29/  468] time: 3.2123, train_loss: 1.86554360, train_accuracy: 0.5234, test_Accuracy: 0.5062\n",
      "Epoch: [ 0] [   30/  468] time: 3.3220, train_loss: 1.84275496, train_accuracy: 0.5859, test_Accuracy: 0.5326\n",
      "Epoch: [ 0] [   31/  468] time: 3.4186, train_loss: 1.85675263, train_accuracy: 0.5156, test_Accuracy: 0.5570\n",
      "Epoch: [ 0] [   32/  468] time: 3.5090, train_loss: 1.79241991, train_accuracy: 0.5625, test_Accuracy: 0.5629\n",
      "Epoch: [ 0] [   33/  468] time: 3.6085, train_loss: 1.80538630, train_accuracy: 0.5156, test_Accuracy: 0.5620\n",
      "Epoch: [ 0] [   34/  468] time: 3.7155, train_loss: 1.75675166, train_accuracy: 0.5312, test_Accuracy: 0.5731\n",
      "Epoch: [ 0] [   35/  468] time: 3.8160, train_loss: 1.71772349, train_accuracy: 0.5938, test_Accuracy: 0.5977\n",
      "Epoch: [ 0] [   36/  468] time: 3.9018, train_loss: 1.67758584, train_accuracy: 0.6406, test_Accuracy: 0.6283\n",
      "Epoch: [ 0] [   37/  468] time: 4.0022, train_loss: 1.65568328, train_accuracy: 0.6719, test_Accuracy: 0.6649\n",
      "Epoch: [ 0] [   38/  468] time: 4.0893, train_loss: 1.62754154, train_accuracy: 0.6797, test_Accuracy: 0.6759\n",
      "Epoch: [ 0] [   39/  468] time: 4.1917, train_loss: 1.61349714, train_accuracy: 0.6016, test_Accuracy: 0.6836\n",
      "Epoch: [ 0] [   40/  468] time: 4.2825, train_loss: 1.60354483, train_accuracy: 0.6562, test_Accuracy: 0.6845\n",
      "Epoch: [ 0] [   41/  468] time: 4.3878, train_loss: 1.58427727, train_accuracy: 0.6797, test_Accuracy: 0.6840\n",
      "Epoch: [ 0] [   42/  468] time: 4.5017, train_loss: 1.46838593, train_accuracy: 0.6641, test_Accuracy: 0.6796\n",
      "Epoch: [ 0] [   43/  468] time: 4.5950, train_loss: 1.49716854, train_accuracy: 0.7109, test_Accuracy: 0.6866\n",
      "Epoch: [ 0] [   44/  468] time: 4.6924, train_loss: 1.42094374, train_accuracy: 0.6875, test_Accuracy: 0.6963\n",
      "Epoch: [ 0] [   45/  468] time: 4.7889, train_loss: 1.48012447, train_accuracy: 0.7266, test_Accuracy: 0.6978\n",
      "Epoch: [ 0] [   46/  468] time: 4.8908, train_loss: 1.44550824, train_accuracy: 0.7109, test_Accuracy: 0.6965\n",
      "Epoch: [ 0] [   47/  468] time: 4.9968, train_loss: 1.42235243, train_accuracy: 0.6719, test_Accuracy: 0.7005\n",
      "Epoch: [ 0] [   48/  468] time: 5.0916, train_loss: 1.28692675, train_accuracy: 0.7344, test_Accuracy: 0.7002\n",
      "Epoch: [ 0] [   49/  468] time: 5.1909, train_loss: 1.33725965, train_accuracy: 0.7188, test_Accuracy: 0.6922\n",
      "Epoch: [ 0] [   50/  468] time: 5.2855, train_loss: 1.44693911, train_accuracy: 0.5781, test_Accuracy: 0.6904\n",
      "Epoch: [ 0] [   51/  468] time: 5.3736, train_loss: 1.30874562, train_accuracy: 0.6562, test_Accuracy: 0.7001\n",
      "Epoch: [ 0] [   52/  468] time: 5.4697, train_loss: 1.32145214, train_accuracy: 0.6797, test_Accuracy: 0.7077\n",
      "Epoch: [ 0] [   53/  468] time: 5.5614, train_loss: 1.23573387, train_accuracy: 0.7422, test_Accuracy: 0.7055\n",
      "Epoch: [ 0] [   54/  468] time: 5.6559, train_loss: 1.24496794, train_accuracy: 0.6953, test_Accuracy: 0.7141\n",
      "Epoch: [ 0] [   55/  468] time: 5.7603, train_loss: 1.26061058, train_accuracy: 0.7500, test_Accuracy: 0.7218\n",
      "Epoch: [ 0] [   56/  468] time: 5.8531, train_loss: 1.16722560, train_accuracy: 0.7109, test_Accuracy: 0.7341\n",
      "Epoch: [ 0] [   57/  468] time: 5.9610, train_loss: 1.14255977, train_accuracy: 0.7734, test_Accuracy: 0.7337\n",
      "Epoch: [ 0] [   58/  468] time: 6.0608, train_loss: 1.15771294, train_accuracy: 0.7422, test_Accuracy: 0.7329\n",
      "Epoch: [ 0] [   59/  468] time: 6.1601, train_loss: 1.18425572, train_accuracy: 0.6875, test_Accuracy: 0.7358\n",
      "Epoch: [ 0] [   60/  468] time: 6.2561, train_loss: 1.12921429, train_accuracy: 0.7578, test_Accuracy: 0.7435\n",
      "Epoch: [ 0] [   61/  468] time: 6.3547, train_loss: 1.06178427, train_accuracy: 0.7578, test_Accuracy: 0.7539\n",
      "Epoch: [ 0] [   62/  468] time: 6.4513, train_loss: 1.14705539, train_accuracy: 0.7422, test_Accuracy: 0.7718\n",
      "Epoch: [ 0] [   63/  468] time: 6.5411, train_loss: 1.16050315, train_accuracy: 0.7109, test_Accuracy: 0.7844\n",
      "Epoch: [ 0] [   64/  468] time: 6.6287, train_loss: 0.98971725, train_accuracy: 0.8125, test_Accuracy: 0.7866\n",
      "Epoch: [ 0] [   65/  468] time: 6.7199, train_loss: 1.07633185, train_accuracy: 0.8125, test_Accuracy: 0.7827\n",
      "Epoch: [ 0] [   66/  468] time: 6.8194, train_loss: 1.00868392, train_accuracy: 0.7734, test_Accuracy: 0.7748\n",
      "Epoch: [ 0] [   67/  468] time: 6.9184, train_loss: 1.00508583, train_accuracy: 0.7344, test_Accuracy: 0.7758\n",
      "Epoch: [ 0] [   68/  468] time: 7.0180, train_loss: 1.00913131, train_accuracy: 0.7422, test_Accuracy: 0.7816\n",
      "Epoch: [ 0] [   69/  468] time: 7.1166, train_loss: 0.97728884, train_accuracy: 0.7891, test_Accuracy: 0.7845\n",
      "Epoch: [ 0] [   70/  468] time: 7.2133, train_loss: 0.95004976, train_accuracy: 0.8281, test_Accuracy: 0.7901\n",
      "Epoch: [ 0] [   71/  468] time: 7.3126, train_loss: 0.94252646, train_accuracy: 0.7734, test_Accuracy: 0.7925\n",
      "Epoch: [ 0] [   72/  468] time: 7.4093, train_loss: 1.00439739, train_accuracy: 0.7734, test_Accuracy: 0.7951\n",
      "Epoch: [ 0] [   73/  468] time: 7.4988, train_loss: 0.90114844, train_accuracy: 0.7969, test_Accuracy: 0.7967\n",
      "Epoch: [ 0] [   74/  468] time: 7.5933, train_loss: 1.06239986, train_accuracy: 0.7188, test_Accuracy: 0.7954\n",
      "Epoch: [ 0] [   75/  468] time: 7.6902, train_loss: 0.92153341, train_accuracy: 0.7734, test_Accuracy: 0.7919\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   76/  468] time: 7.7875, train_loss: 0.97948754, train_accuracy: 0.7500, test_Accuracy: 0.7898\n",
      "Epoch: [ 0] [   77/  468] time: 7.8770, train_loss: 0.81453854, train_accuracy: 0.8516, test_Accuracy: 0.7857\n",
      "Epoch: [ 0] [   78/  468] time: 7.9662, train_loss: 0.87466240, train_accuracy: 0.7500, test_Accuracy: 0.7879\n",
      "Epoch: [ 0] [   79/  468] time: 8.0544, train_loss: 0.82989830, train_accuracy: 0.8438, test_Accuracy: 0.7888\n",
      "Epoch: [ 0] [   80/  468] time: 8.1536, train_loss: 0.92940915, train_accuracy: 0.7578, test_Accuracy: 0.7937\n",
      "Epoch: [ 0] [   81/  468] time: 8.2550, train_loss: 0.86642623, train_accuracy: 0.7812, test_Accuracy: 0.8009\n",
      "Epoch: [ 0] [   82/  468] time: 8.3475, train_loss: 0.94328272, train_accuracy: 0.8047, test_Accuracy: 0.8146\n",
      "Epoch: [ 0] [   83/  468] time: 8.4356, train_loss: 0.80263299, train_accuracy: 0.8594, test_Accuracy: 0.8231\n",
      "Epoch: [ 0] [   84/  468] time: 8.5233, train_loss: 0.77890849, train_accuracy: 0.8281, test_Accuracy: 0.8242\n",
      "Epoch: [ 0] [   85/  468] time: 8.6224, train_loss: 0.83193648, train_accuracy: 0.8359, test_Accuracy: 0.8168\n",
      "Epoch: [ 0] [   86/  468] time: 8.7119, train_loss: 0.75260425, train_accuracy: 0.8203, test_Accuracy: 0.8032\n",
      "Epoch: [ 0] [   87/  468] time: 8.8024, train_loss: 0.78688997, train_accuracy: 0.8047, test_Accuracy: 0.7936\n",
      "Epoch: [ 0] [   88/  468] time: 8.8944, train_loss: 0.81042981, train_accuracy: 0.8281, test_Accuracy: 0.7877\n",
      "Epoch: [ 0] [   89/  468] time: 8.9937, train_loss: 0.73902535, train_accuracy: 0.8203, test_Accuracy: 0.7898\n",
      "Epoch: [ 0] [   90/  468] time: 9.0872, train_loss: 0.74136716, train_accuracy: 0.8203, test_Accuracy: 0.7988\n",
      "Epoch: [ 0] [   91/  468] time: 9.1886, train_loss: 0.74993193, train_accuracy: 0.8516, test_Accuracy: 0.8122\n",
      "Epoch: [ 0] [   92/  468] time: 9.2888, train_loss: 0.86539423, train_accuracy: 0.7891, test_Accuracy: 0.8245\n",
      "Epoch: [ 0] [   93/  468] time: 9.3774, train_loss: 0.69966352, train_accuracy: 0.8516, test_Accuracy: 0.8279\n",
      "Epoch: [ 0] [   94/  468] time: 9.4687, train_loss: 0.75942171, train_accuracy: 0.8125, test_Accuracy: 0.8278\n",
      "Epoch: [ 0] [   95/  468] time: 9.5698, train_loss: 0.73609769, train_accuracy: 0.8594, test_Accuracy: 0.8226\n",
      "Epoch: [ 0] [   96/  468] time: 9.6624, train_loss: 0.67635393, train_accuracy: 0.8359, test_Accuracy: 0.8194\n",
      "Epoch: [ 0] [   97/  468] time: 9.7629, train_loss: 0.73573112, train_accuracy: 0.7969, test_Accuracy: 0.8150\n",
      "Epoch: [ 0] [   98/  468] time: 9.8588, train_loss: 0.58598566, train_accuracy: 0.8438, test_Accuracy: 0.8114\n",
      "Epoch: [ 0] [   99/  468] time: 9.9462, train_loss: 0.77989262, train_accuracy: 0.7812, test_Accuracy: 0.8095\n",
      "Epoch: [ 0] [  100/  468] time: 10.0488, train_loss: 0.65660119, train_accuracy: 0.8281, test_Accuracy: 0.8108\n",
      "Epoch: [ 0] [  101/  468] time: 10.1369, train_loss: 0.74638367, train_accuracy: 0.7891, test_Accuracy: 0.8172\n",
      "Epoch: [ 0] [  102/  468] time: 10.2350, train_loss: 0.74013937, train_accuracy: 0.8047, test_Accuracy: 0.8265\n",
      "Epoch: [ 0] [  103/  468] time: 10.3250, train_loss: 0.52706385, train_accuracy: 0.9062, test_Accuracy: 0.8362\n",
      "Epoch: [ 0] [  104/  468] time: 10.4172, train_loss: 0.71902591, train_accuracy: 0.8203, test_Accuracy: 0.8407\n",
      "Epoch: [ 0] [  105/  468] time: 10.5165, train_loss: 0.54552758, train_accuracy: 0.8750, test_Accuracy: 0.8425\n",
      "Epoch: [ 0] [  106/  468] time: 10.6072, train_loss: 0.68568671, train_accuracy: 0.8438, test_Accuracy: 0.8412\n",
      "Epoch: [ 0] [  107/  468] time: 10.7015, train_loss: 0.59931207, train_accuracy: 0.8828, test_Accuracy: 0.8407\n",
      "Epoch: [ 0] [  108/  468] time: 10.7914, train_loss: 0.81427300, train_accuracy: 0.8047, test_Accuracy: 0.8425\n",
      "Epoch: [ 0] [  109/  468] time: 10.8822, train_loss: 0.58555400, train_accuracy: 0.8750, test_Accuracy: 0.8432\n",
      "Epoch: [ 0] [  110/  468] time: 10.9699, train_loss: 0.72000331, train_accuracy: 0.8047, test_Accuracy: 0.8446\n",
      "Epoch: [ 0] [  111/  468] time: 11.0611, train_loss: 0.60129249, train_accuracy: 0.8516, test_Accuracy: 0.8470\n",
      "Epoch: [ 0] [  112/  468] time: 11.1566, train_loss: 0.59778917, train_accuracy: 0.8906, test_Accuracy: 0.8504\n",
      "Epoch: [ 0] [  113/  468] time: 11.2488, train_loss: 0.66919839, train_accuracy: 0.8750, test_Accuracy: 0.8538\n",
      "Epoch: [ 0] [  114/  468] time: 11.3444, train_loss: 0.60106611, train_accuracy: 0.8672, test_Accuracy: 0.8542\n",
      "Epoch: [ 0] [  115/  468] time: 11.4345, train_loss: 0.63191319, train_accuracy: 0.8281, test_Accuracy: 0.8561\n",
      "Epoch: [ 0] [  116/  468] time: 11.5270, train_loss: 0.62379748, train_accuracy: 0.8281, test_Accuracy: 0.8557\n",
      "Epoch: [ 0] [  117/  468] time: 11.6090, train_loss: 0.64563751, train_accuracy: 0.8359, test_Accuracy: 0.8560\n",
      "Epoch: [ 0] [  118/  468] time: 11.6965, train_loss: 0.61815369, train_accuracy: 0.8672, test_Accuracy: 0.8556\n",
      "Epoch: [ 0] [  119/  468] time: 11.7849, train_loss: 0.62214726, train_accuracy: 0.8438, test_Accuracy: 0.8534\n",
      "Epoch: [ 0] [  120/  468] time: 11.8753, train_loss: 0.49191269, train_accuracy: 0.8906, test_Accuracy: 0.8473\n",
      "Epoch: [ 0] [  121/  468] time: 11.9668, train_loss: 0.52274871, train_accuracy: 0.8750, test_Accuracy: 0.8431\n",
      "Epoch: [ 0] [  122/  468] time: 12.0645, train_loss: 0.59756577, train_accuracy: 0.8203, test_Accuracy: 0.8424\n",
      "Epoch: [ 0] [  123/  468] time: 12.1520, train_loss: 0.68595195, train_accuracy: 0.7812, test_Accuracy: 0.8435\n",
      "Epoch: [ 0] [  124/  468] time: 12.2434, train_loss: 0.53884453, train_accuracy: 0.8281, test_Accuracy: 0.8458\n",
      "Epoch: [ 0] [  125/  468] time: 12.3329, train_loss: 0.62374759, train_accuracy: 0.8516, test_Accuracy: 0.8477\n",
      "Epoch: [ 0] [  126/  468] time: 12.4258, train_loss: 0.64589846, train_accuracy: 0.7891, test_Accuracy: 0.8527\n",
      "Epoch: [ 0] [  127/  468] time: 12.5202, train_loss: 0.62867832, train_accuracy: 0.8203, test_Accuracy: 0.8595\n",
      "Epoch: [ 0] [  128/  468] time: 12.6114, train_loss: 0.57719499, train_accuracy: 0.8281, test_Accuracy: 0.8650\n",
      "Epoch: [ 0] [  129/  468] time: 12.7041, train_loss: 0.58358979, train_accuracy: 0.8281, test_Accuracy: 0.8673\n",
      "Epoch: [ 0] [  130/  468] time: 12.7973, train_loss: 0.49178880, train_accuracy: 0.8828, test_Accuracy: 0.8686\n",
      "Epoch: [ 0] [  131/  468] time: 12.8874, train_loss: 0.55830491, train_accuracy: 0.8672, test_Accuracy: 0.8664\n",
      "Epoch: [ 0] [  132/  468] time: 12.9789, train_loss: 0.53631949, train_accuracy: 0.8516, test_Accuracy: 0.8655\n",
      "Epoch: [ 0] [  133/  468] time: 13.0745, train_loss: 0.60577762, train_accuracy: 0.8359, test_Accuracy: 0.8649\n",
      "Epoch: [ 0] [  134/  468] time: 13.1650, train_loss: 0.54019642, train_accuracy: 0.8359, test_Accuracy: 0.8669\n",
      "Epoch: [ 0] [  135/  468] time: 13.2548, train_loss: 0.62458998, train_accuracy: 0.8047, test_Accuracy: 0.8682\n",
      "Epoch: [ 0] [  136/  468] time: 13.3482, train_loss: 0.49261564, train_accuracy: 0.8750, test_Accuracy: 0.8703\n",
      "Epoch: [ 0] [  137/  468] time: 13.4381, train_loss: 0.40109640, train_accuracy: 0.9141, test_Accuracy: 0.8757\n",
      "Epoch: [ 0] [  138/  468] time: 13.5282, train_loss: 0.44847780, train_accuracy: 0.9062, test_Accuracy: 0.8759\n",
      "Epoch: [ 0] [  139/  468] time: 13.6208, train_loss: 0.51045036, train_accuracy: 0.8594, test_Accuracy: 0.8744\n",
      "Epoch: [ 0] [  140/  468] time: 13.7094, train_loss: 0.45303833, train_accuracy: 0.8984, test_Accuracy: 0.8733\n",
      "Epoch: [ 0] [  141/  468] time: 13.8022, train_loss: 0.48477256, train_accuracy: 0.8516, test_Accuracy: 0.8720\n",
      "Epoch: [ 0] [  142/  468] time: 13.8962, train_loss: 0.42612901, train_accuracy: 0.9141, test_Accuracy: 0.8708\n",
      "Epoch: [ 0] [  143/  468] time: 13.9840, train_loss: 0.45880291, train_accuracy: 0.9297, test_Accuracy: 0.8725\n",
      "Epoch: [ 0] [  144/  468] time: 14.0797, train_loss: 0.40134048, train_accuracy: 0.9297, test_Accuracy: 0.8726\n",
      "Epoch: [ 0] [  145/  468] time: 14.1786, train_loss: 0.42342103, train_accuracy: 0.8984, test_Accuracy: 0.8730\n",
      "Epoch: [ 0] [  146/  468] time: 14.2753, train_loss: 0.50666487, train_accuracy: 0.8828, test_Accuracy: 0.8750\n",
      "Epoch: [ 0] [  147/  468] time: 14.3675, train_loss: 0.59033608, train_accuracy: 0.7891, test_Accuracy: 0.8767\n",
      "Epoch: [ 0] [  148/  468] time: 14.4643, train_loss: 0.49821934, train_accuracy: 0.8438, test_Accuracy: 0.8776\n",
      "Epoch: [ 0] [  149/  468] time: 14.5529, train_loss: 0.46536845, train_accuracy: 0.8594, test_Accuracy: 0.8785\n",
      "Epoch: [ 0] [  150/  468] time: 14.6442, train_loss: 0.58102626, train_accuracy: 0.8281, test_Accuracy: 0.8764\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  151/  468] time: 14.7386, train_loss: 0.52600503, train_accuracy: 0.8750, test_Accuracy: 0.8723\n",
      "Epoch: [ 0] [  152/  468] time: 14.8294, train_loss: 0.45337841, train_accuracy: 0.8984, test_Accuracy: 0.8696\n",
      "Epoch: [ 0] [  153/  468] time: 14.9179, train_loss: 0.42315960, train_accuracy: 0.9141, test_Accuracy: 0.8695\n",
      "Epoch: [ 0] [  154/  468] time: 15.0111, train_loss: 0.41338128, train_accuracy: 0.9062, test_Accuracy: 0.8707\n",
      "Epoch: [ 0] [  155/  468] time: 15.1031, train_loss: 0.50799429, train_accuracy: 0.8359, test_Accuracy: 0.8753\n",
      "Epoch: [ 0] [  156/  468] time: 15.1913, train_loss: 0.49137384, train_accuracy: 0.8906, test_Accuracy: 0.8784\n",
      "Epoch: [ 0] [  157/  468] time: 15.2799, train_loss: 0.41322923, train_accuracy: 0.8750, test_Accuracy: 0.8796\n",
      "Epoch: [ 0] [  158/  468] time: 15.3736, train_loss: 0.46074125, train_accuracy: 0.8594, test_Accuracy: 0.8799\n",
      "Epoch: [ 0] [  159/  468] time: 15.4692, train_loss: 0.39120460, train_accuracy: 0.9375, test_Accuracy: 0.8784\n",
      "Epoch: [ 0] [  160/  468] time: 15.5652, train_loss: 0.53415722, train_accuracy: 0.8438, test_Accuracy: 0.8755\n",
      "Epoch: [ 0] [  161/  468] time: 15.6582, train_loss: 0.48535097, train_accuracy: 0.8750, test_Accuracy: 0.8748\n",
      "Epoch: [ 0] [  162/  468] time: 15.7512, train_loss: 0.33271372, train_accuracy: 0.9297, test_Accuracy: 0.8745\n",
      "Epoch: [ 0] [  163/  468] time: 15.8436, train_loss: 0.39567536, train_accuracy: 0.9219, test_Accuracy: 0.8775\n",
      "Epoch: [ 0] [  164/  468] time: 15.9353, train_loss: 0.53843653, train_accuracy: 0.8438, test_Accuracy: 0.8806\n",
      "Epoch: [ 0] [  165/  468] time: 16.0313, train_loss: 0.35650146, train_accuracy: 0.9141, test_Accuracy: 0.8828\n",
      "Epoch: [ 0] [  166/  468] time: 16.1206, train_loss: 0.41717756, train_accuracy: 0.8984, test_Accuracy: 0.8851\n",
      "Epoch: [ 0] [  167/  468] time: 16.2114, train_loss: 0.40630275, train_accuracy: 0.8984, test_Accuracy: 0.8862\n",
      "Epoch: [ 0] [  168/  468] time: 16.3065, train_loss: 0.35101116, train_accuracy: 0.9141, test_Accuracy: 0.8866\n",
      "Epoch: [ 0] [  169/  468] time: 16.3984, train_loss: 0.46528679, train_accuracy: 0.8828, test_Accuracy: 0.8875\n",
      "Epoch: [ 0] [  170/  468] time: 16.4879, train_loss: 0.52859575, train_accuracy: 0.8125, test_Accuracy: 0.8876\n",
      "Epoch: [ 0] [  171/  468] time: 16.5787, train_loss: 0.37829363, train_accuracy: 0.8984, test_Accuracy: 0.8863\n",
      "Epoch: [ 0] [  172/  468] time: 16.6756, train_loss: 0.52171314, train_accuracy: 0.8516, test_Accuracy: 0.8859\n",
      "Epoch: [ 0] [  173/  468] time: 16.7672, train_loss: 0.42958763, train_accuracy: 0.8984, test_Accuracy: 0.8846\n",
      "Epoch: [ 0] [  174/  468] time: 16.8589, train_loss: 0.36994433, train_accuracy: 0.8906, test_Accuracy: 0.8855\n",
      "Epoch: [ 0] [  175/  468] time: 16.9488, train_loss: 0.47720116, train_accuracy: 0.8828, test_Accuracy: 0.8851\n",
      "Epoch: [ 0] [  176/  468] time: 17.0353, train_loss: 0.41798598, train_accuracy: 0.8984, test_Accuracy: 0.8846\n",
      "Epoch: [ 0] [  177/  468] time: 17.1226, train_loss: 0.49811107, train_accuracy: 0.8672, test_Accuracy: 0.8844\n",
      "Epoch: [ 0] [  178/  468] time: 17.2124, train_loss: 0.41561753, train_accuracy: 0.8828, test_Accuracy: 0.8841\n",
      "Epoch: [ 0] [  179/  468] time: 17.3063, train_loss: 0.35177103, train_accuracy: 0.9219, test_Accuracy: 0.8844\n",
      "Epoch: [ 0] [  180/  468] time: 17.3944, train_loss: 0.51375705, train_accuracy: 0.8516, test_Accuracy: 0.8871\n",
      "Epoch: [ 0] [  181/  468] time: 17.4834, train_loss: 0.45888060, train_accuracy: 0.8672, test_Accuracy: 0.8886\n",
      "Epoch: [ 0] [  182/  468] time: 17.5760, train_loss: 0.30315679, train_accuracy: 0.9297, test_Accuracy: 0.8897\n",
      "Epoch: [ 0] [  183/  468] time: 17.6691, train_loss: 0.45823449, train_accuracy: 0.8594, test_Accuracy: 0.8896\n",
      "Epoch: [ 0] [  184/  468] time: 17.7684, train_loss: 0.42562830, train_accuracy: 0.8750, test_Accuracy: 0.8897\n",
      "Epoch: [ 0] [  185/  468] time: 17.8588, train_loss: 0.42053729, train_accuracy: 0.8906, test_Accuracy: 0.8905\n",
      "Epoch: [ 0] [  186/  468] time: 17.9489, train_loss: 0.51157373, train_accuracy: 0.8359, test_Accuracy: 0.8917\n",
      "Epoch: [ 0] [  187/  468] time: 18.0389, train_loss: 0.38717645, train_accuracy: 0.9141, test_Accuracy: 0.8938\n",
      "Epoch: [ 0] [  188/  468] time: 18.1301, train_loss: 0.50672406, train_accuracy: 0.8594, test_Accuracy: 0.8939\n",
      "Epoch: [ 0] [  189/  468] time: 18.2223, train_loss: 0.34249145, train_accuracy: 0.9297, test_Accuracy: 0.8949\n",
      "Epoch: [ 0] [  190/  468] time: 18.3119, train_loss: 0.47548071, train_accuracy: 0.8672, test_Accuracy: 0.8952\n",
      "Epoch: [ 0] [  191/  468] time: 18.4026, train_loss: 0.41676724, train_accuracy: 0.8906, test_Accuracy: 0.8952\n",
      "Epoch: [ 0] [  192/  468] time: 18.4926, train_loss: 0.41844276, train_accuracy: 0.8828, test_Accuracy: 0.8954\n",
      "Epoch: [ 0] [  193/  468] time: 18.5907, train_loss: 0.37110278, train_accuracy: 0.9062, test_Accuracy: 0.8950\n",
      "Epoch: [ 0] [  194/  468] time: 18.6852, train_loss: 0.38015196, train_accuracy: 0.9062, test_Accuracy: 0.8948\n",
      "Epoch: [ 0] [  195/  468] time: 18.7760, train_loss: 0.41967636, train_accuracy: 0.8984, test_Accuracy: 0.8950\n",
      "Epoch: [ 0] [  196/  468] time: 18.8669, train_loss: 0.39906788, train_accuracy: 0.8750, test_Accuracy: 0.8957\n",
      "Epoch: [ 0] [  197/  468] time: 18.9555, train_loss: 0.42403877, train_accuracy: 0.8750, test_Accuracy: 0.8967\n",
      "Epoch: [ 0] [  198/  468] time: 19.0455, train_loss: 0.38919809, train_accuracy: 0.8906, test_Accuracy: 0.8958\n",
      "Epoch: [ 0] [  199/  468] time: 19.1393, train_loss: 0.46707922, train_accuracy: 0.8828, test_Accuracy: 0.8970\n",
      "Epoch: [ 0] [  200/  468] time: 19.2277, train_loss: 0.49415746, train_accuracy: 0.8516, test_Accuracy: 0.8968\n",
      "Epoch: [ 0] [  201/  468] time: 19.3189, train_loss: 0.38858879, train_accuracy: 0.8594, test_Accuracy: 0.8963\n",
      "Epoch: [ 0] [  202/  468] time: 19.4129, train_loss: 0.29132032, train_accuracy: 0.9219, test_Accuracy: 0.8964\n",
      "Epoch: [ 0] [  203/  468] time: 19.5045, train_loss: 0.39450863, train_accuracy: 0.9062, test_Accuracy: 0.8972\n",
      "Epoch: [ 0] [  204/  468] time: 19.5962, train_loss: 0.42248547, train_accuracy: 0.8828, test_Accuracy: 0.8977\n",
      "Epoch: [ 0] [  205/  468] time: 19.6866, train_loss: 0.44939125, train_accuracy: 0.8906, test_Accuracy: 0.8988\n",
      "Epoch: [ 0] [  206/  468] time: 19.7789, train_loss: 0.35259736, train_accuracy: 0.8984, test_Accuracy: 0.8988\n",
      "Epoch: [ 0] [  207/  468] time: 19.8710, train_loss: 0.40953326, train_accuracy: 0.8828, test_Accuracy: 0.8985\n",
      "Epoch: [ 0] [  208/  468] time: 19.9621, train_loss: 0.38254321, train_accuracy: 0.9219, test_Accuracy: 0.8988\n",
      "Epoch: [ 0] [  209/  468] time: 20.0556, train_loss: 0.25269032, train_accuracy: 0.9453, test_Accuracy: 0.8985\n",
      "Epoch: [ 0] [  210/  468] time: 20.1429, train_loss: 0.47455853, train_accuracy: 0.9062, test_Accuracy: 0.8986\n",
      "Epoch: [ 0] [  211/  468] time: 20.2317, train_loss: 0.34532171, train_accuracy: 0.8828, test_Accuracy: 0.8985\n",
      "Epoch: [ 0] [  212/  468] time: 20.3193, train_loss: 0.40488589, train_accuracy: 0.8828, test_Accuracy: 0.8991\n",
      "Epoch: [ 0] [  213/  468] time: 20.4155, train_loss: 0.38792241, train_accuracy: 0.9219, test_Accuracy: 0.9005\n",
      "Epoch: [ 0] [  214/  468] time: 20.5105, train_loss: 0.27501667, train_accuracy: 0.9453, test_Accuracy: 0.9011\n",
      "Epoch: [ 0] [  215/  468] time: 20.6040, train_loss: 0.30342370, train_accuracy: 0.9062, test_Accuracy: 0.9022\n",
      "Epoch: [ 0] [  216/  468] time: 20.7007, train_loss: 0.43920779, train_accuracy: 0.8828, test_Accuracy: 0.9023\n",
      "Epoch: [ 0] [  217/  468] time: 20.7932, train_loss: 0.27882165, train_accuracy: 0.9219, test_Accuracy: 0.9015\n",
      "Epoch: [ 0] [  218/  468] time: 20.8820, train_loss: 0.42103428, train_accuracy: 0.8828, test_Accuracy: 0.9014\n",
      "Epoch: [ 0] [  219/  468] time: 20.9718, train_loss: 0.33856541, train_accuracy: 0.8984, test_Accuracy: 0.9014\n",
      "Epoch: [ 0] [  220/  468] time: 21.0636, train_loss: 0.38800701, train_accuracy: 0.8594, test_Accuracy: 0.9014\n",
      "Epoch: [ 0] [  221/  468] time: 21.1577, train_loss: 0.38346148, train_accuracy: 0.8906, test_Accuracy: 0.9004\n",
      "Epoch: [ 0] [  222/  468] time: 21.2489, train_loss: 0.31571445, train_accuracy: 0.9297, test_Accuracy: 0.9004\n",
      "Epoch: [ 0] [  223/  468] time: 21.3379, train_loss: 0.43247849, train_accuracy: 0.8750, test_Accuracy: 0.8994\n",
      "Epoch: [ 0] [  224/  468] time: 21.4322, train_loss: 0.54533589, train_accuracy: 0.8438, test_Accuracy: 0.9011\n",
      "Epoch: [ 0] [  225/  468] time: 21.5200, train_loss: 0.34209388, train_accuracy: 0.8750, test_Accuracy: 0.9008\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  226/  468] time: 21.6090, train_loss: 0.35609168, train_accuracy: 0.8984, test_Accuracy: 0.9031\n",
      "Epoch: [ 0] [  227/  468] time: 21.6981, train_loss: 0.32984114, train_accuracy: 0.9453, test_Accuracy: 0.9039\n",
      "Epoch: [ 0] [  228/  468] time: 21.7895, train_loss: 0.44402510, train_accuracy: 0.8594, test_Accuracy: 0.9038\n",
      "Epoch: [ 0] [  229/  468] time: 21.8872, train_loss: 0.37383503, train_accuracy: 0.8984, test_Accuracy: 0.9021\n",
      "Epoch: [ 0] [  230/  468] time: 21.9783, train_loss: 0.26115927, train_accuracy: 0.9375, test_Accuracy: 0.9031\n",
      "Epoch: [ 0] [  231/  468] time: 22.0693, train_loss: 0.41544580, train_accuracy: 0.8828, test_Accuracy: 0.9012\n",
      "Epoch: [ 0] [  232/  468] time: 22.1633, train_loss: 0.37542719, train_accuracy: 0.8828, test_Accuracy: 0.8991\n",
      "Epoch: [ 0] [  233/  468] time: 22.2576, train_loss: 0.29735994, train_accuracy: 0.9062, test_Accuracy: 0.8992\n",
      "Epoch: [ 0] [  234/  468] time: 22.3474, train_loss: 0.35634488, train_accuracy: 0.8984, test_Accuracy: 0.8994\n",
      "Epoch: [ 0] [  235/  468] time: 22.4409, train_loss: 0.33675182, train_accuracy: 0.9062, test_Accuracy: 0.9015\n",
      "Epoch: [ 0] [  236/  468] time: 22.5350, train_loss: 0.38712561, train_accuracy: 0.8750, test_Accuracy: 0.9024\n",
      "Epoch: [ 0] [  237/  468] time: 22.6317, train_loss: 0.33281475, train_accuracy: 0.9219, test_Accuracy: 0.9029\n",
      "Epoch: [ 0] [  238/  468] time: 22.7274, train_loss: 0.35518402, train_accuracy: 0.9297, test_Accuracy: 0.9043\n",
      "Epoch: [ 0] [  239/  468] time: 22.8200, train_loss: 0.30094147, train_accuracy: 0.9141, test_Accuracy: 0.9047\n",
      "Epoch: [ 0] [  240/  468] time: 22.9163, train_loss: 0.38121238, train_accuracy: 0.8750, test_Accuracy: 0.9051\n",
      "Epoch: [ 0] [  241/  468] time: 23.0074, train_loss: 0.28706649, train_accuracy: 0.9297, test_Accuracy: 0.9051\n",
      "Epoch: [ 0] [  242/  468] time: 23.1031, train_loss: 0.41614306, train_accuracy: 0.8750, test_Accuracy: 0.9058\n",
      "Epoch: [ 0] [  243/  468] time: 23.1920, train_loss: 0.29113853, train_accuracy: 0.9297, test_Accuracy: 0.9059\n",
      "Epoch: [ 0] [  244/  468] time: 23.2855, train_loss: 0.25271896, train_accuracy: 0.9453, test_Accuracy: 0.9057\n",
      "Epoch: [ 0] [  245/  468] time: 23.3752, train_loss: 0.32379913, train_accuracy: 0.9062, test_Accuracy: 0.9057\n",
      "Epoch: [ 0] [  246/  468] time: 23.4682, train_loss: 0.27212742, train_accuracy: 0.9375, test_Accuracy: 0.9056\n",
      "Epoch: [ 0] [  247/  468] time: 23.5551, train_loss: 0.29298511, train_accuracy: 0.9141, test_Accuracy: 0.9061\n",
      "Epoch: [ 0] [  248/  468] time: 23.6469, train_loss: 0.24868008, train_accuracy: 0.9375, test_Accuracy: 0.9058\n",
      "Epoch: [ 0] [  249/  468] time: 23.7403, train_loss: 0.39716396, train_accuracy: 0.9141, test_Accuracy: 0.9059\n",
      "Epoch: [ 0] [  250/  468] time: 23.8350, train_loss: 0.45607537, train_accuracy: 0.8828, test_Accuracy: 0.9061\n",
      "Epoch: [ 0] [  251/  468] time: 23.9299, train_loss: 0.36213246, train_accuracy: 0.8828, test_Accuracy: 0.9065\n",
      "Epoch: [ 0] [  252/  468] time: 24.0181, train_loss: 0.29759699, train_accuracy: 0.9062, test_Accuracy: 0.9068\n",
      "Epoch: [ 0] [  253/  468] time: 24.1100, train_loss: 0.32673037, train_accuracy: 0.9141, test_Accuracy: 0.9072\n",
      "Epoch: [ 0] [  254/  468] time: 24.2035, train_loss: 0.29732496, train_accuracy: 0.9062, test_Accuracy: 0.9075\n",
      "Epoch: [ 0] [  255/  468] time: 24.2938, train_loss: 0.25684521, train_accuracy: 0.9453, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [  256/  468] time: 24.3889, train_loss: 0.32835770, train_accuracy: 0.9453, test_Accuracy: 0.9074\n",
      "Epoch: [ 0] [  257/  468] time: 24.4836, train_loss: 0.31745702, train_accuracy: 0.9062, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [  258/  468] time: 24.5794, train_loss: 0.32631463, train_accuracy: 0.8984, test_Accuracy: 0.9071\n",
      "Epoch: [ 0] [  259/  468] time: 24.6689, train_loss: 0.32251552, train_accuracy: 0.9141, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [  260/  468] time: 24.7604, train_loss: 0.35258281, train_accuracy: 0.8828, test_Accuracy: 0.9084\n",
      "Epoch: [ 0] [  261/  468] time: 24.8507, train_loss: 0.43125191, train_accuracy: 0.8906, test_Accuracy: 0.9086\n",
      "Epoch: [ 0] [  262/  468] time: 24.9406, train_loss: 0.26956993, train_accuracy: 0.8984, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [  263/  468] time: 25.0310, train_loss: 0.27673668, train_accuracy: 0.9219, test_Accuracy: 0.9073\n",
      "Epoch: [ 0] [  264/  468] time: 25.1202, train_loss: 0.25590661, train_accuracy: 0.9219, test_Accuracy: 0.9064\n",
      "Epoch: [ 0] [  265/  468] time: 25.2123, train_loss: 0.38301826, train_accuracy: 0.8984, test_Accuracy: 0.9059\n",
      "Epoch: [ 0] [  266/  468] time: 25.3025, train_loss: 0.24090108, train_accuracy: 0.9219, test_Accuracy: 0.9056\n",
      "Epoch: [ 0] [  267/  468] time: 25.3997, train_loss: 0.33920467, train_accuracy: 0.9062, test_Accuracy: 0.9058\n",
      "Epoch: [ 0] [  268/  468] time: 25.4925, train_loss: 0.41726595, train_accuracy: 0.8828, test_Accuracy: 0.9070\n",
      "Epoch: [ 0] [  269/  468] time: 25.5817, train_loss: 0.36037913, train_accuracy: 0.8906, test_Accuracy: 0.9069\n",
      "Epoch: [ 0] [  270/  468] time: 25.6734, train_loss: 0.31559128, train_accuracy: 0.9375, test_Accuracy: 0.9067\n",
      "Epoch: [ 0] [  271/  468] time: 25.7670, train_loss: 0.34569168, train_accuracy: 0.9062, test_Accuracy: 0.9072\n",
      "Epoch: [ 0] [  272/  468] time: 25.8534, train_loss: 0.23418695, train_accuracy: 0.9453, test_Accuracy: 0.9076\n",
      "Epoch: [ 0] [  273/  468] time: 25.9466, train_loss: 0.37305069, train_accuracy: 0.8828, test_Accuracy: 0.9068\n",
      "Epoch: [ 0] [  274/  468] time: 26.0417, train_loss: 0.38843194, train_accuracy: 0.8672, test_Accuracy: 0.9071\n",
      "Epoch: [ 0] [  275/  468] time: 26.1313, train_loss: 0.31457895, train_accuracy: 0.9141, test_Accuracy: 0.9072\n",
      "Epoch: [ 0] [  276/  468] time: 26.2254, train_loss: 0.22249950, train_accuracy: 0.9531, test_Accuracy: 0.9085\n",
      "Epoch: [ 0] [  277/  468] time: 26.3182, train_loss: 0.27017456, train_accuracy: 0.9609, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [  278/  468] time: 26.4096, train_loss: 0.18637277, train_accuracy: 0.9531, test_Accuracy: 0.9103\n",
      "Epoch: [ 0] [  279/  468] time: 26.5054, train_loss: 0.31665272, train_accuracy: 0.8984, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [  280/  468] time: 26.5947, train_loss: 0.46329752, train_accuracy: 0.8359, test_Accuracy: 0.9109\n",
      "Epoch: [ 0] [  281/  468] time: 26.6908, train_loss: 0.45397040, train_accuracy: 0.8594, test_Accuracy: 0.9103\n",
      "Epoch: [ 0] [  282/  468] time: 26.7832, train_loss: 0.19129288, train_accuracy: 0.9375, test_Accuracy: 0.9100\n",
      "Epoch: [ 0] [  283/  468] time: 26.8800, train_loss: 0.26096123, train_accuracy: 0.9297, test_Accuracy: 0.9107\n",
      "Epoch: [ 0] [  284/  468] time: 26.9684, train_loss: 0.33877811, train_accuracy: 0.8984, test_Accuracy: 0.9105\n",
      "Epoch: [ 0] [  285/  468] time: 27.0615, train_loss: 0.27141905, train_accuracy: 0.9141, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [  286/  468] time: 27.1555, train_loss: 0.29296565, train_accuracy: 0.9141, test_Accuracy: 0.9088\n",
      "Epoch: [ 0] [  287/  468] time: 27.2486, train_loss: 0.31932664, train_accuracy: 0.8828, test_Accuracy: 0.9087\n",
      "Epoch: [ 0] [  288/  468] time: 27.3418, train_loss: 0.33719724, train_accuracy: 0.9141, test_Accuracy: 0.9091\n",
      "Epoch: [ 0] [  289/  468] time: 27.4335, train_loss: 0.33970734, train_accuracy: 0.8906, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [  290/  468] time: 27.5276, train_loss: 0.37615311, train_accuracy: 0.8672, test_Accuracy: 0.9099\n",
      "Epoch: [ 0] [  291/  468] time: 27.6209, train_loss: 0.43448341, train_accuracy: 0.8828, test_Accuracy: 0.9115\n",
      "Epoch: [ 0] [  292/  468] time: 27.7114, train_loss: 0.27039340, train_accuracy: 0.9219, test_Accuracy: 0.9122\n",
      "Epoch: [ 0] [  293/  468] time: 27.8051, train_loss: 0.38417992, train_accuracy: 0.8828, test_Accuracy: 0.9132\n",
      "Epoch: [ 0] [  294/  468] time: 27.8974, train_loss: 0.33692411, train_accuracy: 0.9141, test_Accuracy: 0.9126\n",
      "Epoch: [ 0] [  295/  468] time: 27.9918, train_loss: 0.31962138, train_accuracy: 0.9219, test_Accuracy: 0.9124\n",
      "Epoch: [ 0] [  296/  468] time: 28.0832, train_loss: 0.28448331, train_accuracy: 0.9375, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [  297/  468] time: 28.1800, train_loss: 0.38120931, train_accuracy: 0.8984, test_Accuracy: 0.9110\n",
      "Epoch: [ 0] [  298/  468] time: 28.2756, train_loss: 0.26947951, train_accuracy: 0.9141, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [  299/  468] time: 28.3678, train_loss: 0.23900807, train_accuracy: 0.9531, test_Accuracy: 0.9113\n",
      "Epoch: [ 0] [  300/  468] time: 28.4593, train_loss: 0.32880336, train_accuracy: 0.9062, test_Accuracy: 0.9105\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  301/  468] time: 28.5559, train_loss: 0.15861154, train_accuracy: 0.9766, test_Accuracy: 0.9103\n",
      "Epoch: [ 0] [  302/  468] time: 28.6478, train_loss: 0.22605327, train_accuracy: 0.9375, test_Accuracy: 0.9098\n",
      "Epoch: [ 0] [  303/  468] time: 28.7412, train_loss: 0.39532244, train_accuracy: 0.8828, test_Accuracy: 0.9103\n",
      "Epoch: [ 0] [  304/  468] time: 28.8278, train_loss: 0.37956220, train_accuracy: 0.8906, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [  305/  468] time: 28.9128, train_loss: 0.35492954, train_accuracy: 0.8906, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [  306/  468] time: 28.9992, train_loss: 0.23030055, train_accuracy: 0.9375, test_Accuracy: 0.9123\n",
      "Epoch: [ 0] [  307/  468] time: 29.0873, train_loss: 0.28519267, train_accuracy: 0.9375, test_Accuracy: 0.9131\n",
      "Epoch: [ 0] [  308/  468] time: 29.1702, train_loss: 0.44649100, train_accuracy: 0.8594, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [  309/  468] time: 29.2606, train_loss: 0.27539647, train_accuracy: 0.9062, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [  310/  468] time: 29.3500, train_loss: 0.33375245, train_accuracy: 0.8750, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [  311/  468] time: 29.4447, train_loss: 0.27967557, train_accuracy: 0.9219, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [  312/  468] time: 29.5361, train_loss: 0.45650524, train_accuracy: 0.8750, test_Accuracy: 0.9126\n",
      "Epoch: [ 0] [  313/  468] time: 29.6272, train_loss: 0.24364725, train_accuracy: 0.9531, test_Accuracy: 0.9113\n",
      "Epoch: [ 0] [  314/  468] time: 29.7194, train_loss: 0.31718957, train_accuracy: 0.9141, test_Accuracy: 0.9118\n",
      "Epoch: [ 0] [  315/  468] time: 29.8078, train_loss: 0.33713269, train_accuracy: 0.8828, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [  316/  468] time: 29.9011, train_loss: 0.33488843, train_accuracy: 0.8984, test_Accuracy: 0.9138\n",
      "Epoch: [ 0] [  317/  468] time: 29.9908, train_loss: 0.37764323, train_accuracy: 0.8594, test_Accuracy: 0.9154\n",
      "Epoch: [ 0] [  318/  468] time: 30.0838, train_loss: 0.26835591, train_accuracy: 0.9375, test_Accuracy: 0.9153\n",
      "Epoch: [ 0] [  319/  468] time: 30.1754, train_loss: 0.32175285, train_accuracy: 0.9141, test_Accuracy: 0.9168\n",
      "Epoch: [ 0] [  320/  468] time: 30.2668, train_loss: 0.30519751, train_accuracy: 0.9141, test_Accuracy: 0.9173\n",
      "Epoch: [ 0] [  321/  468] time: 30.3607, train_loss: 0.29601878, train_accuracy: 0.9062, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [  322/  468] time: 30.4543, train_loss: 0.24152222, train_accuracy: 0.9297, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [  323/  468] time: 30.5434, train_loss: 0.33099940, train_accuracy: 0.8828, test_Accuracy: 0.9165\n",
      "Epoch: [ 0] [  324/  468] time: 30.6412, train_loss: 0.27782941, train_accuracy: 0.9297, test_Accuracy: 0.9166\n",
      "Epoch: [ 0] [  325/  468] time: 30.7445, train_loss: 0.36524838, train_accuracy: 0.8828, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [  326/  468] time: 30.8382, train_loss: 0.21756932, train_accuracy: 0.9453, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [  327/  468] time: 30.9278, train_loss: 0.22408034, train_accuracy: 0.9531, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [  328/  468] time: 31.0192, train_loss: 0.32583964, train_accuracy: 0.9062, test_Accuracy: 0.9154\n",
      "Epoch: [ 0] [  329/  468] time: 31.1117, train_loss: 0.23178424, train_accuracy: 0.9453, test_Accuracy: 0.9147\n",
      "Epoch: [ 0] [  330/  468] time: 31.2026, train_loss: 0.38295540, train_accuracy: 0.9141, test_Accuracy: 0.9155\n",
      "Epoch: [ 0] [  331/  468] time: 31.2970, train_loss: 0.30912760, train_accuracy: 0.8984, test_Accuracy: 0.9152\n",
      "Epoch: [ 0] [  332/  468] time: 31.3916, train_loss: 0.35263470, train_accuracy: 0.8906, test_Accuracy: 0.9151\n",
      "Epoch: [ 0] [  333/  468] time: 31.4849, train_loss: 0.34957403, train_accuracy: 0.8750, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [  334/  468] time: 31.5780, train_loss: 0.24377742, train_accuracy: 0.9453, test_Accuracy: 0.9163\n",
      "Epoch: [ 0] [  335/  468] time: 31.6703, train_loss: 0.28367165, train_accuracy: 0.9062, test_Accuracy: 0.9161\n",
      "Epoch: [ 0] [  336/  468] time: 31.7600, train_loss: 0.32799089, train_accuracy: 0.8984, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [  337/  468] time: 31.8486, train_loss: 0.18198045, train_accuracy: 0.9375, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [  338/  468] time: 31.9400, train_loss: 0.26950175, train_accuracy: 0.9297, test_Accuracy: 0.9172\n",
      "Epoch: [ 0] [  339/  468] time: 32.0312, train_loss: 0.25629258, train_accuracy: 0.9219, test_Accuracy: 0.9154\n",
      "Epoch: [ 0] [  340/  468] time: 32.1282, train_loss: 0.32791883, train_accuracy: 0.8984, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [  341/  468] time: 32.2207, train_loss: 0.21092992, train_accuracy: 0.9531, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [  342/  468] time: 32.3082, train_loss: 0.32126978, train_accuracy: 0.8984, test_Accuracy: 0.9145\n",
      "Epoch: [ 0] [  343/  468] time: 32.3967, train_loss: 0.29902714, train_accuracy: 0.9062, test_Accuracy: 0.9143\n",
      "Epoch: [ 0] [  344/  468] time: 32.4880, train_loss: 0.33935162, train_accuracy: 0.9219, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [  345/  468] time: 32.5818, train_loss: 0.31895360, train_accuracy: 0.8984, test_Accuracy: 0.9150\n",
      "Epoch: [ 0] [  346/  468] time: 32.6758, train_loss: 0.24467310, train_accuracy: 0.8984, test_Accuracy: 0.9154\n",
      "Epoch: [ 0] [  347/  468] time: 32.7650, train_loss: 0.26882237, train_accuracy: 0.9297, test_Accuracy: 0.9159\n",
      "Epoch: [ 0] [  348/  468] time: 32.8576, train_loss: 0.28889078, train_accuracy: 0.9062, test_Accuracy: 0.9166\n",
      "Epoch: [ 0] [  349/  468] time: 32.9494, train_loss: 0.19489387, train_accuracy: 0.9531, test_Accuracy: 0.9165\n",
      "Epoch: [ 0] [  350/  468] time: 33.0436, train_loss: 0.21401979, train_accuracy: 0.9375, test_Accuracy: 0.9175\n",
      "Epoch: [ 0] [  351/  468] time: 33.1371, train_loss: 0.27570099, train_accuracy: 0.9219, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [  352/  468] time: 33.2308, train_loss: 0.18800753, train_accuracy: 0.9531, test_Accuracy: 0.9165\n",
      "Epoch: [ 0] [  353/  468] time: 33.3236, train_loss: 0.41038120, train_accuracy: 0.9062, test_Accuracy: 0.9164\n",
      "Epoch: [ 0] [  354/  468] time: 33.4151, train_loss: 0.38161466, train_accuracy: 0.8750, test_Accuracy: 0.9169\n",
      "Epoch: [ 0] [  355/  468] time: 33.5099, train_loss: 0.20704004, train_accuracy: 0.9453, test_Accuracy: 0.9164\n",
      "Epoch: [ 0] [  356/  468] time: 33.6013, train_loss: 0.34347236, train_accuracy: 0.8984, test_Accuracy: 0.9173\n",
      "Epoch: [ 0] [  357/  468] time: 33.6948, train_loss: 0.19204429, train_accuracy: 0.9609, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [  358/  468] time: 33.7900, train_loss: 0.33180475, train_accuracy: 0.8828, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [  359/  468] time: 33.8840, train_loss: 0.20293023, train_accuracy: 0.9375, test_Accuracy: 0.9185\n",
      "Epoch: [ 0] [  360/  468] time: 33.9757, train_loss: 0.29130670, train_accuracy: 0.9375, test_Accuracy: 0.9174\n",
      "Epoch: [ 0] [  361/  468] time: 34.0678, train_loss: 0.36487645, train_accuracy: 0.8906, test_Accuracy: 0.9169\n",
      "Epoch: [ 0] [  362/  468] time: 34.1577, train_loss: 0.50973922, train_accuracy: 0.8438, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [  363/  468] time: 34.2497, train_loss: 0.20042801, train_accuracy: 0.9453, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [  364/  468] time: 34.3434, train_loss: 0.19843271, train_accuracy: 0.9453, test_Accuracy: 0.9173\n",
      "Epoch: [ 0] [  365/  468] time: 34.4333, train_loss: 0.37956780, train_accuracy: 0.8906, test_Accuracy: 0.9172\n",
      "Epoch: [ 0] [  366/  468] time: 34.5257, train_loss: 0.34555769, train_accuracy: 0.8984, test_Accuracy: 0.9184\n",
      "Epoch: [ 0] [  367/  468] time: 34.6168, train_loss: 0.22750556, train_accuracy: 0.9453, test_Accuracy: 0.9186\n",
      "Epoch: [ 0] [  368/  468] time: 34.7088, train_loss: 0.30821440, train_accuracy: 0.8906, test_Accuracy: 0.9182\n",
      "Epoch: [ 0] [  369/  468] time: 34.8032, train_loss: 0.39267689, train_accuracy: 0.8750, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [  370/  468] time: 34.8914, train_loss: 0.30598104, train_accuracy: 0.8906, test_Accuracy: 0.9168\n",
      "Epoch: [ 0] [  371/  468] time: 34.9836, train_loss: 0.32095808, train_accuracy: 0.9219, test_Accuracy: 0.9166\n",
      "Epoch: [ 0] [  372/  468] time: 35.0758, train_loss: 0.32872203, train_accuracy: 0.9219, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [  373/  468] time: 35.1698, train_loss: 0.23961732, train_accuracy: 0.9141, test_Accuracy: 0.9158\n",
      "Epoch: [ 0] [  374/  468] time: 35.2627, train_loss: 0.27815419, train_accuracy: 0.9141, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [  375/  468] time: 35.3556, train_loss: 0.26027000, train_accuracy: 0.8984, test_Accuracy: 0.9167\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  376/  468] time: 35.4512, train_loss: 0.15606324, train_accuracy: 0.9688, test_Accuracy: 0.9180\n",
      "Epoch: [ 0] [  377/  468] time: 35.5446, train_loss: 0.41055191, train_accuracy: 0.8828, test_Accuracy: 0.9189\n",
      "Epoch: [ 0] [  378/  468] time: 35.6386, train_loss: 0.32453680, train_accuracy: 0.9219, test_Accuracy: 0.9200\n",
      "Epoch: [ 0] [  379/  468] time: 35.7337, train_loss: 0.23024115, train_accuracy: 0.9375, test_Accuracy: 0.9200\n",
      "Epoch: [ 0] [  380/  468] time: 35.8236, train_loss: 0.28748322, train_accuracy: 0.9375, test_Accuracy: 0.9203\n",
      "Epoch: [ 0] [  381/  468] time: 35.9163, train_loss: 0.35743976, train_accuracy: 0.8828, test_Accuracy: 0.9197\n",
      "Epoch: [ 0] [  382/  468] time: 36.0071, train_loss: 0.20572484, train_accuracy: 0.9297, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [  383/  468] time: 36.0997, train_loss: 0.27861348, train_accuracy: 0.9297, test_Accuracy: 0.9201\n",
      "Epoch: [ 0] [  384/  468] time: 36.1920, train_loss: 0.24169070, train_accuracy: 0.9297, test_Accuracy: 0.9203\n",
      "Epoch: [ 0] [  385/  468] time: 36.2860, train_loss: 0.20590696, train_accuracy: 0.9297, test_Accuracy: 0.9192\n",
      "Epoch: [ 0] [  386/  468] time: 36.3783, train_loss: 0.27554959, train_accuracy: 0.9453, test_Accuracy: 0.9192\n",
      "Epoch: [ 0] [  387/  468] time: 36.4677, train_loss: 0.37722087, train_accuracy: 0.8672, test_Accuracy: 0.9203\n",
      "Epoch: [ 0] [  388/  468] time: 36.5617, train_loss: 0.29562920, train_accuracy: 0.8906, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [  389/  468] time: 36.6515, train_loss: 0.25429255, train_accuracy: 0.9297, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [  390/  468] time: 36.7414, train_loss: 0.24602583, train_accuracy: 0.9531, test_Accuracy: 0.9189\n",
      "Epoch: [ 0] [  391/  468] time: 36.8317, train_loss: 0.21127254, train_accuracy: 0.9375, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [  392/  468] time: 36.9236, train_loss: 0.24869928, train_accuracy: 0.9297, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [  393/  468] time: 37.0144, train_loss: 0.27638793, train_accuracy: 0.9297, test_Accuracy: 0.9196\n",
      "Epoch: [ 0] [  394/  468] time: 37.1097, train_loss: 0.34737769, train_accuracy: 0.9062, test_Accuracy: 0.9190\n",
      "Epoch: [ 0] [  395/  468] time: 37.2020, train_loss: 0.29003957, train_accuracy: 0.9141, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  396/  468] time: 37.2921, train_loss: 0.18686177, train_accuracy: 0.9531, test_Accuracy: 0.9196\n",
      "Epoch: [ 0] [  397/  468] time: 37.3819, train_loss: 0.44015521, train_accuracy: 0.8594, test_Accuracy: 0.9185\n",
      "Epoch: [ 0] [  398/  468] time: 37.4757, train_loss: 0.27106524, train_accuracy: 0.8984, test_Accuracy: 0.9181\n",
      "Epoch: [ 0] [  399/  468] time: 37.5686, train_loss: 0.27997011, train_accuracy: 0.8984, test_Accuracy: 0.9175\n",
      "Epoch: [ 0] [  400/  468] time: 37.6598, train_loss: 0.22601667, train_accuracy: 0.9219, test_Accuracy: 0.9172\n",
      "Epoch: [ 0] [  401/  468] time: 37.7514, train_loss: 0.25559714, train_accuracy: 0.9453, test_Accuracy: 0.9180\n",
      "Epoch: [ 0] [  402/  468] time: 37.8420, train_loss: 0.33526117, train_accuracy: 0.9062, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  403/  468] time: 37.9311, train_loss: 0.22688955, train_accuracy: 0.9297, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  404/  468] time: 38.0215, train_loss: 0.16812432, train_accuracy: 0.9453, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  405/  468] time: 38.1134, train_loss: 0.15904582, train_accuracy: 0.9531, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [  406/  468] time: 38.2059, train_loss: 0.29644150, train_accuracy: 0.9141, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  407/  468] time: 38.2990, train_loss: 0.37823290, train_accuracy: 0.8828, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  408/  468] time: 38.3916, train_loss: 0.29715315, train_accuracy: 0.8984, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [  409/  468] time: 38.4796, train_loss: 0.29981589, train_accuracy: 0.9062, test_Accuracy: 0.9204\n",
      "Epoch: [ 0] [  410/  468] time: 38.5740, train_loss: 0.27890134, train_accuracy: 0.9141, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [  411/  468] time: 38.6668, train_loss: 0.27981859, train_accuracy: 0.9453, test_Accuracy: 0.9212\n",
      "Epoch: [ 0] [  412/  468] time: 38.7616, train_loss: 0.34563750, train_accuracy: 0.8828, test_Accuracy: 0.9217\n",
      "Epoch: [ 0] [  413/  468] time: 38.8542, train_loss: 0.27631378, train_accuracy: 0.9062, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [  414/  468] time: 38.9483, train_loss: 0.33621430, train_accuracy: 0.8984, test_Accuracy: 0.9211\n",
      "Epoch: [ 0] [  415/  468] time: 39.0424, train_loss: 0.21993482, train_accuracy: 0.9375, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  416/  468] time: 39.1357, train_loss: 0.23888555, train_accuracy: 0.9219, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  417/  468] time: 39.2282, train_loss: 0.18356709, train_accuracy: 0.9609, test_Accuracy: 0.9208\n",
      "Epoch: [ 0] [  418/  468] time: 39.3217, train_loss: 0.42149454, train_accuracy: 0.8750, test_Accuracy: 0.9208\n",
      "Epoch: [ 0] [  419/  468] time: 39.4159, train_loss: 0.25276524, train_accuracy: 0.9297, test_Accuracy: 0.9209\n",
      "Epoch: [ 0] [  420/  468] time: 39.5051, train_loss: 0.17485386, train_accuracy: 0.9688, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  421/  468] time: 39.5926, train_loss: 0.23153892, train_accuracy: 0.9297, test_Accuracy: 0.9225\n",
      "Epoch: [ 0] [  422/  468] time: 39.6855, train_loss: 0.20041999, train_accuracy: 0.9297, test_Accuracy: 0.9233\n",
      "Epoch: [ 0] [  423/  468] time: 39.7796, train_loss: 0.36689848, train_accuracy: 0.8828, test_Accuracy: 0.9232\n",
      "Epoch: [ 0] [  424/  468] time: 39.8708, train_loss: 0.23068899, train_accuracy: 0.9297, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  425/  468] time: 39.9645, train_loss: 0.34882641, train_accuracy: 0.9141, test_Accuracy: 0.9245\n",
      "Epoch: [ 0] [  426/  468] time: 40.0573, train_loss: 0.33352596, train_accuracy: 0.8828, test_Accuracy: 0.9241\n",
      "Epoch: [ 0] [  427/  468] time: 40.1503, train_loss: 0.34998521, train_accuracy: 0.8984, test_Accuracy: 0.9238\n",
      "Epoch: [ 0] [  428/  468] time: 40.2461, train_loss: 0.29932961, train_accuracy: 0.9297, test_Accuracy: 0.9241\n",
      "Epoch: [ 0] [  429/  468] time: 40.3398, train_loss: 0.25456470, train_accuracy: 0.9297, test_Accuracy: 0.9244\n",
      "Epoch: [ 0] [  430/  468] time: 40.4328, train_loss: 0.31131506, train_accuracy: 0.9062, test_Accuracy: 0.9231\n",
      "Epoch: [ 0] [  431/  468] time: 40.5273, train_loss: 0.29032373, train_accuracy: 0.9375, test_Accuracy: 0.9225\n",
      "Epoch: [ 0] [  432/  468] time: 40.6144, train_loss: 0.17394067, train_accuracy: 0.9453, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  433/  468] time: 40.7096, train_loss: 0.27644709, train_accuracy: 0.9375, test_Accuracy: 0.9210\n",
      "Epoch: [ 0] [  434/  468] time: 40.8022, train_loss: 0.28163147, train_accuracy: 0.9297, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [  435/  468] time: 40.8950, train_loss: 0.30516452, train_accuracy: 0.8906, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  436/  468] time: 40.9904, train_loss: 0.18872938, train_accuracy: 0.9453, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  437/  468] time: 41.0846, train_loss: 0.18164633, train_accuracy: 0.9688, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  438/  468] time: 41.1753, train_loss: 0.43768841, train_accuracy: 0.9062, test_Accuracy: 0.9210\n",
      "Epoch: [ 0] [  439/  468] time: 41.2660, train_loss: 0.36965811, train_accuracy: 0.8906, test_Accuracy: 0.9219\n",
      "Epoch: [ 0] [  440/  468] time: 41.3571, train_loss: 0.17751414, train_accuracy: 0.9531, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  441/  468] time: 41.4462, train_loss: 0.44101661, train_accuracy: 0.8750, test_Accuracy: 0.9232\n",
      "Epoch: [ 0] [  442/  468] time: 41.5394, train_loss: 0.26731634, train_accuracy: 0.9375, test_Accuracy: 0.9249\n",
      "Epoch: [ 0] [  443/  468] time: 41.6319, train_loss: 0.23726903, train_accuracy: 0.9141, test_Accuracy: 0.9253\n",
      "Epoch: [ 0] [  444/  468] time: 41.7250, train_loss: 0.24515179, train_accuracy: 0.9297, test_Accuracy: 0.9244\n",
      "Epoch: [ 0] [  445/  468] time: 41.8154, train_loss: 0.24008435, train_accuracy: 0.9062, test_Accuracy: 0.9243\n",
      "Epoch: [ 0] [  446/  468] time: 41.9032, train_loss: 0.30075634, train_accuracy: 0.9219, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  447/  468] time: 41.9962, train_loss: 0.35409224, train_accuracy: 0.9141, test_Accuracy: 0.9237\n",
      "Epoch: [ 0] [  448/  468] time: 42.0879, train_loss: 0.20177796, train_accuracy: 0.9453, test_Accuracy: 0.9236\n",
      "Epoch: [ 0] [  449/  468] time: 42.1815, train_loss: 0.24415211, train_accuracy: 0.9297, test_Accuracy: 0.9230\n",
      "Epoch: [ 0] [  450/  468] time: 42.2750, train_loss: 0.22365117, train_accuracy: 0.9453, test_Accuracy: 0.9238\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  451/  468] time: 42.3697, train_loss: 0.25468284, train_accuracy: 0.9062, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  452/  468] time: 42.4626, train_loss: 0.15882659, train_accuracy: 0.9688, test_Accuracy: 0.9224\n",
      "Epoch: [ 0] [  453/  468] time: 42.5534, train_loss: 0.29443750, train_accuracy: 0.9375, test_Accuracy: 0.9219\n",
      "Epoch: [ 0] [  454/  468] time: 42.6466, train_loss: 0.25797576, train_accuracy: 0.9219, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  455/  468] time: 42.7405, train_loss: 0.31461453, train_accuracy: 0.8984, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  456/  468] time: 42.8353, train_loss: 0.29421049, train_accuracy: 0.8984, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  457/  468] time: 42.9286, train_loss: 0.33905104, train_accuracy: 0.9062, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  458/  468] time: 43.0237, train_loss: 0.23832510, train_accuracy: 0.9531, test_Accuracy: 0.9238\n",
      "Epoch: [ 0] [  459/  468] time: 43.1206, train_loss: 0.30897641, train_accuracy: 0.8984, test_Accuracy: 0.9241\n",
      "Epoch: [ 0] [  460/  468] time: 43.2111, train_loss: 0.25105867, train_accuracy: 0.9062, test_Accuracy: 0.9237\n",
      "Epoch: [ 0] [  461/  468] time: 43.3029, train_loss: 0.25834784, train_accuracy: 0.9375, test_Accuracy: 0.9239\n",
      "Epoch: [ 0] [  462/  468] time: 43.3944, train_loss: 0.30068132, train_accuracy: 0.9297, test_Accuracy: 0.9241\n",
      "Epoch: [ 0] [  463/  468] time: 43.4889, train_loss: 0.24384390, train_accuracy: 0.9375, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  464/  468] time: 43.5828, train_loss: 0.31982517, train_accuracy: 0.8828, test_Accuracy: 0.9230\n",
      "Epoch: [ 0] [  465/  468] time: 43.6766, train_loss: 0.24961436, train_accuracy: 0.9297, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [  466/  468] time: 43.7672, train_loss: 0.27273089, train_accuracy: 0.9219, test_Accuracy: 0.9218\n",
      "Epoch: [ 0] [  467/  468] time: 43.8612, train_loss: 0.21285188, train_accuracy: 0.9453, test_Accuracy: 0.9228\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 92.41 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}