2. RNN_Multi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.ops import rnn, rnn_cell\n",
    "from tensorflow.python.ops.rnn_cell import BasicRNNCell,BasicLSTMCell,GRUCell,DropoutWrapper,MultiRNNCell\n",
    "import numpy as np\n",
    "\n",
    "# Import MINST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"data/\", one_hot=True)\n",
    "print (\"MNIST loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# Network Parameters\n",
    "n_input = 28 # MNIST data input (img shape: 28*28)\n",
    "n_steps = 28 # timesteps\n",
    "n_hidden = 128 # hidden layer num of features\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "num_layers = 3\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "# Define weights\n",
    "weights = {\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}\n",
    "print (\"parameters ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#def RNN(x, weights, biases):\n",
    "with tf.variable_scope(\"model\"):\n",
    "    #tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "    # Prepare data shape to match `rnn` function requirements\n",
    "    # Current data input shape: (batch_size, n_steps, n_input)\n",
    "    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n",
    "    \n",
    "    # Permuting batch_size and n_steps\n",
    "    x_t = tf.transpose(x, [1, 0, 2])\n",
    "    # Reshaping to (n_steps*batch_size, n_input)\n",
    "    x_t = tf.reshape(x_t, [-1, n_input])\n",
    "    # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n",
    "    x_t = tf.split(0, n_steps, x_t)\n",
    "\n",
    "    # Define a lstm cell with tensorflow\n",
    "    cell = rnn_cell.BasicRNNCell(n_hidden)\n",
    "    \n",
    "    cell = DropoutWrapper(cell, output_keep_prob=0.5)\n",
    "    cell = MultiRNNCell([cell] * num_layers)\n",
    "\n",
    "    # Get lstm cell output\n",
    "    outputs, states = rnn.rnn(cell, x_t, dtype=tf.float32)\n",
    "\n",
    "    # Linear activation, using rnn inner loop last output\n",
    "    pred = tf.matmul(outputs[-1], weights['out']) + biases['out']\n",
    "\n",
    "#pred = RNN(x, weights, biases)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "print (\"Network ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 1280, Minibatch Loss= 4.402740, Training Accuracy= 0.21875\n",
      "Iter 2560, Minibatch Loss= 3.751143, Training Accuracy= 0.28125\n",
      "Iter 3840, Minibatch Loss= 3.877251, Training Accuracy= 0.32812\n",
      "Iter 5120, Minibatch Loss= 2.848077, Training Accuracy= 0.34375"
     ]
    }
   ],
   "source": [
    "# Initializing the variables\n",
    "init = tf.initialize_all_variables()\n",
    "\n",
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        # Reshape data to get 28 seq of 28 elements\n",
    "        batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n",
    "\n",
    "        # Run optimization op (backprop)\n",
    "        _, acc, loss = sess.run([optimizer,accuracy,cost], feed_dict={x: batch_x, y: batch_y})\n",
    "        \n",
    "        if step % display_step == 0:\n",
    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc)\n",
    "        step += 1\n",
    "    print \"Optimization Finished!\"\n",
    "\n",
    "    # Calculate accuracy for 128 mnist test images\n",
    "    test_len = 128\n",
    "    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n",
    "    test_label = mnist.test.labels[:test_len]\n",
    "    print \"Testing Accuracy:\", \\\n",
    "        sess.run(accuracy, feed_dict={x: test_data, y: test_label})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}