Check_OpenDeep.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:286674e960ef3591228848b777c4dbc5fde5619389246e517bdc1f97ee46236e"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## OpenDeep Example"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Manual Run"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from opendeep.log.logger import config_root_logger\n",
      "from opendeep.models.container import Prototype\n",
      "from opendeep.models.single_layer.basic import BasicLayer, SoftmaxLayer\n",
      "from opendeep.optimization.stochastic_gradient_descent import SGD\n",
      "from opendeep.data.standard_datasets.image.mnist import MNIST, datasets\n",
      "from opendeep.monitor.monitor import Monitor\n",
      "from opendeep.monitor.plot import Plot\n",
      "config_root_logger()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "/Library/Python/2.7/site-packages/pandas/io/excel.py:626: UserWarning: Installed openpyxl is not supported at this time. Use >=1.6.1 and <2.0.0.\n",
        "  .format(openpyxl_compat.start_ver, openpyxl_compat.stop_ver))\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mlp = Prototype()\n",
      "mlp.add(BasicLayer(input_size=28*28, output_size=512, activation='rectifier', noise='dropout'))\n",
      "mlp.add(BasicLayer(output_size=512, activation='rectifier', noise='dropout'))\n",
      "mlp.add(SoftmaxLayer(output_size=10))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,930 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.container.Prototype'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,943 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,944 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,945 - opendeep.utils.config - WARNING - Both the config and defaults are None! Please supply at least one.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,946 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,947 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,948 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,950 - opendeep.models.model - DEBUG - <class 'opendeep.models.container.Prototype'> self.args: {'output_size': None, 'params_hook': None, 'inputs_hook': None, 'input_size': 1, 'hiddens_hook': None, 'outdir': 'outputs/prototype/'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,952 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.container.Prototype'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype/config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,955 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,956 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,957 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,960 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 784, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x10e32ce10>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,961 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:27,980 - opendeep.utils.nnet - DEBUG - Creating weights with shape (784, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:28,006 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:28,030 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,135 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (784, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,137 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,141 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,143 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"

       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,144 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,145 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,147 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x10ecd6790>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,148 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,150 - opendeep.utils.nnet - DEBUG - Creating weights with shape (1, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,157 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,169 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,315 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (1, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,316 - opendeep.models.container - INFO - Overriding model <class 'opendeep.models.single_layer.basic.BasicLayer'> with new inputs_hook!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,317 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,318 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,319 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,319 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,320 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': (512, Elemwise{switch,no_inplace}.0), 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x10ecd6790>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,322 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,451 - opendeep.utils.nnet - DEBUG - Creating weights with shape (512, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,484 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,490 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,589 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (512, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,590 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.SoftmaxLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,591 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,592 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,593 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,593 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,595 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> self.args: {'noise': False, 'weights_init': 'uniform', 'noise_level': None, 'output_size': 10, 'params_hook': None, 'activation': 'softmax', 'hiddens_hook': None, 'out_as_probs': False, 'weights_std': 0.005, 'cost': 'nll', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x117a87e10>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,596 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/softmax_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,597 - opendeep.utils.nnet - DEBUG - Creating weights with shape (1, 10) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,598 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,614 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (1, 10) and activation: softmax\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,621 - opendeep.models.single_layer.basic - DEBUG - Using softmax negative log-likelihood cost!!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,628 - opendeep.models.container - INFO - Overriding model <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> with new inputs_hook!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,629 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.SoftmaxLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,630 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,631 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,632 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,633 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> self.args: {'noise': False, 'weights_init': 'uniform', 'noise_level': None, 'output_size': 10, 'params_hook': None, 'activation': 'softmax', 'hiddens_hook': None, 'out_as_probs': False, 'weights_std': 0.005, 'cost': 'nll', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': (512, Elemwise{switch,no_inplace}.0), 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x117e41d90>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,634 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/softmax_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,681 - opendeep.utils.nnet - DEBUG - Creating weights with shape (512, 10) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,682 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,685 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (512, 10) and activation: softmax\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 11:56:41,687 - opendeep.models.single_layer.basic - DEBUG - Using softmax negative log-likelihood cost!!\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mnist_dataset = MNIST(concat_train_valid=False)\n",
      "\n",
      "optimizer = SGD(model=mlp,\n",
      "                dataset=mnist_dataset,\n",
      "                n_epoch=10,\n",
      "                batch_size=600,\n",
      "                learning_rate=.01,\n",
      "                momentum=.9,\n",
      "                nesterov_momentum=True,\n",
      "                save_frequency=500,\n",
      "                early_stop_threshold=0.997)\n",
      "\n",
      "error = Monitor('error', mlp.get_monitors()['softmax_error'], train=True, valid=True, test=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,493 - opendeep.data.standard_datasets.image.mnist - INFO - Loading MNIST with binary=False and one_hot=False\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,494 - opendeep.data.dataset - INFO - Installing dataset mnist.pkl.gz\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,496 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/opendeep/datasets\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,497 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/opendeep/datasets already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,498 - opendeep.utils.file_ops - DEBUG - File /Users/mwarrick/Documents/opendeep/datasets/mnist.pkl doesn't exist!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,499 - opendeep.data.dataset - DEBUG - Found file /Users/mwarrick/Documents/opendeep/datasets/mnist.pkl.gz\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:35,499 - opendeep.data.dataset - INFO - Installation complete. Yay!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,378 - opendeep.data.standard_datasets.image.mnist - DEBUG - Sequencing MNIST with sequence 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,379 - opendeep.data.standard_datasets.image.mnist - DEBUG - Train shape is: (50000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,379 - opendeep.data.standard_datasets.image.mnist - DEBUG - Valid shape is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,380 - opendeep.data.standard_datasets.image.mnist - DEBUG - Test shape is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,381 - opendeep.data.standard_datasets.image.mnist - DEBUG - loading datasets into shared variables\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,937 - opendeep.optimization.optimizer - DEBUG - Initializing optimizer <class 'opendeep.optimization.stochastic_gradient_descent.SGD'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,938 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:37,940 - opendeep.optimization.optimizer - DEBUG - optimizer config args: {'momentum_decay': 'linear', 'early_stop_length': 30, 'save_frequency': 500, 'nesterov_momentum': True, 'minimum_batch_size': 1, 'learning_rate': 0.01, 'batch_size': 600, 'lr_factor': 0.995, 'momentum_factor': 0, 'n_epoch': 10, 'early_stop_threshold': 0.997, 'lr_decay': 'exponential', 'momentum': 0.9}\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "optimizer.train(monitor_channels=error, plot=None)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,099 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Setting up Stochastic Gradient Descent with momentum for optimizer...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,114 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,127 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,155 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,164 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,186 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,204 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,213 - opendeep.optimization.optimizer - INFO - <class 'opendeep.models.container.Prototype'> params: [W, b, W, b, W, b]\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:38,215 - opendeep.optimization.optimizer - INFO - Compiling f_learn 1/1 function for model <class 'opendeep.models.container.Prototype'>...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:41,236 - opendeep.optimization.optimizer - INFO - f_learn compilation took 3.007529 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:41,238 - opendeep.optimization.optimizer - DEBUG - Compiling monitor functions...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,063 - opendeep.optimization.optimizer - DEBUG - Compilation done. Took 823.6789 milliseconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,063 - opendeep.optimization.optimizer - INFO - -----------TRAINING <class 'opendeep.models.container.Prototype'> function 1/1 FOR 10 EPOCHS (continue_training=False)-----------\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,064 - opendeep.optimization.optimizer - DEBUG - Train dataset size is: (50000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,065 - opendeep.optimization.optimizer - DEBUG - Valid dataset size is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,065 - opendeep.optimization.optimizer - DEBUG - Test dataset size is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,066 - opendeep.optimization.optimizer - INFO - EPOCH 1\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:42,067 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:51,732 - opendeep.optimization.optimizer - INFO - Train cost: 0.083085\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:51,743 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.024722222222222222}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:51,744 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:52,740 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.021960784313725494}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:53,645 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.022499999999999999}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:53,649 - opendeep.optimization.optimizer - INFO - time: 11.58296 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:53,650 - opendeep.optimization.optimizer - INFO - remaining time: 1.737444 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:53,653 - opendeep.optimization.optimizer - INFO - EPOCH 2\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:09:53,654 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:03,601 - opendeep.optimization.optimizer - INFO - Train cost: 0.081086\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:03,605 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.024246031746031751}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:03,606 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:04,376 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.022254901960784314}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:05,126 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.022058823529411766}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:05,130 - opendeep.optimization.optimizer - INFO - time: 11.47646 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:05,131 - opendeep.optimization.optimizer - INFO - remaining time: 1.537294 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:05,133 - opendeep.optimization.optimizer - INFO - EPOCH 3\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:05,133 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:13,482 - opendeep.optimization.optimizer - INFO - Train cost: 0.080406\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:13,483 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.025119047619047624}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:13,484 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,228 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.02161764705882353}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,969 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.022598039215686275}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,973 - opendeep.optimization.optimizer - INFO - time: 9.839857 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,974 - opendeep.optimization.optimizer - INFO - remaining time: 1.279416 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,976 - opendeep.optimization.optimizer - INFO - EPOCH 4\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:14,977 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:24,154 - opendeep.optimization.optimizer - INFO - Train cost: 0.078962\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:24,155 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.023789682539682538}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:24,156 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:24,901 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.021715686274509805}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:25,613 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.021911764705882349}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:25,617 - opendeep.optimization.optimizer - INFO - time: 10.64140 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:25,618 - opendeep.optimization.optimizer - INFO - remaining time: 1.088517 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:25,619 - opendeep.optimization.optimizer - INFO - EPOCH 5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:25,620 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:35,276 - opendeep.optimization.optimizer - INFO - Train cost: 0.078730\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:35,281 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.023928571428571428}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:35,282 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:36,206 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.021323529411764706}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:37,031 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.020980392156862742}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:37,032 - opendeep.optimization.optimizer - INFO - time: 11.41254 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:37,033 - opendeep.optimization.optimizer - INFO - remaining time: 54.95323 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:37,035 - opendeep.optimization.optimizer - INFO - EPOCH 6\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:37,036 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:45,488 - opendeep.optimization.optimizer - INFO - Train cost: 0.076529\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:45,489 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.023253968253968253}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:45,490 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,220 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.02102941176470588}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,961 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.021764705882352939}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,964 - opendeep.optimization.optimizer - INFO - time: 9.928974 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,966 - opendeep.optimization.optimizer - INFO - remaining time: 43.25480 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,967 - opendeep.optimization.optimizer - INFO - EPOCH 7\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:46,968 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:55,367 - opendeep.optimization.optimizer - INFO - Train cost: 0.076379\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:55,375 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.023134920634920633}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:55,376 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,082 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.020784313725490198}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,807 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.021666666666666667}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,807 - opendeep.optimization.optimizer - INFO - time: 9.840363 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,808 - opendeep.optimization.optimizer - INFO - remaining time: 32.02395 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,810 - opendeep.optimization.optimizer - INFO - EPOCH 8\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:10:56,810 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:05,969 - opendeep.optimization.optimizer - INFO - Train cost: 0.075407\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:05,971 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.023571428571428573}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:05,971 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:06,777 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.020980392156862746}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:07,599 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.02073529411764706}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:07,603 - opendeep.optimization.optimizer - INFO - time: 10.79336 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:07,604 - opendeep.optimization.optimizer - INFO - remaining time: 21.37898 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:07,606 - opendeep.optimization.optimizer - INFO - EPOCH 9\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:07,607 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:16,252 - opendeep.optimization.optimizer - INFO - Train cost: 0.074196\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:16,253 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.022281746031746034}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:16,254 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,014 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.020784313725490194}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,722 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.02088235294117647}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,725 - opendeep.optimization.optimizer - INFO - time: 10.11954 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,726 - opendeep.optimization.optimizer - INFO - remaining time: 10.62616 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,728 - opendeep.optimization.optimizer - INFO - EPOCH 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:17,728 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:26,167 - opendeep.optimization.optimizer - INFO - Train cost: 0.071925\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:26,169 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.021805555555555557}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:26,169 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:26,937 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.021225490196078431}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,315 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.020784313725490194}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,323 - opendeep.optimization.optimizer - INFO - Stopping (reached max number of epochs)...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,325 - opendeep.optimization.optimizer - INFO - time: 10.59596 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,326 - opendeep.optimization.optimizer - INFO - remaining time: 0.0 milliseconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,327 - opendeep.optimization.optimizer - DEBUG - Restoring best model parameters...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,337 - opendeep.optimization.optimizer - DEBUG - Saving model parameters...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,339 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.container.Prototype'> parameters to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype/trained_epoch_10.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,390 - opendeep.optimization.optimizer - INFO - ------------TRAIN TIME TOOK 1.772066 minutes---------\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:11:28,403 - opendeep.optimization.optimizer - INFO - ------------TOTAL <class 'opendeep.models.container.Prototype'> TRAIN TIME TOOK 1.772326 minutes---------\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n_examples = 50\n",
      "test_data, test_labels = mnist_dataset.getSubset(datasets.TEST)\n",
      "predictions = mlp.run(test_data[:n_examples].eval())\n",
      "\n",
      "print(\"Predictions:\", predictions)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:12:00,746 - opendeep.models.container - INFO - Compiling f_run...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 12:12:01,035 - opendeep.models.container - INFO - Compilation done! Took 289.4639 milliseconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "('Predictions:', array([7, 2, 1, 0, 4, 1, 4, 9, 5, 9, 0, 6, 9, 0, 1, 5, 9, 7, 3, 4, 9, 6, 6,\n",
        "       5, 4, 0, 7, 4, 0, 1, 3, 1, 3, 4, 7, 2, 7, 1, 2, 1, 1, 7, 4, 2, 3, 5,\n",
        "       1, 2, 4, 4]))\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print(\"Actual     :\", test_labels[:n_examples].eval().astype('int32'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "('Actual     :', array([7, 2, 1, 0, 4, 1, 4, 9, 5, 9, 0, 6, 9, 0, 1, 5, 9, 7, 3, 4, 9, 6, 6,\n",
        "       5, 4, 0, 7, 4, 0, 1, 3, 1, 3, 4, 7, 2, 7, 1, 2, 1, 1, 7, 4, 2, 3, 5,\n",
        "       1, 2, 4, 4], dtype=int32))\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Script Run"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%load_ext autoreload"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%autoreload 2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import lib.opendeep_mnist as opendeep_mnist"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 14:24:10,948 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/opendeep/opendeep/log/logs/error\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 14:24:10,991 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/opendeep/opendeep/log/logs/error already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 14:24:10,992 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/opendeep/opendeep/log/logs/info\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 14:24:10,993 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/opendeep/opendeep/log/logs/info already exists!\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "opendeep_mnist.main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... loading and seting-up data\n",
        "2015-04-22 15:06:29,757 - opendeep.data.standard_datasets.image.mnist - INFO - Loading MNIST with binary=False and one_hot=False\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,758 - opendeep.data.dataset - INFO - Installing dataset mnist.pkl.gz\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,761 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/opendeep/datasets\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,762 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/opendeep/datasets already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,779 - opendeep.utils.file_ops - DEBUG - File /Users/mwarrick/Documents/opendeep/datasets/mnist.pkl doesn't exist!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,780 - opendeep.data.dataset - DEBUG - Found file /Users/mwarrick/Documents/opendeep/datasets/mnist.pkl.gz\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:29,782 - opendeep.data.dataset - INFO - Installation complete. Yay!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:32,507 - opendeep.data.standard_datasets.image.mnist - DEBUG - Sequencing MNIST with sequence 0\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:32,508 - opendeep.data.standard_datasets.image.mnist - DEBUG - Train shape is: (50000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:32,509 - opendeep.data.standard_datasets.image.mnist - DEBUG - Valid shape is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:32,510 - opendeep.data.standard_datasets.image.mnist - DEBUG - Test shape is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:32,511 - opendeep.data.standard_datasets.image.mnist - DEBUG - loading datasets into shared variables\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... building the model structure\n",
        "2015-04-22 15:06:33,014 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.container.Prototype'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,016 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,017 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,018 - opendeep.utils.config - WARNING - Both the config and defaults are None! Please supply at least one.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,020 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,025 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,034 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,044 - opendeep.models.model - DEBUG - <class 'opendeep.models.container.Prototype'> self.args: {'output_size': None, 'params_hook': None, 'inputs_hook': None, 'input_size': 1, 'hiddens_hook': None, 'outdir': 'outputs/prototype/'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,047 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.container.Prototype'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype/config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,053 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,054 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,057 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,059 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,067 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 784, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x11d547490>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,073 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,077 - opendeep.utils.nnet - DEBUG - Creating weights with shape (784, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,130 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,135 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,229 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (784, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,229 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,231 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,232 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,232 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,233 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,234 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x119850a10>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,235 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,236 - opendeep.utils.nnet - DEBUG - Creating weights with shape (1, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,237 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,241 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,354 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (1, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,355 - opendeep.models.container - INFO - Overriding model <class 'opendeep.models.single_layer.basic.BasicLayer'> with new inputs_hook!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,356 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.BasicLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,358 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,359 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,361 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,362 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.BasicLayer'> self.args: {'noise': 'dropout', 'weights_init': 'uniform', 'noise_level': None, 'output_size': 512, 'params_hook': None, 'activation': 'rectifier', 'hiddens_hook': None, 'weights_std': 0.005, 'cost': 'mse', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': (512, Elemwise{switch,no_inplace}.0), 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x119850a10>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,364 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.BasicLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/basiclayer_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,413 - opendeep.utils.nnet - DEBUG - Creating weights with shape (512, 512) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,430 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 512\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,442 - opendeep.models.single_layer.basic - DEBUG - Adding noise switch.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,601 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (512, 512) and activation: rectifier\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,602 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.SoftmaxLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,604 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,606 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,607 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,607 - opendeep.models.model - WARNING - Both input_size and inputs_hook are None! Make sure this is only happening in a Prototype! Setting input_size to 1 for convenience to the Prototype.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,609 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> self.args: {'noise': False, 'weights_init': 'uniform', 'noise_level': None, 'output_size': 10, 'params_hook': None, 'activation': 'softmax', 'hiddens_hook': None, 'out_as_probs': False, 'weights_std': 0.005, 'cost': 'nll', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': None, 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x117ce7510>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,610 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/softmax_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,612 - opendeep.utils.nnet - DEBUG - Creating weights with shape (1, 10) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,616 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,644 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (1, 10) and activation: softmax\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,659 - opendeep.models.single_layer.basic - DEBUG - Using softmax negative log-likelihood cost!!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,667 - opendeep.models.container - INFO - Overriding model <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> with new inputs_hook!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,669 - opendeep.models.model - INFO - Creating a new instance of <class 'opendeep.models.single_layer.basic.SoftmaxLayer'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,670 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,671 - opendeep.utils.file_ops - DEBUG - Attempting to make directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,672 - opendeep.utils.file_ops - DEBUG - Directory /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic already exists!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,673 - opendeep.models.model - DEBUG - <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> self.args: {'noise': False, 'weights_init': 'uniform', 'noise_level': None, 'output_size': 10, 'params_hook': None, 'activation': 'softmax', 'hiddens_hook': None, 'out_as_probs': False, 'weights_std': 0.005, 'cost': 'nll', 'cost_args': {}, 'bias_init': 0.0, 'inputs_hook': (512, Elemwise{switch,no_inplace}.0), 'weights_mean': 0, 'input_size': 1, 'mrg': <theano.sandbox.rng_mrg.MRG_RandomStreams object at 0x118ece590>, 'weights_interval': 'montreal', 'outdir': 'outputs/basic'}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,674 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.single_layer.basic.SoftmaxLayer'> configuration to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/basic/softmax_config.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,716 - opendeep.utils.nnet - DEBUG - Creating weights with shape (512, 10) from Uniform distribution with formula name: montreal\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,718 - opendeep.utils.nnet - DEBUG - Initializing bias variable with shape 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,722 - opendeep.models.single_layer.basic - DEBUG - Initialized a basic fully-connected layer with shape (512, 10) and activation: softmax\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,723 - opendeep.models.single_layer.basic - DEBUG - Using softmax negative log-likelihood cost!!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,728 - opendeep.optimization.optimizer - DEBUG - Initializing optimizer <class 'opendeep.optimization.stochastic_gradient_descent.SGD'>\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,728 - opendeep.utils.config - DEBUG - Input to create_dictionary_like was None.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,730 - opendeep.optimization.optimizer - DEBUG - optimizer config args: {'momentum_decay': 'linear', 'early_stop_length': 30, 'save_frequency': 500, 'nesterov_momentum': True, 'minimum_batch_size': 1, 'learning_rate': 0.01, 'batch_size': 600, 'lr_factor': 0.995, 'momentum_factor': 0, 'n_epoch': 10, 'early_stop_threshold': 0.997, 'lr_decay': 'exponential', 'momentum': 0.9}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... training the model\n",
        "2015-04-22 15:06:33,825 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Setting up Stochastic Gradient Descent with momentum for optimizer...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,832 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,835 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,848 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,854 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,858 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter W\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,862 - opendeep.optimization.stochastic_gradient_descent - DEBUG - Using Nesterov momentum for parameter b\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,864 - opendeep.optimization.optimizer - INFO - <class 'opendeep.models.container.Prototype'> params: [W, b, W, b, W, b]\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:33,865 - opendeep.optimization.optimizer - INFO - Compiling f_learn 1/1 function for model <class 'opendeep.models.container.Prototype'>...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:36,842 - opendeep.optimization.optimizer - INFO - f_learn compilation took 2.975323 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:36,845 - opendeep.optimization.optimizer - DEBUG - Compiling monitor functions...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,955 - opendeep.optimization.optimizer - DEBUG - Compilation done. Took 2.109527 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,956 - opendeep.optimization.optimizer - INFO - -----------TRAINING <class 'opendeep.models.container.Prototype'> function 1/1 FOR 10 EPOCHS (continue_training=False)-----------\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,957 - opendeep.optimization.optimizer - DEBUG - Train dataset size is: (50000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,957 - opendeep.optimization.optimizer - DEBUG - Valid dataset size is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,958 - opendeep.optimization.optimizer - DEBUG - Test dataset size is: (10000, 784)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,960 - opendeep.optimization.optimizer - INFO - EPOCH 1\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:38,962 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:51,088 - opendeep.optimization.optimizer - INFO - Train cost: 1.326765\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:51,089 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.42589285714285713}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:51,090 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:53,585 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.12696078431372551}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:54,860 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.13950980392156861}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:54,865 - opendeep.optimization.optimizer - INFO - time: 15.90473 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:54,869 - opendeep.optimization.optimizer - INFO - remaining time: 2.385709 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:54,894 - opendeep.optimization.optimizer - INFO - EPOCH 2\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:06:54,896 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:13,219 - opendeep.optimization.optimizer - INFO - Train cost: 0.593288\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:13,227 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.18154761904761907}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:13,232 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:14,307 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.094166666666666662}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:15,561 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.10019607843137254}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:15,568 - opendeep.optimization.optimizer - INFO - time: 20.67354 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:15,569 - opendeep.optimization.optimizer - INFO - remaining time: 2.438551 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:15,573 - opendeep.optimization.optimizer - INFO - EPOCH 3\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:15,574 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:26,330 - opendeep.optimization.optimizer - INFO - Train cost: 0.466224\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:26,334 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.13992063492063492}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:26,336 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,140 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.083284313725490194}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,931 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.086519607843137267}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,934 - opendeep.optimization.optimizer - INFO - time: 12.36131 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,935 - opendeep.optimization.optimizer - INFO - remaining time: 1.903206 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,936 - opendeep.optimization.optimizer - INFO - EPOCH 4\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:27,938 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:39,923 - opendeep.optimization.optimizer - INFO - Train cost: 0.405620\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:39,932 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.12051587301587301}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:39,934 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:40,916 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.074558823529411788}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:41,756 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.077843137254901967}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:41,759 - opendeep.optimization.optimizer - INFO - time: 13.82309 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:41,760 - opendeep.optimization.optimizer - INFO - remaining time: 1.569066 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:41,762 - opendeep.optimization.optimizer - INFO - EPOCH 5\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:41,762 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:52,162 - opendeep.optimization.optimizer - INFO - Train cost: 0.362349\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:52,164 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.10815476190476192}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:52,165 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,089 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.0667156862745098}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,893 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.071078431372549031}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,896 - opendeep.optimization.optimizer - INFO - time: 12.13463 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,897 - opendeep.optimization.optimizer - INFO - remaining time: 1.248288 minutes\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,899 - opendeep.optimization.optimizer - INFO - EPOCH 6\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:07:53,899 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:04,682 - opendeep.optimization.optimizer - INFO - Train cost: 0.333982\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:04,684 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.09918650793650792}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:04,685 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:05,648 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.06147058823529411}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:06,376 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.066225490196078443}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:06,379 - opendeep.optimization.optimizer - INFO - time: 12.48080 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:06,382 - opendeep.optimization.optimizer - INFO - remaining time: 58.25208 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:06,384 - opendeep.optimization.optimizer - INFO - EPOCH 7\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:06,384 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:18,960 - opendeep.optimization.optimizer - INFO - Train cost: 0.306776\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:18,961 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.091071428571428581}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:18,962 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:20,011 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.05769607843137254}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:21,283 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.060882352941176464}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:21,295 - opendeep.optimization.optimizer - INFO - time: 14.91148 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:21,296 - opendeep.optimization.optimizer - INFO - remaining time: 43.83840 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:21,299 - opendeep.optimization.optimizer - INFO - EPOCH 8\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:21,300 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:33,125 - opendeep.optimization.optimizer - INFO - Train cost: 0.284762\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:33,130 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.083611111111111108}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:33,133 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:34,198 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.052647058823529422}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:35,940 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.058039215686274508}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:35,946 - opendeep.optimization.optimizer - INFO - time: 14.64675 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:35,948 - opendeep.optimization.optimizer - INFO - remaining time: 29.23409 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:35,952 - opendeep.optimization.optimizer - INFO - EPOCH 9\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:35,953 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:49,575 - opendeep.optimization.optimizer - INFO - Train cost: 0.270094\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:49,585 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.079444444444444456}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:49,586 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:50,902 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.049068627450980389}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:52,132 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.053970588235294117}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:52,146 - opendeep.optimization.optimizer - INFO - time: 16.19378 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:52,149 - opendeep.optimization.optimizer - INFO - remaining time: 14.79223 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:52,190 - opendeep.optimization.optimizer - INFO - EPOCH 10\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:08:52,242 - opendeep.optimization.optimizer - DEBUG - Turning on 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:03,705 - opendeep.optimization.optimizer - INFO - Train cost: 0.256928\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:03,711 - opendeep.optimization.optimizer - INFO - Train monitors: {'error/train': 0.075892857142857137}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:03,712 - opendeep.optimization.optimizer - DEBUG - Turning off 2 noise switches\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:04,536 - opendeep.optimization.optimizer - INFO - Valid monitors: {'error/valid': 0.046225490196078432}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,643 - opendeep.optimization.optimizer - INFO - Test monitors: {'error/test': 0.051960784313725486}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,648 - opendeep.optimization.optimizer - INFO - Stopping (reached max number of epochs)...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,650 - opendeep.optimization.optimizer - INFO - time: 13.46054 seconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,651 - opendeep.optimization.optimizer - INFO - remaining time: 0.0 milliseconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,653 - opendeep.optimization.optimizer - DEBUG - Restoring best model parameters...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,706 - opendeep.optimization.optimizer - DEBUG - Saving model parameters...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,736 - opendeep.models.model - DEBUG - Saving <class 'opendeep.models.container.Prototype'> parameters to /Users/mwarrick/Documents/Neural_Net_Newbies/outputs/prototype/trained_epoch_10.pkl\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,831 - opendeep.optimization.optimizer - INFO - ------------TRAIN TIME TOOK 2.447667 minutes---------\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:05,834 - opendeep.optimization.optimizer - INFO - ------------TOTAL <class 'opendeep.models.container.Prototype'> TRAIN TIME TOOK 2.447961 minutes---------\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "... evaluating model\n",
        "2015-04-22 15:09:06,477 - opendeep.models.container - INFO - Compiling f_run...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015-04-22 15:09:07,472 - opendeep.models.container - INFO - Compilation done! Took 995.8419 milliseconds\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy:  0.96\n"
       ]
      }
     ],
     "prompt_number": 31
    }
   ],
   "metadata": {}
  }
 ]
}