Automated test for notebooks (dmlc#17)

* Automated test for notebooks * Modification to catch kernel died issue
nirbd · Dec 28, 2016 · ae08aa6 · ae08aa6
1 parent af89c68
commit ae08aa6
Show file tree

Hide file tree

Showing 14 changed files with 3,491 additions and 2,203 deletions.
diff --git a/python/basic/data.ipynb b/python/basic/data.ipynb
@@ -189,7 +189,9 @@
     "        if self.cur_batch < self.num_batches:\n",
     "            self.cur_batch += 1\n",
     "            data = [mx.nd.array(g(d[1])) for d,g in zip(self._provide_data, self.data_gen)]\n",
+    "            assert len(data) > 0, \"Empty batch data.\"\n",
     "            label = [mx.nd.array(g(d[1])) for d,g in zip(self._provide_label, self.label_gen)]\n",
+    "            assert len(label) > 0, \"Empty batch label.\"\n",
     "            return SimpleBatch(data, label)\n",
     "        else:\n",
     "            raise StopIteration"
@@ -216,20 +218,21 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "INFO:root:Epoch[0] Train-accuracy=0.078125\n",
-      "INFO:root:Epoch[0] Time cost=0.307\n",
-      "INFO:root:Epoch[1] Train-accuracy=0.103125\n",
-      "INFO:root:Epoch[1] Time cost=0.291\n",
-      "INFO:root:Epoch[2] Train-accuracy=0.118750\n",
-      "INFO:root:Epoch[2] Time cost=0.419\n",
-      "INFO:root:Epoch[3] Train-accuracy=0.096875\n",
-      "INFO:root:Epoch[3] Time cost=0.419\n",
-      "INFO:root:Epoch[4] Train-accuracy=0.109375\n",
-      "INFO:root:Epoch[4] Time cost=0.415\n"
+      "INFO:root:Epoch[0] Train-accuracy=0.090625\n",
+      "INFO:root:Epoch[0] Time cost=0.011\n",
+      "INFO:root:Epoch[1] Train-accuracy=0.078125\n",
+      "INFO:root:Epoch[1] Time cost=0.014\n",
+      "INFO:root:Epoch[2] Train-accuracy=0.087500\n",
+      "INFO:root:Epoch[2] Time cost=0.014\n",
+      "INFO:root:Epoch[3] Train-accuracy=0.100000\n",
+      "INFO:root:Epoch[3] Time cost=0.014\n",
+      "INFO:root:Epoch[4] Train-accuracy=0.115625\n",
+      "INFO:root:Epoch[4] Time cost=0.014\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "import logging\n",
     "logging.basicConfig(level=logging.INFO)\n",
     "\n",
@@ -263,19 +266,20 @@
      "output_type": "stream",
      "text": [
       "INFO:root:Epoch[0] Train-accuracy=0.190625\n",
-      "INFO:root:Epoch[0] Time cost=0.060\n",
-      "INFO:root:Epoch[1] Train-accuracy=0.196875\n",
-      "INFO:root:Epoch[1] Time cost=0.039\n",
-      "INFO:root:Epoch[2] Train-accuracy=0.203125\n",
-      "INFO:root:Epoch[2] Time cost=0.036\n",
-      "INFO:root:Epoch[3] Train-accuracy=0.218750\n",
-      "INFO:root:Epoch[3] Time cost=0.029\n",
+      "INFO:root:Epoch[0] Time cost=0.009\n",
+      "INFO:root:Epoch[1] Train-accuracy=0.209375\n",
+      "INFO:root:Epoch[1] Time cost=0.009\n",
+      "INFO:root:Epoch[2] Train-accuracy=0.187500\n",
+      "INFO:root:Epoch[2] Time cost=0.011\n",
+      "INFO:root:Epoch[3] Train-accuracy=0.246875\n",
+      "INFO:root:Epoch[3] Time cost=0.009\n",
       "INFO:root:Epoch[4] Train-accuracy=0.175000\n",
-      "INFO:root:Epoch[4] Time cost=0.020\n"
+      "INFO:root:Epoch[4] Time cost=0.009\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "data = SimpleIter(['user', 'item'],\n",
     "                  [(n,), (n,)],\n",
     "                  [lambda s: np.random.randint(0, num_users, s),\n",
@@ -307,6 +311,39 @@
     "- [Data loading API](http://mxnet.io/packages/python/io.html)\n",
     "- [Design of efficient data format](http://mxnet.io/system/note_data_loading.html)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "train = mx.io.NDArrayIter(data=np.zeros((1200, 3, 224, 224), dtype='float32'), \n",
+    "    label={'annot': np.zeros((1200, 80), dtype='int8'),\n",
+    "    'label': np.zeros((1200, 1), dtype='int8')}, \n",
+    "    batch_size=10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "a\n"
+     ]
+    }
+   ],
+   "source": [
+    "print \"a\""
+   ]
   }
  ],
  "metadata": {

diff --git a/python/basic/mixed.ipynb b/python/basic/mixed.ipynb
@@ -319,7 +319,9 @@
     }
    ],
    "source": [
+    "# Output may vary\n",
     "learning_rate = 0.1\n",
+    "final_acc = 0\n",
     "for i in range(100):\n",
     "    x, y = toy_data.get(batch_size)\n",
     "    args['data'][:] = x\n",
@@ -330,7 +332,9 @@
     "        weight[:] -= learning_rate * (grad / batch_size)\n",
     "    if i % 10 == 0:\n",
     "        acc = (mx.nd.argmax_channel(ex.outputs[0]).asnumpy() == y).sum()\n",
-    "        print('iteration %d, accuracy %f' % (i, float(acc)/y.shape[0]))    "
+    "        final_acc = acc\n",
+    "        print('iteration %d, accuracy %f' % (i, float(acc)/y.shape[0]))\n",
+    "assert final_acc > 0.95, \"Low training accuracy.\""
    ]
   },
   {
@@ -393,7 +397,8 @@
     "        if 'bias' in name:\n",
     "            arr[:] = 0\n",
     "    # run 50 iterations\n",
-    "    learning_rate = 0.1    \n",
+    "    learning_rate = 0.1 \n",
+    "    acc = 0\n",
     "    for i in range(50):\n",
     "        # broadcast weight from dev 0 to all devices\n",
     "        for j in range(1, len(devs)):\n",
@@ -422,7 +427,8 @@
     "        if i % 10 == 0:\n",
     "            pred = np.concatenate([mx.nd.argmax_channel(ex.outputs[0]).asnumpy() for ex in exs])\n",
     "            acc = (pred == y).sum() / batch_size\n",
-    "            print('iteration %d, accuracy %f' % (i, acc))    "
+    "            print('iteration %d, accuracy %f' % (i, acc))\n",
+    "    return acc"
    ]
   },
   {
@@ -454,8 +460,10 @@
     }
    ],
    "source": [
+    "# Output may vary\n",
     "batch_size = 100\n",
-    "train(net, [batch_size, num_features], lambda : toy_data.get(batch_size), [mx.cpu(), mx.gpu()], [1, 5])"
+    "acc = train(net, [batch_size, num_features], lambda : toy_data.get(batch_size), [mx.cpu(), mx.gpu()], [1, 5])\n",
+    "assert acc > 0.95, \"Low training accuracy.\""
    ]
   },
   {
@@ -741,12 +749,14 @@
     }
    ],
    "source": [
+    "# Output may vary\n",
     "import time\n",
     "batch_size = 1024\n",
     "shape = [batch_size, 1, 28, 28]\n",
     "mnist.reset()\n",
     "tic = time.time()\n",
-    "train(lenet(), shape, lambda:mnist.get(batch_size), [mx.gpu(),], [1,])\n",
+    "acc = train(lenet(), shape, lambda:mnist.get(batch_size), [mx.gpu(),], [1,])\n",
+    "assert acc > 0.9, \"Low training accuracy.\"\n",
     "print('time for train lenent on cpu %f sec' % (time.time() - tic))"
    ]
   },
@@ -787,11 +797,13 @@
     }
    ],
    "source": [
+    "# Output may vary\n",
     "for ndev in (2, 4):\n",
     "    mnist.reset()\n",
     "    tic = time.time()\n",
-    "    train(lenet(), shape, lambda:mnist.get(batch_size), \n",
+    "    acc = train(lenet(), shape, lambda:mnist.get(batch_size), \n",
     "          [mx.gpu(i) for i in range(ndev)], [1]*ndev)\n",
+    "    assert acc > 0.9, \"Low training accuracy.\"\n",
     "    print('time for train lenent on %d GPU %f sec' % (\n",
     "            ndev, time.time() - tic))"
    ]

diff --git a/python/basic/module.ipynb b/python/basic/module.ipynb
@@ -62,14 +62,15 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true
+    "collapsed": false
    },
    "outputs": [],
    "source": [
     "mod = mx.mod.Module(symbol=net, \n",
     "                    context=mx.cpu(),\n",
     "                    data_names=['data'], \n",
-    "                    label_names=['softmax_label'])"
+    "                    label_names=['softmax_label'])\n",
+    "assert len(mod.data_names) == 1 and mod.data_names[0] == 'data', \"Module data bind failed.\""
    ]
   },
   {
@@ -92,25 +93,26 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "INFO:root:Epoch[0] Train-accuracy=0.131250\n",
-      "INFO:root:Epoch[0] Time cost=0.042\n",
-      "INFO:root:Epoch[0] Validation-accuracy=0.118750\n",
-      "INFO:root:Epoch[1] Train-accuracy=0.162500\n",
-      "INFO:root:Epoch[1] Time cost=0.035\n",
-      "INFO:root:Epoch[1] Validation-accuracy=0.093750\n",
-      "INFO:root:Epoch[2] Train-accuracy=0.131250\n",
-      "INFO:root:Epoch[2] Time cost=0.035\n",
-      "INFO:root:Epoch[2] Validation-accuracy=0.203125\n",
-      "INFO:root:Epoch[3] Train-accuracy=0.231250\n",
-      "INFO:root:Epoch[3] Time cost=0.039\n",
-      "INFO:root:Epoch[3] Validation-accuracy=0.146875\n",
-      "INFO:root:Epoch[4] Train-accuracy=0.200000\n",
-      "INFO:root:Epoch[4] Time cost=0.037\n",
-      "INFO:root:Epoch[4] Validation-accuracy=0.234375\n"
+      "INFO:root:Epoch[0] Train-accuracy=0.090625\n",
+      "INFO:root:Epoch[0] Time cost=0.019\n",
+      "INFO:root:Epoch[0] Validation-accuracy=0.187500\n",
+      "INFO:root:Epoch[1] Train-accuracy=0.159375\n",
+      "INFO:root:Epoch[1] Time cost=0.018\n",
+      "INFO:root:Epoch[1] Validation-accuracy=0.103125\n",
+      "INFO:root:Epoch[2] Train-accuracy=0.162500\n",
+      "INFO:root:Epoch[2] Time cost=0.022\n",
+      "INFO:root:Epoch[2] Validation-accuracy=0.200000\n",
+      "INFO:root:Epoch[3] Train-accuracy=0.181250\n",
+      "INFO:root:Epoch[3] Time cost=0.022\n",
+      "INFO:root:Epoch[3] Validation-accuracy=0.096875\n",
+      "INFO:root:Epoch[4] Train-accuracy=0.118750\n",
+      "INFO:root:Epoch[4] Time cost=0.022\n",
+      "INFO:root:Epoch[4] Validation-accuracy=0.193750\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "import logging\n",
     "logging.basicConfig(level=logging.INFO)\n",
     "\n",
@@ -171,20 +173,21 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "batch 0, accuracy 0.437500\n",
-      "batch 1, accuracy 0.406250\n",
-      "batch 2, accuracy 0.281250\n",
-      "batch 3, accuracy 0.437500\n",
-      "batch 4, accuracy 0.218750\n",
-      "batch 5, accuracy 0.375000\n",
-      "batch 6, accuracy 0.406250\n",
-      "batch 7, accuracy 0.343750\n",
-      "batch 8, accuracy 0.250000\n",
-      "batch 9, accuracy 0.468750\n"
+      "batch 0, accuracy 0.156250\n",
+      "batch 1, accuracy 0.312500\n",
+      "batch 2, accuracy 0.312500\n",
+      "batch 3, accuracy 0.187500\n",
+      "batch 4, accuracy 0.187500\n",
+      "batch 5, accuracy 0.125000\n",
+      "batch 6, accuracy 0.250000\n",
+      "batch 7, accuracy 0.187500\n",
+      "batch 8, accuracy 0.156250\n",
+      "batch 9, accuracy 0.187500\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "for preds, i_batch, batch in mod.iter_predict(data.get_iter(batch_size)):\n",
     "    pred_label = preds[0].asnumpy().argmax(axis=1)\n",
     "    label = batch.label[0].asnumpy().astype('int32')\n",
@@ -208,7 +211,7 @@
     {
      "data": {
       "text/plain": [
-       "[('mse', 27.360022735595702), ('accuracy', 0.28125)]"
+       "[('mse', 29.322532463073731), ('accuracy', 0.2125)]"
       ]
      },
      "execution_count": 6,
@@ -217,6 +220,7 @@
     }
    ],
    "source": [
+    "# Output may vary\n",
     "mod.score(data.get_iter(batch_size), ['mse', 'acc'])"
    ]
   },
@@ -240,25 +244,26 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "INFO:root:Epoch[0] Train-accuracy=0.121875\n",
-      "INFO:root:Epoch[0] Time cost=0.036\n",
+      "INFO:root:Epoch[0] Train-accuracy=0.196875\n",
+      "INFO:root:Epoch[0] Time cost=0.019\n",
       "INFO:root:Saved checkpoint to \"mx_mlp-0001.params\"\n",
-      "INFO:root:Epoch[1] Train-accuracy=0.112500\n",
-      "INFO:root:Epoch[1] Time cost=0.038\n",
+      "INFO:root:Epoch[1] Train-accuracy=0.159375\n",
+      "INFO:root:Epoch[1] Time cost=0.019\n",
       "INFO:root:Saved checkpoint to \"mx_mlp-0002.params\"\n",
-      "INFO:root:Epoch[2] Train-accuracy=0.156250\n",
-      "INFO:root:Epoch[2] Time cost=0.035\n",
+      "INFO:root:Epoch[2] Train-accuracy=0.290625\n",
+      "INFO:root:Epoch[2] Time cost=0.018\n",
       "INFO:root:Saved checkpoint to \"mx_mlp-0003.params\"\n",
       "INFO:root:Epoch[3] Train-accuracy=0.125000\n",
-      "INFO:root:Epoch[3] Time cost=0.036\n",
+      "INFO:root:Epoch[3] Time cost=0.019\n",
       "INFO:root:Saved checkpoint to \"mx_mlp-0004.params\"\n",
-      "INFO:root:Epoch[4] Train-accuracy=0.112500\n",
-      "INFO:root:Epoch[4] Time cost=0.036\n",
+      "INFO:root:Epoch[4] Train-accuracy=0.190625\n",
+      "INFO:root:Epoch[4] Time cost=0.018\n",
       "INFO:root:Saved checkpoint to \"mx_mlp-0005.params\"\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "# construct a callback function to save checkpoints\n",
     "model_prefix = 'mx_mlp'\n",
     "checkpoint = mx.callback.do_checkpoint(model_prefix)\n",
@@ -316,14 +321,15 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "INFO:root:Epoch[3] Train-accuracy=0.106250\n",
-      "INFO:root:Epoch[3] Time cost=0.037\n",
-      "INFO:root:Epoch[4] Train-accuracy=0.187500\n",
-      "INFO:root:Epoch[4] Time cost=0.037\n"
+      "INFO:root:Epoch[3] Train-accuracy=0.168750\n",
+      "INFO:root:Epoch[3] Time cost=0.019\n",
+      "INFO:root:Epoch[4] Train-accuracy=0.212500\n",
+      "INFO:root:Epoch[4] Time cost=0.019\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "mod = mx.mod.Module(symbol=sym)\n",
     "mod.fit(data.get_iter(batch_size),\n",
     "        num_epoch=5,\n",
@@ -364,11 +370,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "('accuracy', 0.421875)\n"
+      "('accuracy', 0.3875)\n"
      ]
     }
    ],
    "source": [
+    "# Output may vary\n",
     "# initial state\n",
     "mod = mx.mod.Module(symbol=net)\n",
     "\n",