03explore.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dictionary\n",
    "\n",
    "train_direction = 0 south, 1 north\n",
    "train_type = 0 Local, 1 Limited, 2 Bullet\n",
    "train_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/albarron/anaconda/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "# Import necessary libraries\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import re\n",
    "import random\n",
    "import operator\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "from func import *\n",
    "\n",
    "# inline plot\n",
    "%matplotlib inline\n",
    "#%%javascript\n",
    "#IPython.OutputArea.auto_scroll_threshold = 9999;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3199\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Unnamed: 0',\n",
       " 'created_at',\n",
       " 'favorite_count',\n",
       " 'favorited',\n",
       " 'hashtags',\n",
       " 'id',\n",
       " 'in_reply_to_screen_name',\n",
       " 'in_reply_to_status_id',\n",
       " 'in_reply_to_user_id',\n",
       " 'lang',\n",
       " 'media',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'retweeted',\n",
       " 'retweeted_status',\n",
       " 'source',\n",
       " 'text',\n",
       " 'truncated',\n",
       " 'urls',\n",
       " 'user',\n",
       " 'user_mentions']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%load 'data/raw-twt2016-01-26-14/21/09.csv'\n",
    "df = pd.read_csv(\"data/raw-twt2016-01-26-14-21-09.csv\",sep='\\t',error_bad_lines=False)\n",
    "# df.head(5)\n",
    "print len(df.index)\n",
    "list(df.columns.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Cleanin' the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   error  rush_evening  rush_morning  workday\n",
      "0      1             0             0        0\n",
      "1      0             1             0        0\n",
      "2      0             1             0        0\n",
      "3      0             1             0        0\n",
      "4      0             1             0        0\n",
      "error           3199\n",
      "rush_evening    3199\n",
      "rush_morning    3199\n",
      "workday         3199\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Unnamed: 0',\n",
       " 'created_at',\n",
       " 'favorite_count',\n",
       " 'favorited',\n",
       " 'hashtags',\n",
       " 'id',\n",
       " 'in_reply_to_screen_name',\n",
       " 'in_reply_to_status_id',\n",
       " 'in_reply_to_user_id',\n",
       " 'lang',\n",
       " 'media',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'retweeted',\n",
       " 'retweeted_status',\n",
       " 'source',\n",
       " 'text',\n",
       " 'truncated',\n",
       " 'urls',\n",
       " 'user',\n",
       " 'user_mentions',\n",
       " 'day_of_week',\n",
       " 'day_of_month',\n",
       " 'month',\n",
       " 'time_of_day']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fill in blank hashtags\n",
    "df = df.where((pd.notnull(df)), np.nan)\n",
    "df[\"hashtags\"].fillna('')\n",
    "\n",
    "# Add some date/time things\n",
    "df[\"created_at\"] = pd.to_datetime(df[\"created_at\"], errors='coerce')\n",
    "\n",
    "df[\"day_of_week\"] = df[\"created_at\"].apply(lambda x: x.weekday())\n",
    "df[\"day_of_month\"] = df[\"created_at\"].apply(lambda x: x.day)\n",
    "df[\"month\"] = df[\"created_at\"].apply(lambda x: x.month)\n",
    "df[\"time_of_day\"] = df[\"created_at\"].apply(lambda x: get_time_of_day(x))\n",
    "\n",
    "tod_Dummy = pd.get_dummies(df['time_of_day'])\n",
    "print(tod_Dummy.head(5))\n",
    "print tod_Dummy.count()\n",
    "# del tod_Dummy['shutdown']\n",
    "\n",
    "# df['in_reply_to_screen_name'].fillna(-1)\n",
    "# df['in_reply_to_status_id'].fillna(-1)\n",
    "# df['in_reply_to_user_id'].fillna(-1)\n",
    "# df['retweeted_status'].fillna(-1)\n",
    "# df['retweeted'].fillna(-1)\n",
    "df['retweet_count'].fillna(np.nan)\n",
    "df['favorite_count'].fillna(np.nan)\n",
    "df[\"hashtags\"].fillna(np.nan)\n",
    "df[\"hashtags\"] = df[\"hashtags\"].apply(lambda x: str(x)[1:-1])\n",
    "df.loc[df[\"hashtags\"]=='a',\"hashtags\"] = ''\n",
    "list(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3199"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Potentially remove, just cleaning for analysis sake\n",
    "del df['Unnamed: 0']\n",
    "del df['truncated']\n",
    "del df['user_mentions']\n",
    "del df['urls']\n",
    "del df['source']\n",
    "del df['lang']\n",
    "del df['place']\n",
    "del df['favorited']\n",
    "del df['media']\n",
    "del df['user']\n",
    "\n",
    "# More likely to remove\n",
    "del df['in_reply_to_status_id']\n",
    "del df['in_reply_to_user_id']\n",
    "del df['retweeted']\n",
    "del df['retweeted_status']\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x103c20910>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEACAYAAAC3adEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHW1JREFUeJzt3WuQVfWd7vHvg4ABYxMuNq2g3IyKuXmJODEG2jiSmAkt\nViLDmChKYpmgZS41U4IaIS9yHK2jHKNjEp2E0yYSg0THyzGKqFvMqRG8YIKCSJlDiyCdUYYgKpE2\nv/NiL9q9uO6GXnutbp5P1S7XXr0uv72W7Gf//+umiMDMzGybHnkXYGZmxeJgMDOzFAeDmZmlOBjM\nzCzFwWBmZikOBjMzS8k8GCR9T9ILkv4o6Q5JvSX1l7RA0kpJD0vqVzH9DEmrJK2QND7r+szMLE1Z\nXscg6TDg98AxEfGepN8ADwLHAm9GxHWSLgf6R8R0SccCdwAnAUOBhcBHwxdbmJnVTC26kg4ADpLU\nE+gDrAXOApqTvzcDE5PhJuDOiGiLiNXAKmBMDWo0M7NEpsEQEeuA64FXKQfCXyJiITA4IlqTadYD\n9cksQ4A1FYtYm4wzM7MayTQYJH2EcutgGHAY5ZbD14Dtu4bcVWRmVhA9M17+3wN/iogNAJLuAU4B\nWiUNjohWSQ3An5Pp1wKHV8w/NBmXIslBYma2FyJCe5om62MMrwJ/J+lDkgScDiwH7gMuSKaZAtyb\nDN8HTE7OXBoBHAks2dmCI6LbvmbOnJl7Df58/nz74+frzp8tovrf05m2GCJiiaT5wFJga/LfW4GD\ngXmSpgItwKRk+uWS5lEOj63AtOjIpzEzs32WdVcSEfFD4Ifbjd5AuZtpZ9NfA1yTdV1mZrZzvvK5\ngBobG/MuIVP+fF1bd/583fmzdUSmF7hlRZJ7mMzMOkgSUcXB58y7ksyseIYPH05LS0veZVhGhg0b\nxurVq/d6frcYzPZDyS/HvMuwjOxq/1bbYvAxBjMzS3EwmJlZioPBzMxSHAxmVngXXnghV199dS7r\nvuqqqzjkkEM47LDDcln/Nj/84Q8577zzarIuB4OZ0dAwHEmZvRoahuf9EffKmjVruOGGG3jppZdY\nt25d3uVQvrNQ9ny6qpnR2tpCljc5bm2tzRdaZ2tpaWHQoEEMHDgw71Jqyi0G6xRZ/+Lszr9KbUdL\nly7lxBNPpF+/fkyePJktW7YAsHHjRiZMmEB9fT0DBw5kwoQJ7b/k58+fz6c//enUcm644QbOPvvs\n3a5r06ZNnH/++dTX1zNixAh+9KMfAfDoo48yfvx41q1bR11dHVOnTt3lMi644AJmz54NwLp16+jR\nowc/+clPAHjllVdSwfLAAw9w/PHH079/f0499VSWLVvW/rfXX3+dr371q9TX1zNq1Chuuummna6v\nra2Nc889l3POOYe2trbdfr694WCwTvHBL878XuUarKvbunUrZ599NlOmTGHDhg2cc845/Pa3vwXK\nd1WeOnUqa9as4dVXX6Vv375ccsklADQ1NbF69WpWrlzZvqxf/epXTJkyZbfru/TSS3nrrbdYvXo1\npVKJ22+/nTlz5nD66afzu9/9jsMOO4xNmzbxi1/8YpfLGDduHKVSCYAnnniCUaNGsWjRIgAWLVrE\n2LFjgXLgfeMb3+C2225jw4YNXHzxxTQ1NbF161YiggkTJnD88cfz+uuv8+ijj3LjjTfyyCOPpNa1\nZcsWJk6cSJ8+fZg3bx49e2bQ8ZP3bWD38taxYcUCBETOL/9/Ua3tt1X2+6/6fbNo0aIYMmRIatwp\np5wSP/jBD3aYdunSpTFgwID299OmTYurrroqIiJeeOGFGDBgQLz33nu7XNf7778fvXv3jpdeeql9\n3M9+9rM47bTTIiKiVCrF4YcfvseaX3nllfY6vvWtb8Wtt97aPt+UKVNi9uzZERHx7W9/O66++urU\nvEcffXQsWrQoFi9eHMOGDUv97ZprrompU6dGRMSsWbOiqakpxo0bF9/97nd3W8+utncyfo/fsW4x\nmFmhrFu3jiFD0k/0HTZsGADvvvsuF198McOHD+cjH/kI48aNY+PGje1X+Z5//vnMnTsXKLcWJk2a\nRK9evXa5rjfeeIO2tjaOOOKI1LrWrt3h+WC7NXLkSA466CCWLl3Kk08+yZe//GUOO+wwXn75ZZ54\n4gnGjRsHlI9ZXH/99QwYMIABAwbQv39/XnvtNdatW0dLSwtr165N/e2aa67hz3/+c/t6nnrqKZYt\nW8bll1/eofo6ygefzaxQDj300B2+mF999VWOPPJIrr/+elatWsXTTz/NIYccwh/+8AdOOOEEIgJJ\nnHzyyfTu3Zsnn3ySuXPn8utf/3q36xo0aBC9evWipaWFY445Bih/eW8fTNUYN24c8+fPZ+vWrRx6\n6KGMHTuW5uZmNm7cyHHHHQfA4YcfzpVXXsmMGTN2mP+pp55i5MiRqa6w7X3hC1/gk5/8JJ///Ocp\nlUrU19d3uM5quMVgZoXymc98hp49e3LTTTfR1tbG3XffzZIl5Qc5vvXWW/Tp04e6ujo2bNjArFmz\ndpj/vPPO49JLL6V3796ccsopu11Xjx49mDRpEldeeSWbN2+mpaWF2bNn79X1AmPHjuXmm29uP57Q\n2NjIzTffzKmnntp+mulFF13ET3/60/bP8/bbb/Pggw/y9ttvM2bMGA4++GCuu+46tmzZwvvvv8+L\nL77IM888k1rPP//zP3Puuedy+umn8+abb3a4zmo4GMyMwYOHAcrsVV5+dXr16sXdd9/NnDlzGDhw\nIHfddRdf+cpXAPje977HO++8w6BBgzjllFP40pe+tMP85513Hi+88ELVX+4//vGP6du3LyNHjmTs\n2LF8/etf58ILL6y63m3GjRvH5s2b27uNTj31VN5999329wAnnngit912G5deeikDBgzgqKOOorm5\nGSiH1AMPPMDzzz/PiBEjqK+v56KLLmLTpk07rOuqq65i4sSJnHHGGWzcuLHDte6J765qnaL8iyjv\nfeI7hlarO99ddcuWLQwePJjnnnuOUaNG5V1OLnx3VTOzCrfccgsnnXTSfhsKnSHTYJB0lKSlkp5L\n/vsXSZdJ6i9pgaSVkh6W1K9inhmSVklaIWl8lvWZWfcyYsQIbrrpJq6//vrU+I9//OPU1dW1vw4+\n+GDq6ur2eHC60ty5c9vnq1zOJz7xic7+GLmrWVeSpB7Aa8DJwKXAmxFxnaTLgf4RMV3SscAdwEnA\nUGAh8NHt+43clVQ87krqWrpzV5J1ra6kvwdeiYg1wFlAczK+GZiYDDcBd0ZEW0SsBlYBY2pYo5nZ\nfq+WwfCPwNxkeHBEtAJExHpg28m4Q4A1FfOsTcaZmVmN1CQYJPWi3Bq4Kxm1fRvHbVozs4Ko1ZXP\nZwLPRsQbyftWSYMjolVSA7Dtmu+1wOEV8w1Nxu2g8sKWxsZGGhsbO7tms25r2LBhNbu3v9XetluI\nlEql9pv7dURNDj5L+jXwUEQ0J++vBTZExLW7OPh8MuUupEfwwecuwQefzYqv2oPPmQeDpL5ACzAy\nIt5Kxg0A5lFuHbQAkyJiY/K3GcA3gK3AdyJiwU6W6WAoGAeDWfEVJhiy4GAoHgeDWfEV8XRVMzPr\nAhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeD\nmZmlOBjMzCzFwWBmZikOBjMzS3Ew7IOGhuFIyvXV0DA8781gZt2MH9Szb3Xgh9MkVXhbmBWeH9Rj\nZmZ7xcFgZmYpmQeDpH6S7pK0QtKLkk6W1F/SAkkrJT0sqV/F9DMkrUqmH591fWZmllaLFsONwIMR\nMRr4FPASMB1YGBFHA48BMwAkHQtMAkYDZwK3qNx5bWZmNZJpMEiqAz4XEXMAIqItIv4CnAU0J5M1\nAxOT4SbgzmS61cAqYEyWNZqZWVrWLYYRwBuS5kh6TtKtkvoCgyOiFSAi1gP1yfRDgDUV869NxpmZ\nWY30rMHyTwAuiYhnJM2m3I20/TmFHT7HcNasWe3DjY2NNDY27n2VZmbdUKlUolQqdXi+TK9jkDQY\n+M+IGJm8P5VyMIwCGiOiVVID8HhEjJY0HYiIuDaZ/iFgZkQs3m65vo7hgyoKce6+t4VZ8RXiOoak\nu2iNpKOSUacDLwL3ARck46YA9ybD9wGTJfWWNAI4EliSZY1mZpaWdVcSwGXAHZJ6AX8CLgQOAOZJ\nmgq0UD4TiYhYLmkesBzYCkwrRNPAzGw/4lti7FsduPskqcLbwqzwCtGVZGZmXY+DwczMUhwMZmaW\n4mAwM7MUB4OZmaU4GMzMLMXBYNbJ/MhX6+p8HcO+1YHP3U+q8Lb4oApvCysoX8dgZmZ7xcFgZmYp\nDgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLyTwYJK2W9AdJ\nSyUtScb1l7RA0kpJD0vqVzH9DEmrJK2QND7r+szMLK0WLYa/AY0RcXxEjEnGTQcWRsTRwGPADABJ\nxwKTgNHAmcAtKt+RzMzMaqQWwaCdrOcsoDkZbgYmJsNNwJ0R0RYRq4FVwBjMzKxmahEMATwi6WlJ\n30zGDY6IVoCIWA/UJ+OHAGsq5l2bjDMzsxrpWYN1fDYiXpd0CLBA0kp2vFl9h28cP2vWrPbhxsZG\nGhsb96VGM7Nup1QqUSqVOjxfTR/UI2kmsBn4JuXjDq2SGoDHI2K0pOlARMS1yfQPATMjYvF2y/GD\nej6oohAPZPG2qKjC28IKqhAP6pHUV9KHk+GDgPHAMuA+4IJksinAvcnwfcBkSb0ljQCOBJZkWaOZ\nmaVl3ZU0GLhHUiTruiMiFkh6BpgnaSrQQvlMJCJiuaR5wHJgKzCtEE0DM7P9SFVdSZI+ERHLalBP\nVdyVlKqiEF0G3hYVVXhbWEF1dlfSLZKWSJpWeTGamZl1P1UFQ0R8DvgacDjwrKS5ks7ItDIzM8tF\nh85KknQA5YvRfgxsonzx2hURcXc25e2yDnclfVBFIboMvC0qqvC2sILq1K4kSZ+UNBtYAXwemBAR\no5Ph2ftUqZmZFUq1B5+fAP4dmB8R7273t/Mi4pcZ1beretxi+KCKQvwy9LaoqMLbwgqq2hZDtcHw\nYeDdiHg/ed8D+FBEvLPPle4FB0OqikJ8AXhbVFThbWEF1dlnJS0E+lS875uMMzOzbqbaYPhQRGze\n9iYZ7ptNSWZmlqdqg+FtSSdseyPpRODd3UxvZrk6EEm5vxoahue9IWwvVHuM4STgTmAd5VNUG4B/\njIhnsy1vl/X4GMMHVRSiL9nboqKKgmyL/GuAouwTK+vUg8/JAnsBRydvV0bE1n2ob584GFJVFOIf\nnrdFRRUF2Rb51wBF2SdWlkUwnAIMp+LGexFx+94WuC8cDKkqCvEPz9uiooqCbIv8a4Ci7BMrqzYY\nqrq7qqRfAqOA54H3k9EB5BIMZmaWnWpvu/1p4NhC/Ew3M7NMVXtW0guUDzibmVk3V22LYRCwXNIS\n4K/bRkZEUyZVmZlZbqoNhllZFmFmZsXRkbOShgEfjYiFkvoCB0TEW5lWt+taCnG4oyhnn3hbtFfh\nbfFBFQWoAYqyT6yss2+7fREwH/hZMmoI8B97X56ZmRVVtV1JlwBjgMUAEbFKUn21K0nuxvoM8FpE\nNEnqD/wGGAasBiZFxF+SaWcAU4E24DsRsWBnyzzooIHVrj4TvXpVu+nMzLqWar/d/hoR75WbyCCp\nJx1rp34HWA7UJe+nAwsj4jpJlwMzgOmSjgUmAaOBocBCSR/dWb/RO++83IHVd76DD/4a8HCuNZiZ\nZaHaYHhC0hVAn+RZz9OA+6uZUdJQ4EvAj4DvJ6PPAsYlw81AiXJYNAF3RkQbsFrSKipaKmn5thik\n3rmu38wsK9VexzAd+C9gGXAx8CBwVZXzzgb+hXQLY3BEtAJExHpgW7fUEGBNxXRrk3FmZlYjVbUY\nIuJvwG3Jq2qS/gFojYjnJTXubhUdWW7ZrIrhxuRlZmbblEolSqVSh+er9rbb/4+dfHlHxMg9zPc/\ngK9TPpDcBzgYuIfyLTYaI6JVUgPweESMljS9vNi4Npn/IWBmRCzebrmR96l4dXVNbNp0P3nXUZTT\nAYtyiqa3RXsVBagBirJPrKyzH+35aeCk5PU54MfAr/Y0U0RcERFHJAEyGXgsIs6jfHzigmSyKcC9\nyfB9wGRJvSWNAI4EllRZo5mZdYJqu5Le3G7U/5L0LHD1Xq73X4F5kqYCLZTPRCIilkuaR/kMpq3A\ntEJcyWZmth+ptivphIq3PSi3IL4dEZ/KqrA91OOupHbFaKoXpfvE26K9igLUAEXZJ1bWqc9jAK6v\nGG4juShtL+oyM7OCq7Yr6bSsCzEzs2Ko9glu39/d3yPihs4px8zM8taRJ7idRPmsIYAJlM8WWpVF\nUWZmlp9qg2EocMK222xLmgX8n4j4elaFmZlZPqq9jmEw8F7F+/eScWZm1s1U22K4HVgi6Z7k/UTK\nN78zM7Nuptqzkn4k6XeUr3oGuDAilmZXlpmZ5aXariSAvsCmiLgReC25ZYWZmXUz1T7acyaw7YE6\nAL2o4l5JZmbW9VTbYjib8kN03gaIiHWU75RqZmbdTLXB8F5yM7sAkHRQdiWZmVmeqg2GeZJ+BnxE\n0kXAQjr40B4zM+saqj0r6X8mz3reBBwNXB0Rj2RamZmZ5WKPwSDpAGBhciM9h4GZWTe3x66kiHgf\n+JukfjWox8zMclbtlc+bgWWSHiE5MwkgIi7LpCozs07S0DCc1taWvMtg8OBhrF+/Ou8yqlJtMNyd\nvMzMupRyKOT/FLnW1j0+OK0wdhsMko6IiFcjwvdFMjPbT+zpGMN/bBuQ9NuOLlzSgZIWS1oqaVly\nBTWS+ktaIGmlpIcrj19ImiFplaQVksZ3dJ1mZrZv9hQMlW2fkR1deET8FTgtIo4HjgPOlDQGmE75\nTKejgcdIbrUh6VjKz5IeDZwJ3KLyk9XNzKxG9hQMsYvhqkXEO8nggZS7rgI4iw9u291M+TbeUL7t\nxp0R0RYRqyk/IW7M3qzXzMz2zp4OPn9K0ibKLYc+yTDJ+4iIuj2tQFIP4FlgFPBvEfG0pMER0Up5\nIesl1SeTDwH+s2L2tck4MzOrkd0GQ0QcsK8riIi/AcdLqgPukfQxdmx97EVrZFbFcGPyMjOzbUql\nEqVSqcPzqXxvvNqQ9APgHeCbQGNEtEpqAB6PiNGSplNuiVybTP8QMDMiFm+3nMj79LO6uiY2bbqf\nvOsAUct9uMsqJLwtkioKsi3yrwGKsE+KsT+gKNsiIvZ43LYjD+rZmyIGbTvjSFIf4AxgBXAfcEEy\n2RTg3mT4PmCypN7Jg4COBJZkWaOZmaVVe4Hb3joUaE6OM/QAfhMRD0p6ivIdW6cCLZTPRCIilkua\nBywHtgLTIu+INTPbz9S0K6mzuCupUv7NUyhKc93boqKKAtQARdgnxdgfUJRtkXtXkpmZdT0OBjMz\nS3EwmJlZioPBzMxSHAxmZpbiYDAzsxQHg5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhwM\nZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVlKpsEgaaikxyS9KGmZpMuS8f0lLZC0\nUtLDkvpVzDND0ipJKySNz7I+MzPbUdYthjbg+xHxMeAzwCWSjgGmAwsj4mjgMWAGgKRjgUnAaOBM\n4BaVn+RtZmY1kmkwRMT6iHg+Gd4MrACGAmcBzclkzcDEZLgJuDMi2iJiNbAKGJNljWZmllazYwyS\nhgPHAU8BgyOiFcrhAdQnkw0B1lTMtjYZZ2ZmNdKzFiuR9GFgPvCdiNgsKbabZPv3VZhVMdyYvMzM\nbJtSqUSpVOrwfIrYi+/kjqxA6gk8APwuIm5Mxq0AGiOiVVID8HhEjJY0HYiIuDaZ7iFgZkQs3m6Z\nsVdZ0onq6prYtOl+8q4DRNb7sKoqJLwtkioKsi3yrwGKsE+KsT+gKNsiIvZ43LYWXUm/AJZvC4XE\nfcAFyfAU4N6K8ZMl9ZY0AjgSWFKDGs3MLJFpV5KkzwJfA5ZJWko5tq8ArgXmSZoKtFA+E4mIWC5p\nHrAc2ApMi7wj1sxsP5N5V1IW3JVUKf/mKRSlue5tUVFFAWqAIuyTYuwPKMq2KEpXkpmZdSEOBjMz\nS3EwmJlZioPBzMxSHAxmZpbiYDAzsxQHg5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhwM\nZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLyTQYJP1cUqukP1aM6y9pgaSVkh6W1K/ibzMk\nrZK0QtL4LGszM7Ody7rFMAf4wnbjpgMLI+Jo4DFgBoCkY4FJwGjgTOAWlR/WamZmNZRpMETE74H/\n3m70WUBzMtwMTEyGm4A7I6ItIlYDq4AxWdZnZmY7yuMYQ31EtAJExHqgPhk/BFhTMd3aZJyZmdVQ\nEQ4+R94FmJnZB3rmsM5WSYMjolVSA/DnZPxa4PCK6YYm43ZhVsVwY/IyM7NtSqUSpVKpw/MpItsf\n7JKGA/dHxCeS99cCGyLiWkmXA/0jYnpy8PkO4GTKXUiPAB+NnRQoKfJuaNTVNbFp0/3kXQeIrPdh\nVVVIeFskVRRkW+RfAxRhnxRjf0BRtkVE7PGknkxbDJLmUv4pP1DSq8BM4F+BuyRNBVoon4lERCyX\nNA9YDmwFpu0sFMzMLFuZtxiy4BZDpfx/hUBRfpV5W1RUUYAaoAj7pBj7A4qyLappMRTh4LOZmRWI\ng8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCwljyufzcz2QwfSVW4Y7WAwM6uJv5L/9RTVBZO7kszM\nLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCzFwWBmZikOBjMzSylk\nMEj6oqSXJL0s6fK86zEz258ULhgk9QBuBr4AfAz4J0nH5FtVrZXyLiBjpbwLsH1SyruADJXyLqAQ\nChcMwBhgVUS0RMRW4E7grJxrqrFS3gVkrJR3AbZPSnkXkKFS3gUUQhGDYQiwpuL9a8k4MzOrgS77\nPIa6ugm5rv+995bkun4zs6woIu8HR6RJ+jtgVkR8MXk/HYiIuLZimmIVbWbWRUTEHp/WU8RgOABY\nCZwOvA4sAf4pIlbkWpiZ2X6icF1JEfG+pEuBBZSPgfzcoWBmVjuFazGYmVm+inhW0m5154vfJP1c\nUqukP+ZdS2eTNFTSY5JelLRM0mV519SZJB0oabGkpcnnm5l3TVmQ1EPSc5Luy7uWziZptaQ/JPuw\n251dIqmfpLskrUj+HZ68y2m7UoshufjtZcrHH9YBTwOTI+KlXAvrJJJOBTYDt0fEJ/OupzNJagAa\nIuJ5SR8GngXO6i77DkBS34h4JzlO9n+ByyKiW33BSPoecCJQFxFNedfTmST9CTgxIv4771qyIOl/\nA09ExBxJPYG+EbFpZ9N2tRZDt774LSJ+D3TL/ykjYn1EPJ8MbwZW0M2uT4mId5LBAykfv+s6v7qq\nIGko8CXg3/OuJSOi630nVkVSHfC5iJgDEBFtuwoF6HobwRe/dQOShgPHAYvzraRzJd0sS4H1wCMR\n8XTeNXWy2cC/0M0Cr0IAj0h6WtJFeRfTyUYAb0iak3QF3iqpz64m7mrBYF1c0o00H/hO0nLoNiLi\nbxFxPDAUOFnSsXnX1Fkk/QPQmrT6lLy6m89GxAmUW0WXJF273UVP4ATg35LP+A4wfVcTd7VgWAsc\nUfF+aDLOuoCkX3M+8MuIuDfverKSNNEfB76Ydy2d6LNAU9IP/2vgNEm351xTp4qI15P//hdwD+Wu\n6+7iNWBNRDyTvJ9POSh2qqsFw9PAkZKGSeoNTAa629kR3fXXGMAvgOURcWPehXQ2SYMk9UuG+wBn\nAN3mwHpEXBERR0TESMr/7h6LiPPzrquzSOqbtGaRdBAwHngh36o6T0S0AmskHZWMOh1YvqvpC3eB\n2+5094vfJM0FGoGBkl4FZm47WNTVSfos8DVgWdIPH8AVEfFQvpV1mkOB5uTMuR7AbyLiwZxrsuoN\nBu5JbrfTE7gjIhbkXFNnuwy4Q1Iv4E/AhbuasEudrmpmZtnral1JZmaWMQeDmZmlOBjMzCzFwWBm\nZikOBjMzS3EwmJlZioPBzMxSHAxmZpby/wERyCDI0/jDfAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x103d91290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.plot(x='created_at', y='day_of_week', kind='hist')\n",
    "# fdf = df[[\"created_at\",\"id\",\"text\",\"hashtags\"]]\n",
    "# str(fdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start getting some more detailed data from the trips as well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/albarron/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:10: FutureWarning: convert_objects is deprecated.  Use the data-type specific converters pd.to_datetime, pd.to_timedelta and pd.to_numeric.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3199"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['favorite_count'] = df['favorite_count'].astype(np.int64)\n",
    "# df['retweet_count'] = df['retweet_count'].astype(np.int64)\n",
    "# df['text'] = df['text'].astype(str)\n",
    "# df['id'] = df['id'].astype(np.int64)\n",
    "# df['day_of_week'] = df['day_of_week'].astype(np.int64)\n",
    "# df['day_of_month'] = df['day_of_month'].astype(np.int64)\n",
    "# df['month'] = df['month'].astype(np.int64)\n",
    "# df['time_of_day'] = df['time_of_day'].astype(np.int64)\n",
    "df.loc[df[\"hashtags\"]=='on',\"hashtags\"] = np.nan\n",
    "df.convert_objects(convert_numeric=True)\n",
    "df.dtypes\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>topic_train</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-26 20:32:15</td>\n",
       "      <td>6</td>\n",
       "      <td>SanFrancisco</td>\n",
       "      <td>692082643022680064</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>NOTICE: Ped &amp;amp; Bike detours in place for Ma...</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>error</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-26 19:41:32</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>692069881559134208</td>\n",
       "      <td>therealwall</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@therealwall After the end of the concert we w...</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>rush_evening</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-26 19:28:52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SB50</td>\n",
       "      <td>692066695838498816</td>\n",
       "      <td>AemalTheAFGHAN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@AemalTheAFGHAN @BKDenverSports We're glad to ...</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>rush_evening</td>\n",
       "      <td>'50'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-01-26 18:12:35</td>\n",
       "      <td>1</td>\n",
       "      <td>Sorry, Headphones</td>\n",
       "      <td>692047497238175744</td>\n",
       "      <td>4c4d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@4c4d Oh man. We love that, too. Our favorite ...</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>rush_evening</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-01-26 17:53:20</td>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>692042650933862401</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18</td>\n",
       "      <td>Pssst, hey, regular Caltrain riders: expect a ...</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>rush_evening</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           created_at  favorite_count           hashtags                  id  \\\n",
       "0 2016-01-26 20:32:15               6       SanFrancisco  692082643022680064   \n",
       "1 2016-01-26 19:41:32             NaN                     692069881559134208   \n",
       "2 2016-01-26 19:28:52             NaN               SB50  692066695838498816   \n",
       "3 2016-01-26 18:12:35               1  Sorry, Headphones  692047497238175744   \n",
       "4 2016-01-26 17:53:20               7                     692042650933862401   \n",
       "\n",
       "  in_reply_to_screen_name  retweet_count  \\\n",
       "0                     NaN              7   \n",
       "1             therealwall            NaN   \n",
       "2          AemalTheAFGHAN            NaN   \n",
       "3                    4c4d            NaN   \n",
       "4                     NaN             18   \n",
       "\n",
       "                                                text  day_of_week  \\\n",
       "0  NOTICE: Ped &amp; Bike detours in place for Ma...            1   \n",
       "1  @therealwall After the end of the concert we w...            1   \n",
       "2  @AemalTheAFGHAN @BKDenverSports We're glad to ...            1   \n",
       "3  @4c4d Oh man. We love that, too. Our favorite ...            1   \n",
       "4  Pssst, hey, regular Caltrain riders: expect a ...            1   \n",
       "\n",
       "   day_of_month  month   time_of_day topic_train  \n",
       "0            26      1         error              \n",
       "1            26      1  rush_evening              \n",
       "2            26      1  rush_evening        '50'  \n",
       "3            26      1  rush_evening              \n",
       "4            26      1  rush_evening              "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pull out potential trains from both hashtags and text\n",
    "df[\"topic_train\"] = df[\"text\"].apply(lambda x: check_train_id(x))\n",
    "df[\"topic_train\"] = df[\"topic_train\"].apply(lambda x: str(x)[1:-1])\n",
    "df[\"topic_train\"].fillna(np.nan)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3199"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pd.pivot_table(\n",
    "#   df,values='values',\n",
    "#   index=['month'],\n",
    "#   columns=['day_of_week'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, a word about the below code.\n",
    "In the accompanying `func.py` there is a function called parse_train that returns a `pandas.Series` object. For some reason, when it's returned from a map or apply, it seems to get cast as a string. When applied to a list or a dataframe, this string gets turned into a single field in the row, OR divided into several rows, throwing the count off.\n",
    "\n",
    "To get around this, I return the results of the parse_train function and then CAST it back to a series. This adds a weird 0 index, which I delete. I then fill in the plethora of NaNs and recombine it with the primary dataframe.\n",
    "\n",
    "For context, previous iterations included\n",
    "`df['topic_train'].apply(lambda x:parse_train(x))`\n",
    "which would return a pd.Series object with `str` versions of the returned pd.Series from `parse_train`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ret = []\n",
    "\n",
    "def parse_train(t):\n",
    "# x should be a list with train codes eg 123\n",
    "# {\"id\": \"123\", \"type:\" \"bullet\", direction: \"south\"}\n",
    "\n",
    "    try:\n",
    "        s = t['topic_train'].split(',')\n",
    "    except:\n",
    "        return t['topic_train']\n",
    "    if s[0] == '':\n",
    "#         print \"\"\n",
    "        return np.nan\n",
    "    for x in s:\n",
    "#         print \"Iter\",x[1:-1]\n",
    "        q = {}\n",
    "        # Check train id\n",
    "#         x = parse_train_id(x)\n",
    "        x = str(x)\n",
    "        x = re.sub('[^0-9]','', x)\n",
    "        if len(x)<3: continue\n",
    "\n",
    "        # 1 = north, 0 = south\n",
    "        q[\"t_northbound\"] = 1 if int(x[2]) in [1,3,5,7,9] else 0\n",
    "        q['t_limited'] = 0\n",
    "        q['t_bullet'] = 0\n",
    "        \n",
    "        if x[0] == '1':\n",
    "            q['t_limited'] = 0\n",
    "        elif x[0] == '2':\n",
    "            q[\"t_limited\"] = 1 # limited\n",
    "        elif x[0] == '3':\n",
    "            q[\"t_bullet\"] = 1 # bullet\n",
    "        else:\n",
    "            q['t_limited'] = 0\n",
    "\n",
    "        ret.append({'tweet_id': t['id'],\n",
    "                    'timestamp': t['created_at'], \n",
    "                    'train_id': int(x),\n",
    "                    't_northbound':q[\"t_northbound\"], \n",
    "                    't_limited': q[\"t_limited\"],\n",
    "                    't_bullet': q['t_bullet']})\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "red return: 3199\n",
      "ret return, 528\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-25 23:42:14</td>\n",
       "      <td>155</td>\n",
       "      <td>691768068385718275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-22 22:48:57</td>\n",
       "      <td>151</td>\n",
       "      <td>690667494906814464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-20 22:22:55</td>\n",
       "      <td>151</td>\n",
       "      <td>689936168893329408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-01-19 18:08:25</td>\n",
       "      <td>138</td>\n",
       "      <td>689509733640732672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-01-19 16:32:04</td>\n",
       "      <td>216</td>\n",
       "      <td>689485484838416388</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   t_bullet  t_limited  t_northbound           timestamp  train_id  \\\n",
       "0         0          0             1 2016-01-25 23:42:14       155   \n",
       "1         0          0             1 2016-01-22 22:48:57       151   \n",
       "2         0          0             1 2016-01-20 22:22:55       151   \n",
       "3         0          0             0 2016-01-19 18:08:25       138   \n",
       "4         0          1             0 2016-01-19 16:32:04       216   \n",
       "\n",
       "             tweet_id  \n",
       "0  691768068385718275  \n",
       "1  690667494906814464  \n",
       "2  689936168893329408  \n",
       "3  689509733640732672  \n",
       "4  689485484838416388  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's then filter those train topics into details\n",
    "# Btw this is jank as fuck.\n",
    "\n",
    "# red = df[['id','created_at','topic_train']]\n",
    "red = df.apply(lambda x:parse_train(x),axis=1)\n",
    "print \"red return:\",len(red)\n",
    "print \"ret return,\",len(ret)\n",
    "#red\n",
    "tf = pd.DataFrame(ret)\n",
    "tf.head(5)\n",
    "\n",
    "#events = pd.DataFrame([pd.Series(x) for x in red.apply(parse_train)])\n",
    "#events\n",
    "#del new.iloc[0]\n",
    "#new.fillna('')\n",
    "#df.combine_first(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              topic_train                                               text\n",
      "2                    '50'  @AemalTheAFGHAN @BKDenverSports We're glad to ...\n",
      "8                    '50'  @iamsridhar we are adding extra trains &amp; c...\n",
      "16                  '155'                  #NB155 is 22 mins late. #Caltrain\n",
      "21                   '50'  RT @smctd: #Caltrain Devs: NEW Super Bowl sche...\n",
      "22               '50', ''  Headed to #SantaClara for #SB50? Visit our SB ...\n",
      "29                   '50'  RT @Broncos: We #BeatThePatriots!\\n\\nSee you i...\n",
      "30                   '50'  We look forward to getting #CAR @Panthers, #DE...\n",
      "32                  '151'        #NB 151 is running 10 mins behind #Caltrain\n",
      "33                     ''  Trains 236-254 will board on the NB platform t...\n",
      "44                  '151'     #NB151 is 10 mins down at San Mateo. #Caltrain\n",
      "51                  '138'  #SB138 will board on the northbound platform a...\n",
      "55    '216', '225', '329'  #SB216 delayed 13 minutes at SAT\\n#NB225 delay...\n",
      "58                  '323'  #NB323 reported car on tracks at Charleston Av...\n",
      "61                   '50'  RT @chMtnViewPD: Stuff to know if you plan on ...\n",
      "72                  '371'  #NB371 is currently 11 mins late at Cal Ave. #...\n",
      "74                  '371'  #NB371 will pass &amp; be ahead of 269 at Redw...\n",
      "75                     ''                     @bassemabdouni the train is NB\n",
      "77                  '371'  #NB371 will depart San Jose on track 2. #Caltrain\n",
      "78                  '269'         #NB269 will depart ahead of 371. #Caltrain\n",
      "93           '217', '319'  #NB217 is 12 mi s late. #NB319 departed SJ ahe...\n",
      "94                  '156'       #SB156 is 10 mins late out of TAM. #Caltrain\n",
      "96                  '150'  #SB150 10 mins late @ SUN due to fare evader. ...\n",
      "102                  '50'  RT @scsb50: We are excited to be working with ...\n",
      "107             '323', ''  #NB323 – 10min late @ SBR. #DoorProblems. #Cal...\n",
      "122                 '366'  375/277 @ PA\\n279 @ MV 86 mins late\\nNB366 alm...\n",
      "127                 '366'                  NB366 currently at MIL. #Caltrain\n",
      "136          '273', '366'  SB273 turned southbound @ SUN.\\n375 &amp; 277 ...\n",
      "143                '', ''  @Awkward_Caiti Some, but problem is the SB tra...\n",
      "156                    ''  273 is turning southbound from Sunnyvale to SJ...\n",
      "157          '375', '277'  #NB375 7 #NB277 coupling at LAW due to crew sh...\n",
      "...                   ...                                                ...\n",
      "3008                '268'  #SB268 held at SJ for #370 PAX transfer due to...\n",
      "3009                '272'  #SB272 – 17 min late @ HIL Due to signal issue...\n",
      "3011                '370'                #SB370 – 11 min late HIL. #Caltrain\n",
      "3014                '151'        #NB151 was 10min late out of BEL. #Caltrain\n",
      "3016                '142'            #SB142 is 12 mins late at PA. #Caltrain\n",
      "3025                '376'  #SB376 - 21mins late @ PA. 376 ran around 274 ...\n",
      "3026                '376'  #SB376 will depart SF approx. 10-15min late du...\n",
      "3028                '156'  #SB156 will be about 15min late into CAP. #Cal...\n",
      "3029                '258'                   #SB258 - 12 mins late. #Caltrain\n",
      "3030                '258'         #SB258 departed SF 12 mins late. #Caltrain\n",
      "3031                '254'               #SB254-14 mins late @ RWC. #Caltrain\n",
      "3032                '155'   #NB155 – 10 min late out of San Mateo. #Caltrain\n",
      "3035                '152'  #SB152-10min late @ RWC. Mechanical delay. #Ca...\n",
      "3039                '146'       #SB146 – 13” mins late out of SMT. #Caltrain\n",
      "3054                '104'                  #SB104 is 23 mins late. #Caltrain\n",
      "3066                '221'    #NB221 is running about 15 mins late. #Caltrain\n",
      "3067                '221'  RT @chenmor3: @Caltrain_News NB221 just arrive...\n",
      "3068                '221'  #NB221 departed Gilroy 11 late. Still rolling ...\n",
      "3070                   ''  Congratulations #DubNation @warriors #BayArea ...\n",
      "3072                '380'  #SB380 reported a near miss with a trespasser ...\n",
      "3077                '269'  #NB269 departed PA 10 LATE. 269 is at bike cap...\n",
      "3082                '142'  #SB142 is 18 mins late out of Millbrae. #Caltrain\n",
      "3107         '440', '442'  Mechanical problem with SB 440. It will return...\n",
      "3108                '371'  RT @BikesOnCaltrain: #NB371 will run with a 5-...\n",
      "3109                '152'                  #SB152 is 11 mins late. #Caltrain\n",
      "3146                '135'  #NB135 is delayed 10 mins out of Redwood City....\n",
      "3149                '323'  RT @BikesOnCaltrain: #NB323 departed San Jose ...\n",
      "3164                '267'          #NB267 stopped at MPK with engine issues.\n",
      "3176                '233'   #NB233 is 10 mins delayed at Millbrae. #Caltrain\n",
      "3188                   ''  Final Twitter Update:\\n274-13 mins late @ LAW\\...\n",
      "\n",
      "[620 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print df.loc[df['topic_train'] != '',['topic_train','text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "528"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "528"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.merge(tf, left_on='id',right_on='tweet_id',how='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>id</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time_of_day</th>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">error</th>\n",
       "      <th>1</th>\n",
       "      <td>1.722222</td>\n",
       "      <td>6.869895e+17</td>\n",
       "      <td>3.280000</td>\n",
       "      <td>2.480000</td>\n",
       "      <td>12.360000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>242.720000</td>\n",
       "      <td>6.869895e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.437500</td>\n",
       "      <td>6.126169e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.567568</td>\n",
       "      <td>21.216216</td>\n",
       "      <td>0.324324</td>\n",
       "      <td>0.378378</td>\n",
       "      <td>0.297297</td>\n",
       "      <td>286.351351</td>\n",
       "      <td>6.126169e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.470588</td>\n",
       "      <td>6.223543e+17</td>\n",
       "      <td>4.953488</td>\n",
       "      <td>2.558140</td>\n",
       "      <td>18.162791</td>\n",
       "      <td>0.348837</td>\n",
       "      <td>0.441860</td>\n",
       "      <td>0.511628</td>\n",
       "      <td>281.720930</td>\n",
       "      <td>6.223543e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.480000</td>\n",
       "      <td>6.328767e+17</td>\n",
       "      <td>4.076923</td>\n",
       "      <td>2.826923</td>\n",
       "      <td>16.057692</td>\n",
       "      <td>0.269231</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.346154</td>\n",
       "      <td>263.076923</td>\n",
       "      <td>6.328767e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.777778</td>\n",
       "      <td>6.439176e+17</td>\n",
       "      <td>3.909091</td>\n",
       "      <td>2.314286</td>\n",
       "      <td>15.514286</td>\n",
       "      <td>0.128571</td>\n",
       "      <td>0.471429</td>\n",
       "      <td>0.671429</td>\n",
       "      <td>271.014286</td>\n",
       "      <td>6.439176e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.857143</td>\n",
       "      <td>6.527556e+17</td>\n",
       "      <td>4.622222</td>\n",
       "      <td>2.688889</td>\n",
       "      <td>9.866667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.455556</td>\n",
       "      <td>252.122222</td>\n",
       "      <td>6.527556e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.700000</td>\n",
       "      <td>6.652811e+17</td>\n",
       "      <td>3.562500</td>\n",
       "      <td>1.515152</td>\n",
       "      <td>13.272727</td>\n",
       "      <td>0.060606</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.484848</td>\n",
       "      <td>210.303030</td>\n",
       "      <td>6.652811e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.747415e+17</td>\n",
       "      <td>2.476190</td>\n",
       "      <td>3.681818</td>\n",
       "      <td>9.727273</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>288.454545</td>\n",
       "      <td>6.747415e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">rush_evening</th>\n",
       "      <th>1</th>\n",
       "      <td>1.875000</td>\n",
       "      <td>6.881407e+17</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>15.250000</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>228.875000</td>\n",
       "      <td>6.881407e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4.666667</td>\n",
       "      <td>6.125047e+17</td>\n",
       "      <td>1.818182</td>\n",
       "      <td>1.090909</td>\n",
       "      <td>20.545455</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>211.818182</td>\n",
       "      <td>6.125047e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.333333</td>\n",
       "      <td>6.225477e+17</td>\n",
       "      <td>2.100000</td>\n",
       "      <td>1.750000</td>\n",
       "      <td>18.250000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>198.050000</td>\n",
       "      <td>6.225477e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.400000</td>\n",
       "      <td>6.352651e+17</td>\n",
       "      <td>2.428571</td>\n",
       "      <td>2.285714</td>\n",
       "      <td>22.285714</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>146.785714</td>\n",
       "      <td>6.352651e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.447491e+17</td>\n",
       "      <td>1.640000</td>\n",
       "      <td>1.840000</td>\n",
       "      <td>17.520000</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.240000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>174.800000</td>\n",
       "      <td>6.447491e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.250000</td>\n",
       "      <td>6.529508e+17</td>\n",
       "      <td>2.210526</td>\n",
       "      <td>2.210526</td>\n",
       "      <td>10.157895</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>219.210526</td>\n",
       "      <td>6.529508e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.200000</td>\n",
       "      <td>6.663183e+17</td>\n",
       "      <td>4.100000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>174.700000</td>\n",
       "      <td>6.663183e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.800000</td>\n",
       "      <td>6.740357e+17</td>\n",
       "      <td>3.700000</td>\n",
       "      <td>1.700000</td>\n",
       "      <td>7.300000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>184.300000</td>\n",
       "      <td>6.740357e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rush_morning</th>\n",
       "      <th>10</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.520161e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>6.520161e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">workday</th>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.872953e+17</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>268.000000</td>\n",
       "      <td>6.872953e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.114410e+17</td>\n",
       "      <td>1.571429</td>\n",
       "      <td>1.714286</td>\n",
       "      <td>17.714286</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>232.714286</td>\n",
       "      <td>6.114410e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.333333</td>\n",
       "      <td>6.223477e+17</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>1.300000</td>\n",
       "      <td>17.800000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>247.900000</td>\n",
       "      <td>6.223477e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.342669e+17</td>\n",
       "      <td>2.333333</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>19.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>252.333333</td>\n",
       "      <td>6.342669e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.714286</td>\n",
       "      <td>6.434607e+17</td>\n",
       "      <td>5.933333</td>\n",
       "      <td>2.866667</td>\n",
       "      <td>14.066667</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>242.466667</td>\n",
       "      <td>6.434607e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.525711e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>9.166667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>373.666667</td>\n",
       "      <td>6.525711e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.724328e+17</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>220.333333</td>\n",
       "      <td>6.724328e+17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    favorite_count            id  retweet_count  day_of_week  \\\n",
       "time_of_day  month                                                             \n",
       "error        1            1.722222  6.869895e+17       3.280000     2.480000   \n",
       "             6            1.437500  6.126169e+17       2.000000     3.567568   \n",
       "             7            2.470588  6.223543e+17       4.953488     2.558140   \n",
       "             8            2.480000  6.328767e+17       4.076923     2.826923   \n",
       "             9            2.777778  6.439176e+17       3.909091     2.314286   \n",
       "             10           1.857143  6.527556e+17       4.622222     2.688889   \n",
       "             11           2.700000  6.652811e+17       3.562500     1.515152   \n",
       "             12           2.000000  6.747415e+17       2.476190     3.681818   \n",
       "rush_evening 1            1.875000  6.881407e+17       3.750000     0.750000   \n",
       "             6            4.666667  6.125047e+17       1.818182     1.090909   \n",
       "             7            1.333333  6.225477e+17       2.100000     1.750000   \n",
       "             8            1.400000  6.352651e+17       2.428571     2.285714   \n",
       "             9            2.000000  6.447491e+17       1.640000     1.840000   \n",
       "             10           1.250000  6.529508e+17       2.210526     2.210526   \n",
       "             11           1.200000  6.663183e+17       4.100000     0.700000   \n",
       "             12           1.800000  6.740357e+17       3.700000     1.700000   \n",
       "rush_morning 10           1.000000  6.520161e+17       1.000000     3.000000   \n",
       "workday      1            1.000000  6.872953e+17       6.000000     2.000000   \n",
       "             6            1.000000  6.114410e+17       1.571429     1.714286   \n",
       "             7            2.333333  6.223477e+17       5.800000     1.300000   \n",
       "             8                 NaN  6.342669e+17       2.333333     2.666667   \n",
       "             9            1.714286  6.434607e+17       5.933333     2.866667   \n",
       "             10           2.000000  6.525711e+17       4.000000     3.000000   \n",
       "             12                NaN  6.724328e+17       4.666667     3.000000   \n",
       "\n",
       "                    day_of_month  t_bullet  t_limited  t_northbound  \\\n",
       "time_of_day  month                                                    \n",
       "error        1         12.360000  0.320000   0.200000      0.480000   \n",
       "             6         21.216216  0.324324   0.378378      0.297297   \n",
       "             7         18.162791  0.348837   0.441860      0.511628   \n",
       "             8         16.057692  0.269231   0.461538      0.346154   \n",
       "             9         15.514286  0.128571   0.471429      0.671429   \n",
       "             10         9.866667  0.166667   0.466667      0.455556   \n",
       "             11        13.272727  0.060606   0.393939      0.484848   \n",
       "             12         9.727273  0.272727   0.318182      0.454545   \n",
       "rush_evening 1         15.250000  0.375000   0.250000      0.625000   \n",
       "             6         20.545455  0.181818   0.454545      0.545455   \n",
       "             7         18.250000  0.050000   0.550000      0.550000   \n",
       "             8         22.285714  0.000000   0.071429      0.785714   \n",
       "             9         17.520000  0.080000   0.240000      0.800000   \n",
       "             10        10.157895  0.105263   0.315789      0.684211   \n",
       "             11        16.000000  0.100000   0.200000      0.900000   \n",
       "             12         7.300000  0.100000   0.300000      0.500000   \n",
       "rush_morning 10         8.000000  0.000000   0.000000      1.000000   \n",
       "workday      1         13.000000  0.500000   0.500000      1.000000   \n",
       "             6         17.714286  0.285714   0.571429      0.714286   \n",
       "             7         17.800000  0.300000   0.700000      0.500000   \n",
       "             8         19.666667  0.333333   0.666667      0.333333   \n",
       "             9         14.066667  0.266667   0.666667      0.866667   \n",
       "             10         9.166667  0.500000   0.000000      0.666667   \n",
       "             12         3.000000  0.000000   1.000000      1.000000   \n",
       "\n",
       "                      train_id      tweet_id  \n",
       "time_of_day  month                            \n",
       "error        1      242.720000  6.869895e+17  \n",
       "             6      286.351351  6.126169e+17  \n",
       "             7      281.720930  6.223543e+17  \n",
       "             8      263.076923  6.328767e+17  \n",
       "             9      271.014286  6.439176e+17  \n",
       "             10     252.122222  6.527556e+17  \n",
       "             11     210.303030  6.652811e+17  \n",
       "             12     288.454545  6.747415e+17  \n",
       "rush_evening 1      228.875000  6.881407e+17  \n",
       "             6      211.818182  6.125047e+17  \n",
       "             7      198.050000  6.225477e+17  \n",
       "             8      146.785714  6.352651e+17  \n",
       "             9      174.800000  6.447491e+17  \n",
       "             10     219.210526  6.529508e+17  \n",
       "             11     174.700000  6.663183e+17  \n",
       "             12     184.300000  6.740357e+17  \n",
       "rush_morning 10     199.000000  6.520161e+17  \n",
       "workday      1      268.000000  6.872953e+17  \n",
       "             6      232.714286  6.114410e+17  \n",
       "             7      247.900000  6.223477e+17  \n",
       "             8      252.333333  6.342669e+17  \n",
       "             9      242.466667  6.434607e+17  \n",
       "             10     373.666667  6.525711e+17  \n",
       "             12     220.333333  6.724328e+17  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['time_of_day','month']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['created_at',\n",
       " 'favorite_count',\n",
       " 'hashtags',\n",
       " 'id',\n",
       " 'in_reply_to_screen_name',\n",
       " 'retweet_count',\n",
       " 'text',\n",
       " 'day_of_week',\n",
       " 'day_of_month',\n",
       " 'month',\n",
       " 'time_of_day',\n",
       " 'topic_train',\n",
       " 't_bullet',\n",
       " 't_limited',\n",
       " 't_northbound',\n",
       " 'timestamp',\n",
       " 'train_id',\n",
       " 'tweet_id']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x11009f490>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEACAYAAAC3adEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGwVJREFUeJzt3XuUFPXd5/H3B4EEfIRwEUZEZwDj7cka70Yl0GrUPCYg\nbiKPmiBKlmOironPJkdQI+SPrNFzlGN0TbyymIQkisbbmnhv0OwqGlFRvHDMMqDoZA0hiEoA890/\nuhi6hhnoGbqnamY+r3P62FXdPfXpX0l/+/f7VVUrIjAzM9uiV9YBzMwsX1wYzMwsxYXBzMxSXBjM\nzCzFhcHMzFJcGMzMLKWmhUHSbZKaJL3cymP/TdI/JQ0uWzdT0nJJr0k6qZbZzMysdbXuMcwFTm65\nUtJI4ESgsWzdAcBk4ADg34AbJanG+czMrIWaFoaIeBr4WysPzQF+0GLdqcBvImJzRKwAlgNH1jKf\nmZltq9PnGCRNBFZFxNIWD+0JrCpbfidZZ2Zmnah3Z25MUj/gUkrDSGZmlkOdWhiAMUAD8FIyfzAS\neEHSkZR6CHuXPXdksm4bknyBJzOzDoiIHc7ddsZQkpIbEfFKRNRFxOiIGAW8DRwSEX8B7gf+XVJf\nSaOAfYDFbf3RiOi2t1mzZmWewe/P768nvr/u/N4iKv8+XevDVecD/xvYV9JKSee2eEqwtWgsA+4E\nlgEPAedHe96JmZlVRU2HkiLirB08PrrF8pXAlbXMZGZm2+czn3OoUChkHaGm/P66tu78/rrze2sP\ndcXRGkkeZTIzaydJRAWTz519VJKZ5UBDQwONjY07fqJ1SfX19axYsaLDr3ePwawHSr45Zh3DaqSt\n/Vtpj8FzDFYVdXUNSMr0VlfXkHUzmHUL7jFYVZTOV8x6n/hbcKXcY+je3GMwM7OqcmEws9w799xz\nueKKKzLZ9uWXX87uu+/OiBEjMtn+Fj/60Y+YMmVKp2zLhcHMaj5H1FXnf1atWsW1117L66+/zurV\nq7OOQ2f9RI0PVzUzmpoaqeUcUVNT1/zNrcbGRoYOHcqQIUOyjtKp3GMws9xZsmQJhx12GAMHDuSM\nM85gw4YNAKxdu5YJEyYwbNgwhgwZwoQJE5q/yS9YsIDDDz889XeuvfZaTjvttO1ua926dZx99tkM\nGzaMUaNG8eMf/xiAxx9/nJNOOonVq1czYMAApk2b1ubfOOecc5gzZw4Aq1evplevXvzsZz8D4K23\n3koVlgcffJBDDjmEQYMGMXbsWJYu3frTNO+++y5f//rXGTZsGGPGjOH6669vdXubN2/mrLPO4vTT\nT2fz5s3bfX8d4cJgZrmyadMmTjvtNKZOncqaNWs4/fTTufvuu4HSVZWnTZvGqlWrWLlyJf379+eC\nCy4AYOLEiaxYsYI33nij+W/98pe/ZOrUqdvd3oUXXsgHH3zAihUrKBaL3HHHHcydO5cTTjiB3//+\n94wYMYJ169Zx++23t/k3xo8fT7FYBGDhwoWMGTOGRYsWAbBo0SLGjRsHlAret771LW655RbWrFnD\neeedx8SJE9m0aRMRwYQJEzjkkEN49913efzxx7nuuut49NFHU9vasGEDkyZNol+/ftx555307l2D\ngZ+sLwPbwUvHhuULEBAZ3/z/RaVatlXt91/l+2bRokWx5557ptYdc8wx8cMf/nCb5y5ZsiQGDx7c\nvHz++efH5ZdfHhERr7zySgwePDg2btzY5rY++eST6Nu3b7z++uvN62666aY47rjjIiKiWCzGXnvt\ntcPMb731VnOOb3/723HzzTc3v27q1KkxZ86ciIj4zne+E1dccUXqtfvtt18sWrQonn322aivr089\nduWVV8a0adMiImL27NkxceLEGD9+fHzve9/bbp622jtZv8PPWPcYzCxXVq9ezZ57pn/Vt76+HoCP\nP/6Y8847j4aGBj7zmc8wfvx41q5d23zM/tlnn838+fOBUm9h8uTJ9OnTp81tvf/++2zevJm99976\nG2H19fW8806rvxHWptGjR7PrrruyZMkSnnrqKb761a8yYsQI3nzzTRYuXMj48eOB0pzFNddcw+DB\ngxk8eDCDBg3i7bffZvXq1TQ2NvLOO++kHrvyyiv5y1/+0rydZ555hqVLl3LJJZe0K197efLZzHJl\njz322OaDeeXKleyzzz5cc801LF++nOeee47dd9+dl156iUMPPZSIQBJHHXUUffv25amnnmL+/Pn8\n+te/3u62hg4dSp8+fWhsbGT//fcHSh/eLQtTJcaPH8+CBQvYtGkTe+yxB+PGjWPevHmsXbuWgw8+\nGIC99tqLyy67jJkzZ27z+meeeYbRo0enhsJaOvnkkznooIM4/vjjKRaLDBs2rN05K+Eeg5nlytFH\nH03v3r25/vrr2bx5M/fccw+LF5d+zPGDDz6gX79+DBgwgDVr1jB79uxtXj9lyhQuvPBC+vbtyzHH\nHLPdbfXq1YvJkydz2WWXsX79ehobG5kzZ06HzhcYN24cN9xwQ/N8QqFQ4IYbbmDs2LHNh5lOnz6d\nn//8583v58MPP+Shhx7iww8/5Mgjj2S33Xbj6quvZsOGDXzyySe8+uqrPP/886ntfP/73+ess87i\nhBNO4K9//Wu7c1bChcHMGD68nq2/wlv9W+nvV6ZPnz7cc889zJ07lyFDhnDXXXfxta99DYCLL76Y\njz76iKFDh3LMMcdwyimnbPP6KVOm8Morr1T84f7Tn/6U/v37M3r0aMaNG8c3v/lNzj235Y9N7tj4\n8eNZv35987DR2LFj+fjjj5uXAQ477DBuueUWLrzwQgYPHsy+++7LvHnzgFKRevDBB3nxxRcZNWoU\nw4YNY/r06axbt26bbV1++eVMmjSJE088kbVr17Y76474WklWFb5WUtfSna+VtGHDBoYPH84LL7zA\nmDFjso6TCV8rycyszI033sgRRxzRY4tCNXjy2cy6jVGjRgFw7733ptZ/7nOfY+XKlc3LWyarb7rp\nJs4888yK/vb8+fM577zzUpeliAgaGhpSJ6l1Bx5KsqrwUFLX0p2HksxDSWZmVmUuDGZmllLTwiDp\nNklNkl4uW3e1pNckvSjpbkkDyh6bKWl58vhJtcxmZmatq+kcg6SxwHrgjog4KFn3JeCJiPinpJ9Q\nunbHTEkHAr8CjgBGAo8Bn21tMsFzDPnjOYaupaGhgcbGxqxjWI3U19ezYsWKbdZXOsdQ06OSIuJp\nSfUt1j1WtvgM8LXk/kTgNxGxGVghaTlwJPBsLTOa9UStfWiYbZH1HMM04KHk/p7AqrLH3knWmZlZ\nJ8rsPAZJlwGbImL7V7lqQ/k1UgqFAoVCoTrBzMy6iWKx2Pw7Ee1R8/MYkqGkB7bMMSTrzgGmA8dH\nxD+SdTMozTdclSz/AZgVEdsMJXmOIX88x2CWf3k6j2HLlbRKC9KXgR8AE7cUhcT9wBmS+koaBewD\nLO6EfGZmVqamQ0mS5gMFYIiklcAs4FKgL/Bocmr5MxFxfkQsk3QnsAzYBJzvboGZWefzJTGsKjyU\nZJZ/eRpKMutR6uoakJTpra6uIetmsC7MPQarCvcYylK4LSyn3GMwM7MOcWEwM7MUFwYzM0txYTAz\nsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYzM0txYTAzsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MU\nFwYzM0txYTAzsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYzM0upaWGQdJukJkkvl60bJOkRSW9I\neljSwLLHZkpaLuk1SSfVMpuZmbWu1j2GucDJLdbNAB6LiP2AJ4CZAJIOBCYDBwD/BtwoSTXOt1Pq\n6hqQlOmtrq4h62Yws26mpoUhIp4G/tZi9anAvOT+PGBScn8i8JuI2BwRK4DlwJG1zLezmpoagcj0\nVspgZlY9WcwxDIuIJoCIeA8YlqzfE1hV9rx3knVmZtaJemcdgNJX33abPXt28/1CoUChUKhSHDOz\n7qFYLFIsFtv9OkV06HO58g1I9cADEXFQsvwaUIiIJkl1wJMRcYCkGUBExFXJ8/4AzIqIZ1v5m1Hr\n3JUoTYFknUO4LZpTuC22pshFW1i+SCIidjh32xlDSUpuW9wPnJPcnwrcV7b+DEl9JY0C9gEWd0I+\nMzMrU9OhJEnzgQIwRNJKYBbwE+AuSdOARkpHIhERyyTdCSwDNgHn56JbYGbWw9R8KKkWPJSUSpGL\nIQO3RVkKt4XlVJ6GkszMrAtxYTAzsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYzM0txYTAzsxQX\nBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYzM0txYTAzsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYz\nM0upqDBI+k+1DmJmZvlQaY/hRkmLJZ0vaWBNE5mZWaYqKgwR8UXgG8BewJ8kzZd0Yk2TmZlZJhQR\nlT9Z2gWYBPwUWAcIuDQi7qlNvDZzRHty1zAHkHUO4bZoTuG22JoiF21h+SKJiNCOnlfpHMNBkuYA\nrwHHAxMi4oDk/pwOBrxY0iuSXpb0K0l9JQ2S9IikNyQ97GErM7POV1GPQdJC4FZgQUR83OKxKRHx\ni3ZtVBoBPA3sHxEbJf0WeAg4EPhrRFwt6RJgUETMaOX17jFsTZGLb4Zui7IUbgvLqar2GICvAPO3\nFAVJvST1B2hvUSizC7CrpN5AP+Ad4FRgXvL4PErDVmZm1okqLQyPUfrw3qJ/sq5DImI1cA2wklJB\n+HtEPAYMj4im5DnvAcM6ug0zM+uY3hU+79MRsX7LQkSs39Jj6AhJn6HUO6gH/g7cJekbbNv/brMv\nPHv27Ob7hUKBQqHQ0ThmZt1SsVikWCy2+3WVzjH8EfivEfFCsnwYcENEHN3uLZZe/3Xg5IiYnixP\nAb5AaTK7EBFNkuqAJ5NJ7pav9xzD1hS5GEt2W5SlcFtYTlU6x1Bpj+F7lL7Vr6Z0iGod8O87kW8l\n8AVJnwb+AZwAPAesB84BrgKmAvftxDbMzKwDKj6PQVIfYL9k8Y2I2LRTG5ZmAWcAm4AlwH8BdgPu\npHQiXSMwOSLWtvJa9xi2psjFN0O3RVkKt4XlVKU9hvYUhmOABsp6GRFxR0cD7gwXhlSKXHwAuC3K\nUrgtLKeqOpQk6RfAGOBF4JNkdQCZFAYzM6udSucYDgcOzMXXdDMzq6lKz2N4hdKEs5mZdXOV9hiG\nAsskLaZ0FBEAETGxJqnMbCd9KpnryNbw4fW8996KrGNYO1V6HsP41tZHxMKqJ6qAJ59TKXIxyei2\nKEuRk7bIPgPkZZ9YSS2OSqoHPhsRjyVnPe8SER/sZM4OcWFIpcjFPzy3RVmKnLRF9hkgL/vESqp9\n2e3pwALgpmTVnsC9HY9nZmZ5Venk8wXAsZR+nIeIWI4vcGdm1i1VWhj+EREbtywkl8p2/9DMrBuq\ntDAslHQp0C/5ree7gAdqF8vMzLJS6VFJvYBvASdRmtV6GLg1qxlgTz6nUuRics9tUZYiJ22RfQbI\nyz6xkqoflZQnLgypFLn4h+e2KEuRk7bIPgPkZZ9YSbWvlfR/aeX/sogY3YFsZmaWY+25VtIWnwZO\nBwZXP46ZmWWtw0NJkv4UEYdVOU+l2/ZQ0tYUueiquy3KUuSkLbLPAHnZJ1ZS7aGkQ8sWe1HqQVTa\n2zAzsy6k0g/3a8rubwZWAJOrnsbMzDLno5J2LgfZd9fz0VV3W5SlyElbZJ8B8rJPrKTaQ0n/sb3H\nI+LaSoOZmVm+teeopCOA+5PlCcBiYHktQpmZWXYqPfN5EfCVLZfZlrQb8L8iYlyN87WVx0NJW1Pk\noqvutihLkZO2yD4D5GWfWElVL7sNDAc2li1vTNaZmVk3U+lQ0h3AYkm/S5YnAfNqE8nMzLLUnl9w\nOxT4YrK4KCKW7NSGpYHArcDngH8C04A3gd8C9SSHxEbE31t5rYeStqbIRVfdbVGWIidtkX0GyMs+\nsZJqDyUB9AfWRcR1wNuSRnU4Xcl1wEMRcQDweeB1YAbwWETsBzwBzNzJbZiZWTtVOvk8i9KRSftF\nxL6SRgB3RcSxHdqoNABYEhFjWqx/HRgfEU2S6oBiROzfyuvdY9iaIhffyNwWZSly0hbZZ4C87BMr\nqXaP4TRgIvAhQESsBnbreDxGAe9LmivpBUk3S+oPDI+IpmQb7+GfDzUz63SVTj5vjIiQFACSdq3C\ndg8FLoiI5yXNoTSM1PKrRZtfNWbPnt18v1AoUCgUdjKSmVn3UiwWKRaL7X5dpUNJ3wc+C5wIXElp\nonh+RFzf7i2W/t5w4P9s+T0HSWMpFYYxQKFsKOnJZA6i5es9lLQ1RS666m6LshQ5aYvsM0Be9omV\nVP0X3JLfem7+ac+IeHQnAy4EpkfEm8kcRv/koTURcZWkS4BBETGjlde6MGxNkYt/eG6LshQ5aYvs\nM0Be9omVVK0wSNqF0pFCx1UrXPJ3P0/pcNU+wJ+Bc4FdgDuBvYBGSoerrm3ltS4MW1Pk4h+e26Is\nRU7aIvsMkJd9YiVV7TFIehz4z62dU5AFF4ZUilz8w3NblKXISVtknwHysk+spKpXVwXWA0slPUpy\nZBJARFzUwXxmZpZTlRaGe5KbmZl1c9sdSpK0d0Ss7MQ8FfFQUipFLrrqbouyFDlpi+wzQF72iZVU\n6wS3e8v+4N07ncrMzHJvR0NJ5ZVldC2DtNe++x6R6fb79q10FM7MrGvZ0adbtHE/c8uX35jp9nfd\ndbu/dmpm1mXtqDB8XtI6Sj2Hfsl9kuWIiAE1Tbdd2fYYdtllUKbbNzOrle0WhojYpbOCmJlZPrTn\n9xjMzKwHcGEwM7MUFwYzM0txYTAzsxQXBjMzS3FhMDOzFBcGM+vW6uoakJT5ra6uIeumqJiv62Bm\n3VpTUyN5uHBDU9MOr12XG+4xmJlZiguDmZmluDCYmVmKC4OZmaW4MJiZWYoLg5mZpbgwmJlZSqaF\nQVIvSS9Iuj9ZHiTpEUlvSHpY0sAs85mZ9URZ9xi+CywrW54BPBYR+wFPADMzSWVm1oNlVhgkjQRO\nAW4tW30qMC+5Pw+Y1Nm5zMx6uix7DHOAH5A+V314RDQBRMR7wLAsgpmZ9WSZXCtJ0leApoh4UVJh\nO0/dzgVOZpfdLyQ3MzPbolgsUiwW2/06RXT+xaUk/Xfgm8BmoB+wG/A74HCgEBFNkuqAJyPigFZe\nH1lfFGvAgImsW/cAWecAkcU+3CaFhNsiSZGTtsg+A+Rhn+Rjf0Be2iIidng1v0yGkiLi0ojYOyJG\nA2cAT0TEFOAB4JzkaVOB+7LIZ2bWk2V9VFJLPwFOlPQGcEKybGZmnSjz32OIiIXAwuT+GuBL2SYy\nM+vZ8tZjMDOzjLkwmJlZiguDmZmluDCYmVmKC4OZmaW4MJiZWYoLg5mZpbgwmJlZiguDmZmluDCY\nmVmKC4OZmaW4MJiZWYoLg5mZpbgwmJlZiguDmZmluDCYmVmKC4OZmaW4MJiZWYoLg5mZpbgwmJlZ\niguDmZmluDCYmVmKC4OZmaVkUhgkjZT0hKRXJS2VdFGyfpCkRyS9IelhSQOzyGdm1pNl1WPYDPxH\nRPwrcDRwgaT9gRnAYxGxH/AEMDOjfGZmPVYmhSEi3ouIF5P764HXgJHAqcC85GnzgElZ5DMz68ky\nn2OQ1AAcDDwDDI+IJigVD2BYdsnMzHqm3lluXNK/AAuA70bEeknR4iktl8vMLrtfSG5mZrZFsVik\nWCy2+3WK2M5nbw1J6g08CPw+Iq5L1r0GFCKiSVId8GREHNDKa2O7NaMTDBgwkXXrHiDrHCCy2oep\nFBJuiyRFTtoi+wyQh32Sj/0BeWmLiNCOnpflUNLtwLItRSFxP3BOcn8qcF9nhzIz6+kyGUqSdCzw\nDWCppCWUyvmlwFXAnZKmAY3A5CzymZn1ZJkUhoj4I7BLGw9/qTOzmJlZWuZHJZmZWb64MJiZWYoL\ng5mZpbgwmJlZiguDmZmluDCYmVmKC4OZmaW4MJiZWYoLg5mZpbgwmJlZiguDmZmluDCYmVmKC4OZ\nmaW4MJiZWYoLg5mZpbgwmJlZiguDmZmluDCYmVmKC4OZmaVk8pvPZmY9z6eQlHWIirgwmJl1in8A\nkXGGygqTh5LMzCwll4VB0pclvS7pTUmXZJ3HzKwnyV1hkNQLuAE4GfhX4ExJ+2ebqrMVsw5QY8Ws\nA9hOKWYdoIaKWQfIhdwVBuBIYHlENEbEJuA3wKkZZ+pkxawD1Fgx6wC2U4pZB6ihYtYBciGPhWFP\nYFXZ8tvJOjMz6wRd9qikAQMmZLr9jRsXZ7p9M7NaUUTWh0+lSfoCMDsivpwszwAiIq4qe06+QpuZ\ndRERscNjVvNYGHYB3gBOAN4FFgNnRsRrmQYzM+shcjeUFBGfSLoQeITSHMhtLgpmZp0ndz0GMzPL\nVh6PStqu7nzym6TbJDVJejnrLNUmaaSkJyS9KmmppIuyzlRNkj4l6VlJS5L3NyvrTLUgqZekFyTd\nn3WWapO0QtJLyT7sdkeXSBoo6S5JryX/Do9q87ldqceQnPz2JqX5h9XAc8AZEfF6psGqRNJYYD1w\nR0QclHWeapJUB9RFxIuS/gX4E3Bqd9l3AJL6R8RHyTzZH4GLIqJbfcBIuhg4DBgQEROzzlNNkv4M\nHBYRf8s6Sy1I+p/AwoiYK6k30D8i1rX23K7WY+jWJ79FxNNAt/yfMiLei4gXk/vrgdfoZuenRMRH\nyd1PUZq/6zrfuiogaSRwCnBr1llqRHS9z8SKSBoAfDEi5gJExOa2igJ0vUbwyW/dgKQG4GDg2WyT\nVFcyzLIEeA94NCKeyzpTlc0BfkA3K3hlAnhU0nOSpmcdpspGAe9LmpsMBd4sqV9bT+5qhcG6uGQY\naQHw3aTn0G1ExD8j4hBgJHCUpAOzzlQtkr4CNCW9PlHp9Zu7lmMj4lBKvaILkqHd7qI3cCjwP5L3\n+BEwo60nd7XC8A6wd9nyyGSddQHJuOYC4BcRcV/WeWol6aI/CXw56yxVdCwwMRmH/zVwnKQ7Ms5U\nVRHxbvLf/wf8jtLQdXfxNrAqIp5PlhdQKhSt6mqF4TlgH0n1kvoCZwDd7eiI7vptDOB2YFlEXJd1\nkGqTNFTSwOR+P+BEoNtMrEfEpRGxd0SMpvTv7omIODvrXNUiqX/Sm0XSrsBJwCvZpqqeiGgCVkna\nN1l1ArCsrefn7gS37enuJ79Jmg8UgCGSVgKztkwWdXWSjgW+ASxNxuEDuDQi/pBtsqrZA5iXHDnX\nC/htRDyUcSar3HDgd8nldnoDv4qIRzLOVG0XAb+S1Af4M3BuW0/sUoermplZ7XW1oSQzM6sxFwYz\nM0txYTAzsxQXBjMzS3FhMDOzFBcGMzNLcWEwM7MUFwYzM0v5/weTUOwwZPbBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110089f90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.plot(x='time_of_day',y='day_of_week',kind='hist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pd.scatter_matrix(df,alpha=0.1,figsize=(15,15), diagonal='hist');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>id</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">1</th>\n",
       "      <th>count</th>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>3.500000e+01</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>3.500000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>13.057143</td>\n",
       "      <td>2.057143</td>\n",
       "      <td>1.714286</td>\n",
       "      <td>6.872701e+17</td>\n",
       "      <td>3.542857</td>\n",
       "      <td>0.342857</td>\n",
       "      <td>0.228571</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>241.000000</td>\n",
       "      <td>6.872701e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.862479</td>\n",
       "      <td>1.493965</td>\n",
       "      <td>1.462042</td>\n",
       "      <td>1.777861e+15</td>\n",
       "      <td>3.860378</td>\n",
       "      <td>0.481594</td>\n",
       "      <td>0.426043</td>\n",
       "      <td>0.505433</td>\n",
       "      <td>94.183176</td>\n",
       "      <td>1.777861e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.840946e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>138.000000</td>\n",
       "      <td>6.840946e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.861119e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>146.500000</td>\n",
       "      <td>6.861119e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.867537e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>225.000000</td>\n",
       "      <td>6.867537e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>15.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.878042e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>326.000000</td>\n",
       "      <td>6.878042e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>25.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.917681e+17</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>375.000000</td>\n",
       "      <td>6.917681e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">6</th>\n",
       "      <th>count</th>\n",
       "      <td>55.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>5.500000e+01</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>5.500000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>20.636364</td>\n",
       "      <td>2.836364</td>\n",
       "      <td>1.900000</td>\n",
       "      <td>6.124448e+17</td>\n",
       "      <td>1.909091</td>\n",
       "      <td>0.290909</td>\n",
       "      <td>0.418182</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>264.618182</td>\n",
       "      <td>6.124448e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.873240</td>\n",
       "      <td>1.853562</td>\n",
       "      <td>1.586124</td>\n",
       "      <td>2.110101e+15</td>\n",
       "      <td>1.126659</td>\n",
       "      <td>0.458368</td>\n",
       "      <td>0.497807</td>\n",
       "      <td>0.494413</td>\n",
       "      <td>88.131739</td>\n",
       "      <td>2.110101e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.079546e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>104.000000</td>\n",
       "      <td>6.079546e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>17.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.113585e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>205.500000</td>\n",
       "      <td>6.113585e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.120688e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>264.000000</td>\n",
       "      <td>6.120688e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>26.000000</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>6.144020e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>344.000000</td>\n",
       "      <td>6.144020e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.156911e+17</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>442.000000</td>\n",
       "      <td>6.156911e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">7</th>\n",
       "      <th>count</th>\n",
       "      <td>73.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>7.300000e+01</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>7.300000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>18.136986</td>\n",
       "      <td>2.164384</td>\n",
       "      <td>2.192308</td>\n",
       "      <td>6.224064e+17</td>\n",
       "      <td>4.287671</td>\n",
       "      <td>0.260274</td>\n",
       "      <td>0.506849</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>254.164384</td>\n",
       "      <td>6.224064e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.194375</td>\n",
       "      <td>1.280340</td>\n",
       "      <td>1.744001</td>\n",
       "      <td>3.334705e+15</td>\n",
       "      <td>9.187067</td>\n",
       "      <td>0.441821</td>\n",
       "      <td>0.503413</td>\n",
       "      <td>0.503035</td>\n",
       "      <td>77.017677</td>\n",
       "      <td>3.334705e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.160466e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>6.160466e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.207510e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>216.000000</td>\n",
       "      <td>6.207510e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.231497e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>258.000000</td>\n",
       "      <td>6.231497e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>25.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.750000</td>\n",
       "      <td>6.247474e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>313.000000</td>\n",
       "      <td>6.247474e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>31.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.272640e+17</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>386.000000</td>\n",
       "      <td>6.272640e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">8</th>\n",
       "      <th>count</th>\n",
       "      <td>69.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>6.900000e+01</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>6.900000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>17.478261</td>\n",
       "      <td>2.710145</td>\n",
       "      <td>2.300000</td>\n",
       "      <td>6.334217e+17</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>0.217391</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>239.014493</td>\n",
       "      <td>6.334217e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.827346</td>\n",
       "      <td>1.572759</td>\n",
       "      <td>1.263547</td>\n",
       "      <td>3.632823e+15</td>\n",
       "      <td>4.676810</td>\n",
       "      <td>0.415493</td>\n",
       "      <td>0.491618</td>\n",
       "      <td>0.499360</td>\n",
       "      <td>85.203475</td>\n",
       "      <td>3.632823e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.272702e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>135.000000</td>\n",
       "      <td>6.272702e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.308697e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>150.000000</td>\n",
       "      <td>6.308697e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>19.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.341380e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>257.000000</td>\n",
       "      <td>6.341380e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">9</th>\n",
       "      <th>std</th>\n",
       "      <td>9.011368</td>\n",
       "      <td>1.341858</td>\n",
       "      <td>3.257780</td>\n",
       "      <td>3.245817e+15</td>\n",
       "      <td>5.066955</td>\n",
       "      <td>0.344745</td>\n",
       "      <td>0.499291</td>\n",
       "      <td>0.447400</td>\n",
       "      <td>91.119182</td>\n",
       "      <td>3.245817e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.385084e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>135.000000</td>\n",
       "      <td>6.385084e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.414256e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>152.000000</td>\n",
       "      <td>6.414256e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.435046e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>6.435046e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>23.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.467907e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>279.000000</td>\n",
       "      <td>6.467907e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>6.493411e+17</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>449.000000</td>\n",
       "      <td>6.493411e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">10</th>\n",
       "      <th>count</th>\n",
       "      <td>116.000000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>1.160000e+02</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>1.160000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.862069</td>\n",
       "      <td>2.629310</td>\n",
       "      <td>1.795918</td>\n",
       "      <td>6.527716e+17</td>\n",
       "      <td>4.163793</td>\n",
       "      <td>0.172414</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>0.508621</td>\n",
       "      <td>252.560345</td>\n",
       "      <td>6.527716e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.033951</td>\n",
       "      <td>1.130834</td>\n",
       "      <td>1.731805</td>\n",
       "      <td>2.555516e+15</td>\n",
       "      <td>4.816537</td>\n",
       "      <td>0.379378</td>\n",
       "      <td>0.494649</td>\n",
       "      <td>0.502095</td>\n",
       "      <td>80.750748</td>\n",
       "      <td>2.555516e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.496141e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>6.496141e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.515610e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>190.750000</td>\n",
       "      <td>6.515610e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.520133e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>262.000000</td>\n",
       "      <td>6.520133e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.541057e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>284.750000</td>\n",
       "      <td>6.541057e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>6.602246e+17</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>425.000000</td>\n",
       "      <td>6.602246e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">11</th>\n",
       "      <th>count</th>\n",
       "      <td>43.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>13.906977</td>\n",
       "      <td>1.325581</td>\n",
       "      <td>2.400000</td>\n",
       "      <td>6.655223e+17</td>\n",
       "      <td>3.690476</td>\n",
       "      <td>0.069767</td>\n",
       "      <td>0.348837</td>\n",
       "      <td>0.581395</td>\n",
       "      <td>202.023256</td>\n",
       "      <td>6.655223e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.733662</td>\n",
       "      <td>1.209505</td>\n",
       "      <td>1.957890</td>\n",
       "      <td>3.138674e+15</td>\n",
       "      <td>3.032250</td>\n",
       "      <td>0.257770</td>\n",
       "      <td>0.482243</td>\n",
       "      <td>0.499169</td>\n",
       "      <td>71.815401</td>\n",
       "      <td>3.138674e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.612370e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>135.000000</td>\n",
       "      <td>6.612370e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.632601e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>146.000000</td>\n",
       "      <td>6.632601e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.649104e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>155.000000</td>\n",
       "      <td>6.649104e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.676899e+17</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>266.000000</td>\n",
       "      <td>6.676899e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.713959e+17</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>375.000000</td>\n",
       "      <td>6.713959e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">12</th>\n",
       "      <th>count</th>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>3.500000e+01</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>3.500000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.457143</td>\n",
       "      <td>3.057143</td>\n",
       "      <td>1.937500</td>\n",
       "      <td>6.743419e+17</td>\n",
       "      <td>3.029412</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.371429</td>\n",
       "      <td>0.514286</td>\n",
       "      <td>252.857143</td>\n",
       "      <td>6.743419e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.636379</td>\n",
       "      <td>1.764734</td>\n",
       "      <td>1.340087</td>\n",
       "      <td>2.747556e+15</td>\n",
       "      <td>2.110368</td>\n",
       "      <td>0.405840</td>\n",
       "      <td>0.490241</td>\n",
       "      <td>0.507093</td>\n",
       "      <td>103.178356</td>\n",
       "      <td>2.747556e+15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.717135e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>6.717135e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.500000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.724021e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>149.000000</td>\n",
       "      <td>6.724021e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>6.729805e+17</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>233.000000</td>\n",
       "      <td>6.729805e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>11.500000</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>6.754842e+17</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>344.000000</td>\n",
       "      <td>6.754842e+17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>29.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.819313e+17</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>448.000000</td>\n",
       "      <td>6.819313e+17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             day_of_month  day_of_week  favorite_count            id  \\\n",
       "month                                                                  \n",
       "1     count     35.000000    35.000000       28.000000  3.500000e+01   \n",
       "      mean      13.057143     2.057143        1.714286  6.872701e+17   \n",
       "      std        4.862479     1.493965        1.462042  1.777861e+15   \n",
       "      min        4.000000     0.000000        1.000000  6.840946e+17   \n",
       "      25%        9.500000     1.000000        1.000000  6.861119e+17   \n",
       "      50%       12.000000     1.000000        1.000000  6.867537e+17   \n",
       "      75%       15.000000     4.000000        2.000000  6.878042e+17   \n",
       "      max       25.000000     4.000000        8.000000  6.917681e+17   \n",
       "6     count     55.000000    55.000000       20.000000  5.500000e+01   \n",
       "      mean      20.636364     2.836364        1.900000  6.124448e+17   \n",
       "      std        5.873240     1.853562        1.586124  2.110101e+15   \n",
       "      min        8.000000     0.000000        1.000000  6.079546e+17   \n",
       "      25%       17.500000     1.000000        1.000000  6.113585e+17   \n",
       "      50%       20.000000     3.000000        1.000000  6.120688e+17   \n",
       "      75%       26.000000     4.500000        2.250000  6.144020e+17   \n",
       "      max       30.000000     6.000000        6.000000  6.156911e+17   \n",
       "7     count     73.000000    73.000000       26.000000  7.300000e+01   \n",
       "      mean      18.136986     2.164384        2.192308  6.224064e+17   \n",
       "      std        9.194375     1.280340        1.744001  3.334705e+15   \n",
       "      min        1.000000     0.000000        1.000000  6.160466e+17   \n",
       "      25%       14.000000     1.000000        1.000000  6.207510e+17   \n",
       "      50%       20.000000     2.000000        2.000000  6.231497e+17   \n",
       "      75%       25.000000     3.000000        2.750000  6.247474e+17   \n",
       "      max       31.000000     5.000000        8.000000  6.272640e+17   \n",
       "8     count     69.000000    69.000000       30.000000  6.900000e+01   \n",
       "      mean      17.478261     2.710145        2.300000  6.334217e+17   \n",
       "      std        9.827346     1.572759        1.263547  3.632823e+15   \n",
       "      min        1.000000     0.000000        1.000000  6.272702e+17   \n",
       "      25%       10.000000     1.000000        1.000000  6.308697e+17   \n",
       "      50%       19.000000     3.000000        2.000000  6.341380e+17   \n",
       "...                   ...          ...             ...           ...   \n",
       "9     std        9.011368     1.341858        3.257780  3.245817e+15   \n",
       "      min        1.000000     0.000000        1.000000  6.385084e+17   \n",
       "      25%        9.000000     1.250000        1.000000  6.414256e+17   \n",
       "      50%       14.000000     2.000000        1.000000  6.435046e+17   \n",
       "      75%       23.000000     3.000000        3.000000  6.467907e+17   \n",
       "      max       30.000000     5.000000       20.000000  6.493411e+17   \n",
       "10    count    116.000000   116.000000       49.000000  1.160000e+02   \n",
       "      mean       9.862069     2.629310        1.795918  6.527716e+17   \n",
       "      std        7.033951     1.130834        1.731805  2.555516e+15   \n",
       "      min        1.000000     0.000000        1.000000  6.496141e+17   \n",
       "      25%        7.000000     2.000000        1.000000  6.515610e+17   \n",
       "      50%        8.000000     3.000000        1.000000  6.520133e+17   \n",
       "      75%       14.000000     3.000000        2.000000  6.541057e+17   \n",
       "      max       30.000000     5.000000       11.000000  6.602246e+17   \n",
       "11    count     43.000000    43.000000       25.000000  4.300000e+01   \n",
       "      mean      13.906977     1.325581        2.400000  6.655223e+17   \n",
       "      std        8.733662     1.209505        1.957890  3.138674e+15   \n",
       "      min        2.000000     0.000000        1.000000  6.612370e+17   \n",
       "      25%        7.500000     0.000000        1.000000  6.632601e+17   \n",
       "      50%       12.000000     1.000000        2.000000  6.649104e+17   \n",
       "      75%       20.000000     2.000000        3.000000  6.676899e+17   \n",
       "      max       30.000000     4.000000        7.000000  6.713959e+17   \n",
       "12    count     35.000000    35.000000       16.000000  3.500000e+01   \n",
       "      mean       8.457143     3.057143        1.937500  6.743419e+17   \n",
       "      std        7.636379     1.764734        1.340087  2.747556e+15   \n",
       "      min        1.000000     0.000000        1.000000  6.717135e+17   \n",
       "      25%        3.500000     2.000000        1.000000  6.724021e+17   \n",
       "      50%        5.000000     3.000000        1.500000  6.729805e+17   \n",
       "      75%       11.500000     4.500000        2.250000  6.754842e+17   \n",
       "      max       29.000000     6.000000        6.000000  6.819313e+17   \n",
       "\n",
       "             retweet_count    t_bullet   t_limited  t_northbound    train_id  \\\n",
       "month                                                                          \n",
       "1     count      35.000000   35.000000   35.000000     35.000000   35.000000   \n",
       "      mean        3.542857    0.342857    0.228571      0.542857  241.000000   \n",
       "      std         3.860378    0.481594    0.426043      0.505433   94.183176   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  138.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  146.500000   \n",
       "      50%         3.000000    0.000000    0.000000      1.000000  225.000000   \n",
       "      75%         4.000000    1.000000    0.000000      1.000000  326.000000   \n",
       "      max        24.000000    1.000000    1.000000      1.000000  375.000000   \n",
       "6     count      55.000000   55.000000   55.000000     55.000000   55.000000   \n",
       "      mean        1.909091    0.290909    0.418182      0.400000  264.618182   \n",
       "      std         1.126659    0.458368    0.497807      0.494413   88.131739   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  104.000000   \n",
       "      25%         1.000000    0.000000    0.000000      0.000000  205.500000   \n",
       "      50%         2.000000    0.000000    0.000000      0.000000  264.000000   \n",
       "      75%         3.000000    1.000000    1.000000      1.000000  344.000000   \n",
       "      max         7.000000    1.000000    1.000000      1.000000  442.000000   \n",
       "7     count      73.000000   73.000000   73.000000     73.000000   73.000000   \n",
       "      mean        4.287671    0.260274    0.506849      0.520548  254.164384   \n",
       "      std         9.187067    0.441821    0.503413      0.503035   77.017677   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  134.000000   \n",
       "      25%         1.000000    0.000000    0.000000      0.000000  216.000000   \n",
       "      50%         2.000000    0.000000    1.000000      1.000000  258.000000   \n",
       "      75%         3.000000    1.000000    1.000000      1.000000  313.000000   \n",
       "      max        60.000000    1.000000    1.000000      1.000000  386.000000   \n",
       "8     count      69.000000   69.000000   69.000000     69.000000   69.000000   \n",
       "      mean        3.666667    0.217391    0.391304      0.434783  239.014493   \n",
       "      std         4.676810    0.415493    0.491618      0.499360   85.203475   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  135.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  150.000000   \n",
       "      50%         2.000000    0.000000    0.000000      0.000000  257.000000   \n",
       "...                    ...         ...         ...           ...         ...   \n",
       "9     std         5.066955    0.344745    0.499291      0.447400   91.119182   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  135.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  152.000000   \n",
       "      50%         2.000000    0.000000    0.000000      1.000000  254.000000   \n",
       "      75%         4.000000    0.000000    1.000000      1.000000  279.000000   \n",
       "      max        30.000000    1.000000    1.000000      1.000000  449.000000   \n",
       "10    count     116.000000  116.000000  116.000000    116.000000  116.000000   \n",
       "      mean        4.163793    0.172414    0.413793      0.508621  252.560345   \n",
       "      std         4.816537    0.379378    0.494649      0.502095   80.750748   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  134.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  190.750000   \n",
       "      50%         3.000000    0.000000    0.000000      1.000000  262.000000   \n",
       "      75%         4.000000    0.000000    1.000000      1.000000  284.750000   \n",
       "      max        33.000000    1.000000    1.000000      1.000000  425.000000   \n",
       "11    count      42.000000   43.000000   43.000000     43.000000   43.000000   \n",
       "      mean        3.690476    0.069767    0.348837      0.581395  202.023256   \n",
       "      std         3.032250    0.257770    0.482243      0.499169   71.815401   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  135.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  146.000000   \n",
       "      50%         3.000000    0.000000    0.000000      1.000000  155.000000   \n",
       "      75%         3.750000    0.000000    1.000000      1.000000  266.000000   \n",
       "      max        15.000000    1.000000    1.000000      1.000000  375.000000   \n",
       "12    count      34.000000   35.000000   35.000000     35.000000   35.000000   \n",
       "      mean        3.029412    0.200000    0.371429      0.514286  252.857143   \n",
       "      std         2.110368    0.405840    0.490241      0.507093  103.178356   \n",
       "      min         1.000000    0.000000    0.000000      0.000000  134.000000   \n",
       "      25%         2.000000    0.000000    0.000000      0.000000  149.000000   \n",
       "      50%         2.000000    0.000000    0.000000      1.000000  233.000000   \n",
       "      75%         3.000000    0.000000    1.000000      1.000000  344.000000   \n",
       "      max        10.000000    1.000000    1.000000      1.000000  448.000000   \n",
       "\n",
       "                 tweet_id  \n",
       "month                      \n",
       "1     count  3.500000e+01  \n",
       "      mean   6.872701e+17  \n",
       "      std    1.777861e+15  \n",
       "      min    6.840946e+17  \n",
       "      25%    6.861119e+17  \n",
       "      50%    6.867537e+17  \n",
       "      75%    6.878042e+17  \n",
       "      max    6.917681e+17  \n",
       "6     count  5.500000e+01  \n",
       "      mean   6.124448e+17  \n",
       "      std    2.110101e+15  \n",
       "      min    6.079546e+17  \n",
       "      25%    6.113585e+17  \n",
       "      50%    6.120688e+17  \n",
       "      75%    6.144020e+17  \n",
       "      max    6.156911e+17  \n",
       "7     count  7.300000e+01  \n",
       "      mean   6.224064e+17  \n",
       "      std    3.334705e+15  \n",
       "      min    6.160466e+17  \n",
       "      25%    6.207510e+17  \n",
       "      50%    6.231497e+17  \n",
       "      75%    6.247474e+17  \n",
       "      max    6.272640e+17  \n",
       "8     count  6.900000e+01  \n",
       "      mean   6.334217e+17  \n",
       "      std    3.632823e+15  \n",
       "      min    6.272702e+17  \n",
       "      25%    6.308697e+17  \n",
       "      50%    6.341380e+17  \n",
       "...                   ...  \n",
       "9     std    3.245817e+15  \n",
       "      min    6.385084e+17  \n",
       "      25%    6.414256e+17  \n",
       "      50%    6.435046e+17  \n",
       "      75%    6.467907e+17  \n",
       "      max    6.493411e+17  \n",
       "10    count  1.160000e+02  \n",
       "      mean   6.527716e+17  \n",
       "      std    2.555516e+15  \n",
       "      min    6.496141e+17  \n",
       "      25%    6.515610e+17  \n",
       "      50%    6.520133e+17  \n",
       "      75%    6.541057e+17  \n",
       "      max    6.602246e+17  \n",
       "11    count  4.300000e+01  \n",
       "      mean   6.655223e+17  \n",
       "      std    3.138674e+15  \n",
       "      min    6.612370e+17  \n",
       "      25%    6.632601e+17  \n",
       "      50%    6.649104e+17  \n",
       "      75%    6.676899e+17  \n",
       "      max    6.713959e+17  \n",
       "12    count  3.500000e+01  \n",
       "      mean   6.743419e+17  \n",
       "      std    2.747556e+15  \n",
       "      min    6.717135e+17  \n",
       "      25%    6.724021e+17  \n",
       "      50%    6.729805e+17  \n",
       "      75%    6.754842e+17  \n",
       "      max    6.819313e+17  \n",
       "\n",
       "[64 rows x 10 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('month').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = df[df['train_id'] > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>topic_train</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day_of_week</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>54</td>\n",
       "      <td>27</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>102</td>\n",
       "      <td>48</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>0</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>135</td>\n",
       "      <td>59</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>0</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>117</td>\n",
       "      <td>44</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75</td>\n",
       "      <td>24</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>48</td>\n",
       "      <td>32</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             created_at  favorite_count  hashtags   id  \\\n",
       "day_of_week                                              \n",
       "0                    54              27        54   54   \n",
       "1                   102              48       102  102   \n",
       "2                   135              59       135  135   \n",
       "3                   117              44       117  117   \n",
       "4                    75              24        75   75   \n",
       "5                    48              32        48   48   \n",
       "6                     5               5         5    5   \n",
       "\n",
       "             in_reply_to_screen_name  retweet_count  text  day_of_month  \\\n",
       "day_of_week                                                               \n",
       "0                                  0             49    54            54   \n",
       "1                                  0            102   102           102   \n",
       "2                                  0            135   135           135   \n",
       "3                                  0            117   117           117   \n",
       "4                                  0             75    75            75   \n",
       "5                                  0             48    48            48   \n",
       "6                                  0              4     5             5   \n",
       "\n",
       "             month  time_of_day  topic_train  t_bullet  t_limited  \\\n",
       "day_of_week                                                         \n",
       "0               54           54           54        54         54   \n",
       "1              102          102          102       102        102   \n",
       "2              135          135          135       135        135   \n",
       "3              117          117          117       117        117   \n",
       "4               75           75           75        75         75   \n",
       "5               48           48           48        48         48   \n",
       "6                5            5            5         5          5   \n",
       "\n",
       "             t_northbound  timestamp  train_id  tweet_id  \n",
       "day_of_week                                               \n",
       "0                      54         54        54        54  \n",
       "1                     102        102       102       102  \n",
       "2                     135        135       135       135  \n",
       "3                     117        117       117       117  \n",
       "4                      75         75        75        75  \n",
       "5                      48         48        48        48  \n",
       "6                       5          5         5         5  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('day_of_week').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>topic_train</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>28</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>55</td>\n",
       "      <td>20</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>73</td>\n",
       "      <td>26</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>69</td>\n",
       "      <td>30</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>110</td>\n",
       "      <td>45</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>0</td>\n",
       "      <td>106</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>116</td>\n",
       "      <td>49</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>0</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>43</td>\n",
       "      <td>25</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       created_at  favorite_count  hashtags   id  in_reply_to_screen_name  \\\n",
       "month                                                                       \n",
       "1              35              28        35   35                        0   \n",
       "6              55              20        55   55                        0   \n",
       "7              73              26        73   73                        0   \n",
       "8              69              30        69   69                        0   \n",
       "9             110              45       110  110                        0   \n",
       "10            116              49       116  116                        0   \n",
       "11             43              25        43   43                        0   \n",
       "12             35              16        35   35                        0   \n",
       "\n",
       "       retweet_count  text  day_of_week  day_of_month  time_of_day  \\\n",
       "month                                                                \n",
       "1                 35    35           35            35           35   \n",
       "6                 55    55           55            55           55   \n",
       "7                 73    73           73            73           73   \n",
       "8                 69    69           69            69           69   \n",
       "9                106   110          110           110          110   \n",
       "10               116   116          116           116          116   \n",
       "11                42    43           43            43           43   \n",
       "12                34    35           35            35           35   \n",
       "\n",
       "       topic_train  t_bullet  t_limited  t_northbound  timestamp  train_id  \\\n",
       "month                                                                        \n",
       "1               35        35         35            35         35        35   \n",
       "6               55        55         55            55         55        55   \n",
       "7               73        73         73            73         73        73   \n",
       "8               69        69         69            69         69        69   \n",
       "9              110       110        110           110        110       110   \n",
       "10             116       116        116           116        116       116   \n",
       "11              43        43         43            43         43        43   \n",
       "12              35        35         35            35         35        35   \n",
       "\n",
       "       tweet_id  \n",
       "month            \n",
       "1            35  \n",
       "6            55  \n",
       "7            73  \n",
       "8            69  \n",
       "9           110  \n",
       "10          116  \n",
       "11           43  \n",
       "12           35  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('month').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>topic_train</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time_of_day</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>error</th>\n",
       "      <td>372</td>\n",
       "      <td>185</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>0</td>\n",
       "      <td>366</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "      <td>372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rush_evening</th>\n",
       "      <td>117</td>\n",
       "      <td>38</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rush_morning</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>workday</th>\n",
       "      <td>46</td>\n",
       "      <td>15</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              created_at  favorite_count  hashtags   id  \\\n",
       "time_of_day                                               \n",
       "error                372             185       372  372   \n",
       "rush_evening         117              38       117  117   \n",
       "rush_morning           1               1         1    1   \n",
       "workday               46              15        46   46   \n",
       "\n",
       "              in_reply_to_screen_name  retweet_count  text  day_of_week  \\\n",
       "time_of_day                                                               \n",
       "error                               0            366   372          372   \n",
       "rush_evening                        0            117   117          117   \n",
       "rush_morning                        0              1     1            1   \n",
       "workday                             0             46    46           46   \n",
       "\n",
       "              day_of_month  month  topic_train  t_bullet  t_limited  \\\n",
       "time_of_day                                                           \n",
       "error                  372    372          372       372        372   \n",
       "rush_evening           117    117          117       117        117   \n",
       "rush_morning             1      1            1         1          1   \n",
       "workday                 46     46           46        46         46   \n",
       "\n",
       "              t_northbound  timestamp  train_id  tweet_id  \n",
       "time_of_day                                                \n",
       "error                  372        372       372       372  \n",
       "rush_evening           117        117       117       117  \n",
       "rush_morning             1          1         1         1  \n",
       "workday                 46         46        46        46  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby('time_of_day').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>id</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>favorite_count</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.056542</td>\n",
       "      <td>0.551203</td>\n",
       "      <td>0.100412</td>\n",
       "      <td>-0.013727</td>\n",
       "      <td>0.057577</td>\n",
       "      <td>0.184392</td>\n",
       "      <td>0.019418</td>\n",
       "      <td>0.071410</td>\n",
       "      <td>0.182412</td>\n",
       "      <td>-0.056542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>-0.056542</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.024774</td>\n",
       "      <td>-0.083374</td>\n",
       "      <td>-0.201225</td>\n",
       "      <td>0.119187</td>\n",
       "      <td>-0.048254</td>\n",
       "      <td>-0.091769</td>\n",
       "      <td>0.070766</td>\n",
       "      <td>-0.083925</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>retweet_count</th>\n",
       "      <td>0.551203</td>\n",
       "      <td>0.024774</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.019303</td>\n",
       "      <td>-0.058542</td>\n",
       "      <td>0.030757</td>\n",
       "      <td>0.096546</td>\n",
       "      <td>0.100636</td>\n",
       "      <td>-0.041501</td>\n",
       "      <td>0.148645</td>\n",
       "      <td>0.024774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day_of_week</th>\n",
       "      <td>0.100412</td>\n",
       "      <td>-0.083374</td>\n",
       "      <td>0.019303</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.063651</td>\n",
       "      <td>0.022254</td>\n",
       "      <td>0.047302</td>\n",
       "      <td>-0.022492</td>\n",
       "      <td>-0.061165</td>\n",
       "      <td>0.132644</td>\n",
       "      <td>-0.083374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day_of_month</th>\n",
       "      <td>-0.013727</td>\n",
       "      <td>-0.201225</td>\n",
       "      <td>-0.058542</td>\n",
       "      <td>-0.063651</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.204438</td>\n",
       "      <td>0.014090</td>\n",
       "      <td>0.057579</td>\n",
       "      <td>0.053588</td>\n",
       "      <td>-0.007253</td>\n",
       "      <td>-0.201225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>0.057577</td>\n",
       "      <td>0.119187</td>\n",
       "      <td>0.030757</td>\n",
       "      <td>0.022254</td>\n",
       "      <td>-0.204438</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.145967</td>\n",
       "      <td>0.041591</td>\n",
       "      <td>0.045134</td>\n",
       "      <td>-0.039853</td>\n",
       "      <td>0.119187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_bullet</th>\n",
       "      <td>0.184392</td>\n",
       "      <td>-0.048254</td>\n",
       "      <td>0.096546</td>\n",
       "      <td>0.047302</td>\n",
       "      <td>0.014090</td>\n",
       "      <td>-0.145967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.416707</td>\n",
       "      <td>-0.010226</td>\n",
       "      <td>0.651110</td>\n",
       "      <td>-0.048254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_limited</th>\n",
       "      <td>0.019418</td>\n",
       "      <td>-0.091769</td>\n",
       "      <td>0.100636</td>\n",
       "      <td>-0.022492</td>\n",
       "      <td>0.057579</td>\n",
       "      <td>0.041591</td>\n",
       "      <td>-0.416707</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.041419</td>\n",
       "      <td>0.110541</td>\n",
       "      <td>-0.091769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_northbound</th>\n",
       "      <td>0.071410</td>\n",
       "      <td>0.070766</td>\n",
       "      <td>-0.041501</td>\n",
       "      <td>-0.061165</td>\n",
       "      <td>0.053588</td>\n",
       "      <td>0.045134</td>\n",
       "      <td>-0.010226</td>\n",
       "      <td>-0.041419</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.008933</td>\n",
       "      <td>0.070766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_id</th>\n",
       "      <td>0.182412</td>\n",
       "      <td>-0.083925</td>\n",
       "      <td>0.148645</td>\n",
       "      <td>0.132644</td>\n",
       "      <td>-0.007253</td>\n",
       "      <td>-0.039853</td>\n",
       "      <td>0.651110</td>\n",
       "      <td>0.110541</td>\n",
       "      <td>0.008933</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.083925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tweet_id</th>\n",
       "      <td>-0.056542</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.024774</td>\n",
       "      <td>-0.083374</td>\n",
       "      <td>-0.201225</td>\n",
       "      <td>0.119187</td>\n",
       "      <td>-0.048254</td>\n",
       "      <td>-0.091769</td>\n",
       "      <td>0.070766</td>\n",
       "      <td>-0.083925</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                favorite_count        id  retweet_count  day_of_week  \\\n",
       "favorite_count        1.000000 -0.056542       0.551203     0.100412   \n",
       "id                   -0.056542  1.000000       0.024774    -0.083374   \n",
       "retweet_count         0.551203  0.024774       1.000000     0.019303   \n",
       "day_of_week           0.100412 -0.083374       0.019303     1.000000   \n",
       "day_of_month         -0.013727 -0.201225      -0.058542    -0.063651   \n",
       "month                 0.057577  0.119187       0.030757     0.022254   \n",
       "t_bullet              0.184392 -0.048254       0.096546     0.047302   \n",
       "t_limited             0.019418 -0.091769       0.100636    -0.022492   \n",
       "t_northbound          0.071410  0.070766      -0.041501    -0.061165   \n",
       "train_id              0.182412 -0.083925       0.148645     0.132644   \n",
       "tweet_id             -0.056542  1.000000       0.024774    -0.083374   \n",
       "\n",
       "                day_of_month     month  t_bullet  t_limited  t_northbound  \\\n",
       "favorite_count     -0.013727  0.057577  0.184392   0.019418      0.071410   \n",
       "id                 -0.201225  0.119187 -0.048254  -0.091769      0.070766   \n",
       "retweet_count      -0.058542  0.030757  0.096546   0.100636     -0.041501   \n",
       "day_of_week        -0.063651  0.022254  0.047302  -0.022492     -0.061165   \n",
       "day_of_month        1.000000 -0.204438  0.014090   0.057579      0.053588   \n",
       "month              -0.204438  1.000000 -0.145967   0.041591      0.045134   \n",
       "t_bullet            0.014090 -0.145967  1.000000  -0.416707     -0.010226   \n",
       "t_limited           0.057579  0.041591 -0.416707   1.000000     -0.041419   \n",
       "t_northbound        0.053588  0.045134 -0.010226  -0.041419      1.000000   \n",
       "train_id           -0.007253 -0.039853  0.651110   0.110541      0.008933   \n",
       "tweet_id           -0.201225  0.119187 -0.048254  -0.091769      0.070766   \n",
       "\n",
       "                train_id  tweet_id  \n",
       "favorite_count  0.182412 -0.056542  \n",
       "id             -0.083925  1.000000  \n",
       "retweet_count   0.148645  0.024774  \n",
       "day_of_week     0.132644 -0.083374  \n",
       "day_of_month   -0.007253 -0.201225  \n",
       "month          -0.039853  0.119187  \n",
       "t_bullet        0.651110 -0.048254  \n",
       "t_limited       0.110541 -0.091769  \n",
       "t_northbound    0.008933  0.070766  \n",
       "train_id        1.000000 -0.083925  \n",
       "tweet_id       -0.083925  1.000000  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}