sing_job.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file dependencies\n",
    "from tacc_stats.pickler.job_stats import Job\n",
    "import cPickle as pickle\n",
    "import argparse\n",
    "from os import listdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data manipulation dependencies\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle file with job data\n",
    "pickle_file = open('/oasis/projects/nsf/sys200/tcooper/xsede_stats/comet_pickles/2016-08-21/3864417', 'rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open pickle file\n",
    "# extract job\n",
    "jobid = pickle.load(pickle_file)\n",
    "pickle_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# general stats\n",
    "hosts = jobid.hosts\n",
    "start = pd.to_datetime(round(jobid.start_time), unit='s').time()\n",
    "end = pd.to_datetime(round(jobid.end_time), unit='s').time()\n",
    "numCycles = len(jobid.times) + 2\n",
    "stat_types = jobid.hosts['comet-03-09'].stats.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "type_avgs = {}\n",
    "times = [start]\n",
    "devices = []\n",
    "\n",
    "for host_name, host in jobid.hosts.iteritems():\n",
    "    typ='all'\n",
    "    dev='all'\n",
    "    \n",
    "    if 'all' in typ: typ_keys = host.stats.keys()\n",
    "    else: typ_keys = typ\n",
    "        \n",
    "    for time in host.times:\n",
    "        times.append( pd.to_datetime(round(time), unit='s').time() )\n",
    "    \n",
    "    for type_name in typ_keys:\n",
    "        type_data = host.stats[type_name]\n",
    "        type_avgs[type_name] = []\n",
    "        \n",
    "        if 'all' in dev: dev_keys = type_data.keys()\n",
    "        else: dev_keys = dev\n",
    "            \n",
    "        for dev_name in dev_keys:          \n",
    "            devices.append( (dev_name,type_name) )\n",
    "            type_avgs[type_name].append( [np.mean(row).round(decimals=3)\n",
    "                                   for row in type_data[dev_name]] )\n",
    "times.append(end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('wombat-clilov', 'mdc'),\n",
       " ('panda-clilov', 'mdc'),\n",
       " ('-', 'vm'),\n",
       " ('0/09.2', 'intel_hsw_qpi'),\n",
       " ('1/08.2', 'intel_hsw_qpi'),\n",
       " ('0/08.2', 'intel_hsw_qpi'),\n",
       " ('1/09.2', 'intel_hsw_qpi'),\n",
       " ('1/1', 'intel_hsw_cbo'),\n",
       " ('1/0', 'intel_hsw_cbo'),\n",
       " ('1/11', 'intel_hsw_cbo'),\n",
       " ('1/10', 'intel_hsw_cbo'),\n",
       " ('1/7', 'intel_hsw_cbo'),\n",
       " ('1/6', 'intel_hsw_cbo'),\n",
       " ('1/5', 'intel_hsw_cbo'),\n",
       " ('1/4', 'intel_hsw_cbo'),\n",
       " ('1/3', 'intel_hsw_cbo'),\n",
       " ('1/2', 'intel_hsw_cbo'),\n",
       " ('0/8', 'intel_hsw_cbo'),\n",
       " ('0/9', 'intel_hsw_cbo'),\n",
       " ('0/6', 'intel_hsw_cbo'),\n",
       " ('0/7', 'intel_hsw_cbo'),\n",
       " ('0/4', 'intel_hsw_cbo'),\n",
       " ('0/5', 'intel_hsw_cbo'),\n",
       " ('0/2', 'intel_hsw_cbo'),\n",
       " ('0/3', 'intel_hsw_cbo'),\n",
       " ('0/0', 'intel_hsw_cbo'),\n",
       " ('0/1', 'intel_hsw_cbo'),\n",
       " ('0/10', 'intel_hsw_cbo'),\n",
       " ('0/11', 'intel_hsw_cbo'),\n",
       " ('1/9', 'intel_hsw_cbo'),\n",
       " ('1/8', 'intel_hsw_cbo'),\n",
       " ('1', 'intel_hsw_pcu'),\n",
       " ('0', 'intel_hsw_pcu'),\n",
       " ('wombat-clilov', 'osc'),\n",
       " ('panda-clilov', 'osc'),\n",
       " ('mlx4_0.1', 'ib'),\n",
       " ('20', 'intel_hsw'),\n",
       " ('21', 'intel_hsw'),\n",
       " ('22', 'intel_hsw'),\n",
       " ('23', 'intel_hsw'),\n",
       " ('1', 'intel_hsw'),\n",
       " ('0', 'intel_hsw'),\n",
       " ('3', 'intel_hsw'),\n",
       " ('2', 'intel_hsw'),\n",
       " ('5', 'intel_hsw'),\n",
       " ('4', 'intel_hsw'),\n",
       " ('7', 'intel_hsw'),\n",
       " ('6', 'intel_hsw'),\n",
       " ('9', 'intel_hsw'),\n",
       " ('8', 'intel_hsw'),\n",
       " ('11', 'intel_hsw'),\n",
       " ('10', 'intel_hsw'),\n",
       " ('13', 'intel_hsw'),\n",
       " ('12', 'intel_hsw'),\n",
       " ('15', 'intel_hsw'),\n",
       " ('14', 'intel_hsw'),\n",
       " ('17', 'intel_hsw'),\n",
       " ('16', 'intel_hsw'),\n",
       " ('19', 'intel_hsw'),\n",
       " ('18', 'intel_hsw'),\n",
       " ('mlx4_0/1', 'ib_sw'),\n",
       " ('-', 'ps'),\n",
       " ('1/12.5', 'intel_hsw_hau'),\n",
       " ('0/12.1', 'intel_hsw_hau'),\n",
       " ('1/12.1', 'intel_hsw_hau'),\n",
       " ('0/12.5', 'intel_hsw_hau'),\n",
       " ('ib0', 'net'),\n",
       " ('lo', 'net'),\n",
       " ('ib0.8448', 'net'),\n",
       " ('peth0', 'net'),\n",
       " ('eth0', 'net'),\n",
       " ('samtools', 'proc'),\n",
       " ('java', 'proc'),\n",
       " ('bowtie2-align-s', 'proc'),\n",
       " ('python', 'proc'),\n",
       " ('_batch_command.', 'proc'),\n",
       " ('perl', 'proc'),\n",
       " ('beast', 'proc'),\n",
       " ('slurm_script', 'proc'),\n",
       " ('sh', 'proc'),\n",
       " ('srun', 'proc'),\n",
       " ('nscd', 'proc'),\n",
       " ('munged', 'proc'),\n",
       " ('beastwrapper_1.', 'proc'),\n",
       " ('wombat-clilov', 'llite'),\n",
       " ('panda-clilov', 'llite'),\n",
       " ('1', 'mem'),\n",
       " ('0', 'mem'),\n",
       " ('1/17.0', 'intel_hsw_imc'),\n",
       " ('1/17.1', 'intel_hsw_imc'),\n",
       " ('1/14.1', 'intel_hsw_imc'),\n",
       " ('1/14.0', 'intel_hsw_imc'),\n",
       " ('0/14.0', 'intel_hsw_imc'),\n",
       " ('0/14.1', 'intel_hsw_imc'),\n",
       " ('0/17.1', 'intel_hsw_imc'),\n",
       " ('0/17.0', 'intel_hsw_imc'),\n",
       " ('1', 'numa'),\n",
       " ('0', 'numa'),\n",
       " ('/dev/shm', 'tmpfs'),\n",
       " ('-', 'lnet'),\n",
       " ('mlx4_0/1', 'ib_ext'),\n",
       " ('-', 'sysv_shm'),\n",
       " ('/home/abyrd', 'nfs'),\n",
       " ('/home/jroatkul', 'nfs'),\n",
       " ('/home/dantakli', 'nfs'),\n",
       " ('/home/jpg', 'nfs'),\n",
       " ('-', 'vfs'),\n",
       " ('20', 'cpu'),\n",
       " ('21', 'cpu'),\n",
       " ('22', 'cpu'),\n",
       " ('23', 'cpu'),\n",
       " ('1', 'cpu'),\n",
       " ('0', 'cpu'),\n",
       " ('3', 'cpu'),\n",
       " ('2', 'cpu'),\n",
       " ('5', 'cpu'),\n",
       " ('4', 'cpu'),\n",
       " ('7', 'cpu'),\n",
       " ('6', 'cpu'),\n",
       " ('9', 'cpu'),\n",
       " ('8', 'cpu'),\n",
       " ('11', 'cpu'),\n",
       " ('10', 'cpu'),\n",
       " ('13', 'cpu'),\n",
       " ('12', 'cpu'),\n",
       " ('15', 'cpu'),\n",
       " ('14', 'cpu'),\n",
       " ('17', 'cpu'),\n",
       " ('16', 'cpu'),\n",
       " ('19', 'cpu'),\n",
       " ('18', 'cpu'),\n",
       " ('md0', 'block'),\n",
       " ('md1', 'block'),\n",
       " ('sda', 'block'),\n",
       " ('sdb', 'block'),\n",
       " ('0/10.1', 'intel_hsw_r2pci'),\n",
       " ('1/10.1', 'intel_hsw_r2pci')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_stats = { time:{ stat:[] for stat in type_avgs } for time in times }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for type_avg,data in type_avgs.iteritems():\n",
    "    for dev in range( len(data) ):\n",
    "        all_stats[dt.time(21, 59, 44)][type_avg] .append(  data[dev][0]  .round())\n",
    "        all_stats[dt.time(22, 9, 45)] [type_avg] .append(   data[dev][1] .round())   \n",
    "        all_stats[dt.time(22, 19, 45)][type_avg] .append(  data[dev][2]  .round()) \n",
    "        all_stats[dt.time(22, 29, 45)][type_avg] .append(  data[dev][3]  .round())\n",
    "        all_stats[dt.time(22, 39, 45)][type_avg] .append(  data[dev][4]  .round())\n",
    "        all_stats[dt.time(22, 49, 45)][type_avg] .append(  data[dev][5]  .round())   \n",
    "        all_stats[dt.time(0, 38, 3)]  [type_avg] .append(    data[dev][6].round())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Descriptive Statistics Across Devices\n",
    "\n",
    "#### Observing the general pattern of variation between devices for each type of statistic, in support of detecting accuracy of averages used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_stats = { type_stat:{} for type_stat in stat_types}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for time,stat in all_stats.iteritems():\n",
    "    for stat,avgList in stat.iteritems():\n",
    "        dev_stats[stat][\"Average by Device\"] = avgList\n",
    "        dev_stats[stat][\"Overall Average\"] = np.mean(avgList)\n",
    "        dev_stats[stat][\"Standard Deviation\"] = np.std(avgList)\n",
    "        dev_stats[stat][\"High\"] = np.max(avgList)\n",
    "        dev_stats[stat][\"Low\"] = np.min(avgList)\n",
    "        \n",
    "        all_stats[time][stat] = np.mean(avgList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>block</th>\n",
       "      <th>cpu</th>\n",
       "      <th>ib</th>\n",
       "      <th>ib_ext</th>\n",
       "      <th>ib_sw</th>\n",
       "      <th>intel_hsw</th>\n",
       "      <th>intel_hsw_cbo</th>\n",
       "      <th>intel_hsw_hau</th>\n",
       "      <th>intel_hsw_imc</th>\n",
       "      <th>intel_hsw_pcu</th>\n",
       "      <th>...</th>\n",
       "      <th>net</th>\n",
       "      <th>nfs</th>\n",
       "      <th>numa</th>\n",
       "      <th>osc</th>\n",
       "      <th>proc</th>\n",
       "      <th>ps</th>\n",
       "      <th>sysv_shm</th>\n",
       "      <th>tmpfs</th>\n",
       "      <th>vfs</th>\n",
       "      <th>vm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>High</th>\n",
       "      <td>2.08676e+07</td>\n",
       "      <td>42844</td>\n",
       "      <td>0</td>\n",
       "      <td>5.59724e+09</td>\n",
       "      <td>1.39875e+10</td>\n",
       "      <td>2.71434e+12</td>\n",
       "      <td>8.53007e+11</td>\n",
       "      <td>2.53178e+12</td>\n",
       "      <td>7.58326e+11</td>\n",
       "      <td>7.54966e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>1.67843e+09</td>\n",
       "      <td>300529</td>\n",
       "      <td>1.72527e+07</td>\n",
       "      <td>62443</td>\n",
       "      <td>7.3787e+18</td>\n",
       "      <td>6.56631e+06</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>403900</td>\n",
       "      <td>3.52013e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Low</th>\n",
       "      <td>0</td>\n",
       "      <td>42435</td>\n",
       "      <td>0</td>\n",
       "      <td>5.59724e+09</td>\n",
       "      <td>1.39875e+10</td>\n",
       "      <td>2.16492e+09</td>\n",
       "      <td>1.45602e+11</td>\n",
       "      <td>2.25263e+12</td>\n",
       "      <td>6.40926e+11</td>\n",
       "      <td>7.03824e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>1130</td>\n",
       "      <td>0</td>\n",
       "      <td>7.29715e+06</td>\n",
       "      <td>55451</td>\n",
       "      <td>33608</td>\n",
       "      <td>6.56631e+06</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>403900</td>\n",
       "      <td>3.52013e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Overall Average</th>\n",
       "      <td>1.09625e+07</td>\n",
       "      <td>42791.1</td>\n",
       "      <td>0</td>\n",
       "      <td>5.59724e+09</td>\n",
       "      <td>1.39875e+10</td>\n",
       "      <td>1.27788e+12</td>\n",
       "      <td>4.30336e+11</td>\n",
       "      <td>2.39221e+12</td>\n",
       "      <td>6.99462e+11</td>\n",
       "      <td>7.29395e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>4.79776e+08</td>\n",
       "      <td>88660.8</td>\n",
       "      <td>1.22749e+07</td>\n",
       "      <td>58947</td>\n",
       "      <td>6.62191e+17</td>\n",
       "      <td>6.56631e+06</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>403900</td>\n",
       "      <td>3.52013e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Standard Deviation</th>\n",
       "      <td>9.91751e+06</td>\n",
       "      <td>84.5133</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.2481e+11</td>\n",
       "      <td>2.82124e+11</td>\n",
       "      <td>1.39575e+11</td>\n",
       "      <td>5.85019e+10</td>\n",
       "      <td>2.55712e+10</td>\n",
       "      <td>...</td>\n",
       "      <td>6.61004e+08</td>\n",
       "      <td>124227</td>\n",
       "      <td>4.97779e+06</td>\n",
       "      <td>3496</td>\n",
       "      <td>1.9662e+18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          block      cpu ib       ib_ext        ib_sw  \\\n",
       "High                2.08676e+07    42844  0  5.59724e+09  1.39875e+10   \n",
       "Low                           0    42435  0  5.59724e+09  1.39875e+10   \n",
       "Overall Average     1.09625e+07  42791.1  0  5.59724e+09  1.39875e+10   \n",
       "Standard Deviation  9.91751e+06  84.5133  0            0            0   \n",
       "\n",
       "                      intel_hsw intel_hsw_cbo intel_hsw_hau intel_hsw_imc  \\\n",
       "High                2.71434e+12   8.53007e+11   2.53178e+12   7.58326e+11   \n",
       "Low                 2.16492e+09   1.45602e+11   2.25263e+12   6.40926e+11   \n",
       "Overall Average     1.27788e+12   4.30336e+11   2.39221e+12   6.99462e+11   \n",
       "Standard Deviation   9.2481e+11   2.82124e+11   1.39575e+11   5.85019e+10   \n",
       "\n",
       "                   intel_hsw_pcu     ...               net      nfs  \\\n",
       "High                 7.54966e+11     ...       1.67843e+09   300529   \n",
       "Low                  7.03824e+11     ...              1130        0   \n",
       "Overall Average      7.29395e+11     ...       4.79776e+08  88660.8   \n",
       "Standard Deviation   2.55712e+10     ...       6.61004e+08   124227   \n",
       "\n",
       "                           numa    osc         proc           ps sysv_shm  \\\n",
       "High                1.72527e+07  62443   7.3787e+18  6.56631e+06        0   \n",
       "Low                 7.29715e+06  55451        33608  6.56631e+06        0   \n",
       "Overall Average     1.22749e+07  58947  6.62191e+17  6.56631e+06        0   \n",
       "Standard Deviation  4.97779e+06   3496   1.9662e+18            0        0   \n",
       "\n",
       "                   tmpfs     vfs           vm  \n",
       "High                   2  403900  3.52013e+07  \n",
       "Low                    2  403900  3.52013e+07  \n",
       "Overall Average        2  403900  3.52013e+07  \n",
       "Standard Deviation     0       0            0  \n",
       "\n",
       "[4 rows x 26 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc_df = pd.DataFrame( dev_stats ).drop('Average by Device', axis=0 )\n",
    "desc_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Focus on Trends\n",
    "\n",
    "#### Continuation of developing dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{datetime.time(0, 38, 3): {'block': 11241274.25,\n",
       "  'cpu': 43924.708333333336,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 5635720866.0,\n",
       "  'ib_sw': 14083693436.0,\n",
       "  'intel_hsw': 1306175137326.2917,\n",
       "  'intel_hsw_cbo': 441898973602.625,\n",
       "  'intel_hsw_hau': 2455565252230.5,\n",
       "  'intel_hsw_imc': 717965801725.125,\n",
       "  'intel_hsw_pcu': 738496093222.5,\n",
       "  'intel_hsw_qpi': 330643396049.75,\n",
       "  'intel_hsw_r2pci': 1085601473503.5,\n",
       "  'llite': 1165814886.0,\n",
       "  'lnet': 6372874127.0,\n",
       "  'mdc': 54376.0,\n",
       "  'mem': 15753095168.0,\n",
       "  'net': 483060553.39999998,\n",
       "  'nfs': 93288.5,\n",
       "  'numa': 12567728.5,\n",
       "  'osc': 61535.0,\n",
       "  'proc': 1.2297829382476877e+18,\n",
       "  'ps': 6739856.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 404499.0,\n",
       "  'vm': 36115552.0},\n",
       " datetime.time(21, 59, 44): {'block': 0.0,\n",
       "  'cpu': 0.0,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 0.0,\n",
       "  'ib_sw': 0.0,\n",
       "  'intel_hsw': 0.0,\n",
       "  'intel_hsw_cbo': 0.0,\n",
       "  'intel_hsw_hau': 0.0,\n",
       "  'intel_hsw_imc': 0.0,\n",
       "  'intel_hsw_pcu': 0.0,\n",
       "  'intel_hsw_qpi': 0.0,\n",
       "  'intel_hsw_r2pci': 0.0,\n",
       "  'llite': 0.0,\n",
       "  'lnet': 0.0,\n",
       "  'mdc': 0.0,\n",
       "  'mem': 13568753664.0,\n",
       "  'net': 0.0,\n",
       "  'nfs': 0.0,\n",
       "  'numa': 0.0,\n",
       "  'osc': 0.0,\n",
       "  'proc': 372138.23076923075,\n",
       "  'ps': 399.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 415154.0,\n",
       "  'vm': 0.0},\n",
       " datetime.time(22, 9, 45): {'block': 2210393.25,\n",
       "  'cpu': 8560.125,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 742707505.0,\n",
       "  'ib_sw': 1855977587.0,\n",
       "  'intel_hsw': 218464931213.41666,\n",
       "  'intel_hsw_cbo': 83370447611.125,\n",
       "  'intel_hsw_hau': 477973787620.5,\n",
       "  'intel_hsw_imc': 139563227030.125,\n",
       "  'intel_hsw_pcu': 169702123181.5,\n",
       "  'intel_hsw_qpi': 63080780693.5,\n",
       "  'intel_hsw_r2pci': 202126804915.5,\n",
       "  'llite': 105571070.5,\n",
       "  'lnet': 1160953426.0,\n",
       "  'mdc': 10366.5,\n",
       "  'mem': 14044895744.0,\n",
       "  'net': 63618267.799999997,\n",
       "  'nfs': 17621.0,\n",
       "  'numa': 1938144.0,\n",
       "  'osc': 12965.5,\n",
       "  'proc': 5.6759212534519648e+17,\n",
       "  'ps': 1392432.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 415400.0,\n",
       "  'vm': 7273413.0},\n",
       " datetime.time(22, 19, 45): {'block': 4363898.5,\n",
       "  'cpu': 17116.625,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 2124022459.0,\n",
       "  'ib_sw': 5307860778.0,\n",
       "  'intel_hsw': 500188376227.70831,\n",
       "  'intel_hsw_cbo': 168085566204.375,\n",
       "  'intel_hsw_hau': 956469797884.75,\n",
       "  'intel_hsw_imc': 279605833887.25,\n",
       "  'intel_hsw_pcu': 306375352309.0,\n",
       "  'intel_hsw_qpi': 127011499988.5,\n",
       "  'intel_hsw_r2pci': 417618957320.5,\n",
       "  'llite': 316471178.0,\n",
       "  'lnet': 2382550442.0,\n",
       "  'mdc': 21118.0,\n",
       "  'mem': 14916957056.0,\n",
       "  'net': 182035070.80000001,\n",
       "  'nfs': 36200.5,\n",
       "  'numa': 4001166.0,\n",
       "  'osc': 23145.5,\n",
       "  'proc': 5.6759212534519667e+17,\n",
       "  'ps': 2661077.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 415301.0,\n",
       "  'vm': 13441317.0},\n",
       " datetime.time(22, 29, 45): {'block': 6568380.5,\n",
       "  'cpu': 25673.416666666668,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 3146897070.0,\n",
       "  'ib_sw': 7863993849.0,\n",
       "  'intel_hsw': 741826304272.375,\n",
       "  'intel_hsw_cbo': 254059197808.20834,\n",
       "  'intel_hsw_hau': 1434754423387.75,\n",
       "  'intel_hsw_imc': 419394725508.75,\n",
       "  'intel_hsw_pcu': 459015692748.5,\n",
       "  'intel_hsw_qpi': 191017636417.25,\n",
       "  'intel_hsw_r2pci': 626977604040.0,\n",
       "  'llite': 425457629.0,\n",
       "  'lnet': 3031516248.0,\n",
       "  'mdc': 30938.5,\n",
       "  'mem': 15796083840.0,\n",
       "  'net': 269732900.39999998,\n",
       "  'nfs': 49999.0,\n",
       "  'numa': 6570716.5,\n",
       "  'osc': 33587.0,\n",
       "  'proc': 5.6759212534538298e+17,\n",
       "  'ps': 3963527.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 415325.0,\n",
       "  'vm': 20499563.0},\n",
       " datetime.time(22, 39, 45): {'block': 8689852.0,\n",
       "  'cpu': 34230.625,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 4337567698.0,\n",
       "  'ib_sw': 10839530636.0,\n",
       "  'intel_hsw': 1046498018213.1666,\n",
       "  'intel_hsw_cbo': 344221897673.33331,\n",
       "  'intel_hsw_hau': 1913865926005.25,\n",
       "  'intel_hsw_imc': 559727318460.625,\n",
       "  'intel_hsw_pcu': 571077866255.0,\n",
       "  'intel_hsw_qpi': 257503632674.75,\n",
       "  'intel_hsw_r2pci': 853100400994.5,\n",
       "  'llite': 640862942.0,\n",
       "  'lnet': 4311930214.0,\n",
       "  'mdc': 41282.0,\n",
       "  'mem': 15956769664.0,\n",
       "  'net': 371739373.19999999,\n",
       "  'nfs': 64791.75,\n",
       "  'numa': 9634908.0,\n",
       "  'osc': 46947.5,\n",
       "  'proc': 6.6219081290270298e+17,\n",
       "  'ps': 5209279.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 410832.0,\n",
       "  'vm': 27739777.0},\n",
       " datetime.time(22, 49, 45): {'block': 10962533.75,\n",
       "  'cpu': 42791.083333333336,\n",
       "  'ib': 0.0,\n",
       "  'ib_ext': 5597235685.0,\n",
       "  'ib_sw': 13987514012.0,\n",
       "  'intel_hsw': 1277884379514.125,\n",
       "  'intel_hsw_cbo': 430336082202.25,\n",
       "  'intel_hsw_hau': 2392207091677.0,\n",
       "  'intel_hsw_imc': 699462072009.0,\n",
       "  'intel_hsw_pcu': 729394695997.0,\n",
       "  'intel_hsw_qpi': 322039535040.0,\n",
       "  'intel_hsw_r2pci': 1058971704510.0,\n",
       "  'llite': 1053455387.5,\n",
       "  'lnet': 6310672441.0,\n",
       "  'mdc': 52620.0,\n",
       "  'mem': 16010846080.0,\n",
       "  'net': 479776010.39999998,\n",
       "  'nfs': 88660.75,\n",
       "  'numa': 12274940.0,\n",
       "  'osc': 58947.0,\n",
       "  'proc': 6.6219081290278694e+17,\n",
       "  'ps': 6566311.0,\n",
       "  'sysv_shm': 0.0,\n",
       "  'tmpfs': 2.0,\n",
       "  'vfs': 403900.0,\n",
       "  'vm': 35201308.0}}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>block</th>\n",
       "      <th>cpu</th>\n",
       "      <th>ib</th>\n",
       "      <th>ib_ext</th>\n",
       "      <th>ib_sw</th>\n",
       "      <th>intel_hsw</th>\n",
       "      <th>intel_hsw_cbo</th>\n",
       "      <th>intel_hsw_hau</th>\n",
       "      <th>intel_hsw_imc</th>\n",
       "      <th>intel_hsw_pcu</th>\n",
       "      <th>...</th>\n",
       "      <th>net</th>\n",
       "      <th>nfs</th>\n",
       "      <th>numa</th>\n",
       "      <th>osc</th>\n",
       "      <th>proc</th>\n",
       "      <th>ps</th>\n",
       "      <th>sysv_shm</th>\n",
       "      <th>tmpfs</th>\n",
       "      <th>vfs</th>\n",
       "      <th>vm</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cycle</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>First</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.721382e+05</td>\n",
       "      <td>399</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>415154</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Second</th>\n",
       "      <td>2210393.25</td>\n",
       "      <td>8560.125000</td>\n",
       "      <td>0</td>\n",
       "      <td>742707505</td>\n",
       "      <td>1855977587</td>\n",
       "      <td>2.184649e+11</td>\n",
       "      <td>8.337045e+10</td>\n",
       "      <td>4.779738e+11</td>\n",
       "      <td>1.395632e+11</td>\n",
       "      <td>1.697021e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>63618267.8</td>\n",
       "      <td>17621.00</td>\n",
       "      <td>1938144.0</td>\n",
       "      <td>12965.5</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>1392432</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>415400</td>\n",
       "      <td>7273413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Third</th>\n",
       "      <td>4363898.50</td>\n",
       "      <td>17116.625000</td>\n",
       "      <td>0</td>\n",
       "      <td>2124022459</td>\n",
       "      <td>5307860778</td>\n",
       "      <td>5.001884e+11</td>\n",
       "      <td>1.680856e+11</td>\n",
       "      <td>9.564698e+11</td>\n",
       "      <td>2.796058e+11</td>\n",
       "      <td>3.063754e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>182035070.8</td>\n",
       "      <td>36200.50</td>\n",
       "      <td>4001166.0</td>\n",
       "      <td>23145.5</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>2661077</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>415301</td>\n",
       "      <td>13441317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fourth</th>\n",
       "      <td>6568380.50</td>\n",
       "      <td>25673.416667</td>\n",
       "      <td>0</td>\n",
       "      <td>3146897070</td>\n",
       "      <td>7863993849</td>\n",
       "      <td>7.418263e+11</td>\n",
       "      <td>2.540592e+11</td>\n",
       "      <td>1.434754e+12</td>\n",
       "      <td>4.193947e+11</td>\n",
       "      <td>4.590157e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>269732900.4</td>\n",
       "      <td>49999.00</td>\n",
       "      <td>6570716.5</td>\n",
       "      <td>33587.0</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>3963527</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>415325</td>\n",
       "      <td>20499563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fifth</th>\n",
       "      <td>8689852.00</td>\n",
       "      <td>34230.625000</td>\n",
       "      <td>0</td>\n",
       "      <td>4337567698</td>\n",
       "      <td>10839530636</td>\n",
       "      <td>1.046498e+12</td>\n",
       "      <td>3.442219e+11</td>\n",
       "      <td>1.913866e+12</td>\n",
       "      <td>5.597273e+11</td>\n",
       "      <td>5.710779e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>371739373.2</td>\n",
       "      <td>64791.75</td>\n",
       "      <td>9634908.0</td>\n",
       "      <td>46947.5</td>\n",
       "      <td>6.621908e+17</td>\n",
       "      <td>5209279</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>410832</td>\n",
       "      <td>27739777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sixth</th>\n",
       "      <td>10962533.75</td>\n",
       "      <td>42791.083333</td>\n",
       "      <td>0</td>\n",
       "      <td>5597235685</td>\n",
       "      <td>13987514012</td>\n",
       "      <td>1.277884e+12</td>\n",
       "      <td>4.303361e+11</td>\n",
       "      <td>2.392207e+12</td>\n",
       "      <td>6.994621e+11</td>\n",
       "      <td>7.293947e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>479776010.4</td>\n",
       "      <td>88660.75</td>\n",
       "      <td>12274940.0</td>\n",
       "      <td>58947.0</td>\n",
       "      <td>6.621908e+17</td>\n",
       "      <td>6566311</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>403900</td>\n",
       "      <td>35201308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Seventh</th>\n",
       "      <td>11241274.25</td>\n",
       "      <td>43924.708333</td>\n",
       "      <td>0</td>\n",
       "      <td>5635720866</td>\n",
       "      <td>14083693436</td>\n",
       "      <td>1.306175e+12</td>\n",
       "      <td>4.418990e+11</td>\n",
       "      <td>2.455565e+12</td>\n",
       "      <td>7.179658e+11</td>\n",
       "      <td>7.384961e+11</td>\n",
       "      <td>...</td>\n",
       "      <td>483060553.4</td>\n",
       "      <td>93288.50</td>\n",
       "      <td>12567728.5</td>\n",
       "      <td>61535.0</td>\n",
       "      <td>1.229783e+18</td>\n",
       "      <td>6739856</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>404499</td>\n",
       "      <td>36115552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               block           cpu  ib      ib_ext        ib_sw     intel_hsw  \\\n",
       "Cycle                                                                           \n",
       "First           0.00      0.000000   0           0            0  0.000000e+00   \n",
       "Second    2210393.25   8560.125000   0   742707505   1855977587  2.184649e+11   \n",
       "Third     4363898.50  17116.625000   0  2124022459   5307860778  5.001884e+11   \n",
       "Fourth    6568380.50  25673.416667   0  3146897070   7863993849  7.418263e+11   \n",
       "Fifth     8689852.00  34230.625000   0  4337567698  10839530636  1.046498e+12   \n",
       "Sixth    10962533.75  42791.083333   0  5597235685  13987514012  1.277884e+12   \n",
       "Seventh  11241274.25  43924.708333   0  5635720866  14083693436  1.306175e+12   \n",
       "\n",
       "         intel_hsw_cbo  intel_hsw_hau  intel_hsw_imc  intel_hsw_pcu    ...     \\\n",
       "Cycle                                                                  ...      \n",
       "First     0.000000e+00   0.000000e+00   0.000000e+00   0.000000e+00    ...      \n",
       "Second    8.337045e+10   4.779738e+11   1.395632e+11   1.697021e+11    ...      \n",
       "Third     1.680856e+11   9.564698e+11   2.796058e+11   3.063754e+11    ...      \n",
       "Fourth    2.540592e+11   1.434754e+12   4.193947e+11   4.590157e+11    ...      \n",
       "Fifth     3.442219e+11   1.913866e+12   5.597273e+11   5.710779e+11    ...      \n",
       "Sixth     4.303361e+11   2.392207e+12   6.994621e+11   7.293947e+11    ...      \n",
       "Seventh   4.418990e+11   2.455565e+12   7.179658e+11   7.384961e+11    ...      \n",
       "\n",
       "                 net       nfs        numa      osc          proc       ps  \\\n",
       "Cycle                                                                        \n",
       "First            0.0      0.00         0.0      0.0  3.721382e+05      399   \n",
       "Second    63618267.8  17621.00   1938144.0  12965.5  5.675921e+17  1392432   \n",
       "Third    182035070.8  36200.50   4001166.0  23145.5  5.675921e+17  2661077   \n",
       "Fourth   269732900.4  49999.00   6570716.5  33587.0  5.675921e+17  3963527   \n",
       "Fifth    371739373.2  64791.75   9634908.0  46947.5  6.621908e+17  5209279   \n",
       "Sixth    479776010.4  88660.75  12274940.0  58947.0  6.621908e+17  6566311   \n",
       "Seventh  483060553.4  93288.50  12567728.5  61535.0  1.229783e+18  6739856   \n",
       "\n",
       "         sysv_shm  tmpfs     vfs        vm  \n",
       "Cycle                                       \n",
       "First           0      2  415154         0  \n",
       "Second          0      2  415400   7273413  \n",
       "Third           0      2  415301  13441317  \n",
       "Fourth          0      2  415325  20499563  \n",
       "Fifth           0      2  410832  27739777  \n",
       "Sixth           0      2  403900  35201308  \n",
       "Seventh         0      2  404499  36115552  \n",
       "\n",
       "[7 rows x 26 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set indices\n",
    "indexList = [\"First\", \"Second\", \"Third\", \"Fourth\", \"Fifth\", \"Sixth\", \"Seventh\"]\n",
    "\n",
    "# index by time of check\n",
    "df_1 = pd.DataFrame( all_stats[dt.time(21, 59, 44)], index=[\"First\"] )#.set_index(indexList)\n",
    "df_2 = pd.DataFrame( all_stats[dt.time(22, 9, 45)],  index=[\"Second\"] ) #.set_index(indexList)\n",
    "df_3 = pd.DataFrame( all_stats[dt.time(22, 19, 45)], index=[\"Third\"] )#.set_index(indexList)\n",
    "df_4 = pd.DataFrame( all_stats[dt.time(22, 29, 45)], index=[\"Fourth\"] )#.set_index(indexList)\n",
    "df_5 = pd.DataFrame( all_stats[dt.time(22, 39, 45)], index=[\"Fifth\"] )#.set_index(indexList)\n",
    "df_6 = pd.DataFrame( all_stats[dt.time(22, 49, 45)], index=[\"Sixth\"] )#.set_index(indexList)\n",
    "df_7 = pd.DataFrame( all_stats[dt.time(0, 38, 3)],   index=[\"Seventh\"] )  #.set_index(indexList)\n",
    "\n",
    "# merge dataframes\n",
    "df_time = pd.concat([df_1, df_2, df_3, df_4, df_5, df_6, df_7])\n",
    "df_time.index.name = \"Cycle\"\n",
    "df_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_time = df_time[[\"intel_hsw\",\"ib\",\"llite\",\n",
    "         # \"ib_sw\",\"ib_ext\", \"mdc\",\"osc\",\n",
    "         \"lnet\", \"block\",\"cpu\",\"mem\", \"net\",\n",
    "         \"nfs\",\"numa\",\"proc\",\"sysv_shm\",\"tmpfs\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select and rename stats\n",
    "df_time = df_time.rename( columns={\"index\": \"Job Name\",\n",
    "                         \"intel_hsw\": \"Intel Haswell Processor (per core)\",\n",
    "                         \"ib\": \"Infiniband Usage\",\n",
    "                         #\"ib_sw\": \"Infiniband Usage\",\n",
    "                         #\"ib_ext\": \"Infiniband Usage\",\n",
    "                         \"llite\": \"Lustre Filesystem Usage (per mount)\",\n",
    "                         \"lnet\": \"Lustre Network Usage\",\n",
    "                         #\"mdc\": \"Lustre Network Usage\",\n",
    "                         #\"osc\": \"Lustre Filesystem Usage\",\n",
    "                         \"block\": \"Block Device Statistics (per device)\",\n",
    "                         \"cpu\": \"Scheduler Accounting (per CPU)\",\n",
    "                         \"mem\": \"Memory Usage\",\n",
    "                         \"net\": \"Network Device Usage (per device)\",\n",
    "                         \"nfs\": \"NFS System Usage\",\n",
    "                         \"numa\": \"NUMA Statistics (per socket)\",\n",
    "                         \"proc\": \"Process Specific Data\",\n",
    "                         \"sysv_shm\": \"System Shared Memory Segment Usage\",\n",
    "                         \"tmpfs\": \"RAM-Backed Filesystem Usage (per mount)\"\n",
    "                        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reorder columns\n",
    "df_time = df_time.reindex(columns=[\n",
    "                           \"Block Device Statistics (per device)\",\n",
    "                           \"Intel Haswell Processor (per core)\",\n",
    "                           \"Infiniband Usage\",\n",
    "                           #\"Infiniband Usage\",\n",
    "                           #\"Infiniband Usage\",\n",
    "                           \"Lustre Filesystem Usage (per mount)\",\n",
    "                           \"Lustre Network Usage\",\n",
    "                           #\"Lustre Network Usage\",\n",
    "                           #\"Lustre Filesystem Usage\",\n",
    "                           \"Memory Usage\",\n",
    "                           \"Network Device Usage (per device)\",\n",
    "                           \"NFS System Usage\",\n",
    "                           \"NUMA Statistics (per socket)\",\n",
    "                           \"Process Specific Data\",\n",
    "                           \"RAM-Backed Filesystem Usage (per mount)\",\n",
    "                           \"Scheduler Accounting (per CPU)\",\n",
    "                           \"System Shared Memory Segment Usage\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Block Device Statistics (per device)</th>\n",
       "      <th>Intel Haswell Processor (per core)</th>\n",
       "      <th>Infiniband Usage</th>\n",
       "      <th>Lustre Filesystem Usage (per mount)</th>\n",
       "      <th>Lustre Network Usage</th>\n",
       "      <th>Memory Usage</th>\n",
       "      <th>Network Device Usage (per device)</th>\n",
       "      <th>NFS System Usage</th>\n",
       "      <th>NUMA Statistics (per socket)</th>\n",
       "      <th>Process Specific Data</th>\n",
       "      <th>RAM-Backed Filesystem Usage (per mount)</th>\n",
       "      <th>Scheduler Accounting (per CPU)</th>\n",
       "      <th>System Shared Memory Segment Usage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cycle</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>First</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>13568753664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.721382e+05</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Second</th>\n",
       "      <td>2210393.25</td>\n",
       "      <td>2.184649e+11</td>\n",
       "      <td>0</td>\n",
       "      <td>1.055711e+08</td>\n",
       "      <td>1160953426</td>\n",
       "      <td>14044895744</td>\n",
       "      <td>63618267.8</td>\n",
       "      <td>17621.00</td>\n",
       "      <td>1938144.0</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>2</td>\n",
       "      <td>8560.125000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Third</th>\n",
       "      <td>4363898.50</td>\n",
       "      <td>5.001884e+11</td>\n",
       "      <td>0</td>\n",
       "      <td>3.164712e+08</td>\n",
       "      <td>2382550442</td>\n",
       "      <td>14916957056</td>\n",
       "      <td>182035070.8</td>\n",
       "      <td>36200.50</td>\n",
       "      <td>4001166.0</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>2</td>\n",
       "      <td>17116.625000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fourth</th>\n",
       "      <td>6568380.50</td>\n",
       "      <td>7.418263e+11</td>\n",
       "      <td>0</td>\n",
       "      <td>4.254576e+08</td>\n",
       "      <td>3031516248</td>\n",
       "      <td>15796083840</td>\n",
       "      <td>269732900.4</td>\n",
       "      <td>49999.00</td>\n",
       "      <td>6570716.5</td>\n",
       "      <td>5.675921e+17</td>\n",
       "      <td>2</td>\n",
       "      <td>25673.416667</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fifth</th>\n",
       "      <td>8689852.00</td>\n",
       "      <td>1.046498e+12</td>\n",
       "      <td>0</td>\n",
       "      <td>6.408629e+08</td>\n",
       "      <td>4311930214</td>\n",
       "      <td>15956769664</td>\n",
       "      <td>371739373.2</td>\n",
       "      <td>64791.75</td>\n",
       "      <td>9634908.0</td>\n",
       "      <td>6.621908e+17</td>\n",
       "      <td>2</td>\n",
       "      <td>34230.625000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sixth</th>\n",
       "      <td>10962533.75</td>\n",
       "      <td>1.277884e+12</td>\n",
       "      <td>0</td>\n",
       "      <td>1.053455e+09</td>\n",
       "      <td>6310672441</td>\n",
       "      <td>16010846080</td>\n",
       "      <td>479776010.4</td>\n",
       "      <td>88660.75</td>\n",
       "      <td>12274940.0</td>\n",
       "      <td>6.621908e+17</td>\n",
       "      <td>2</td>\n",
       "      <td>42791.083333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Seventh</th>\n",
       "      <td>11241274.25</td>\n",
       "      <td>1.306175e+12</td>\n",
       "      <td>0</td>\n",
       "      <td>1.165815e+09</td>\n",
       "      <td>6372874127</td>\n",
       "      <td>15753095168</td>\n",
       "      <td>483060553.4</td>\n",
       "      <td>93288.50</td>\n",
       "      <td>12567728.5</td>\n",
       "      <td>1.229783e+18</td>\n",
       "      <td>2</td>\n",
       "      <td>43924.708333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Block Device Statistics (per device)  \\\n",
       "Cycle                                           \n",
       "First                                    0.00   \n",
       "Second                             2210393.25   \n",
       "Third                              4363898.50   \n",
       "Fourth                             6568380.50   \n",
       "Fifth                              8689852.00   \n",
       "Sixth                             10962533.75   \n",
       "Seventh                           11241274.25   \n",
       "\n",
       "         Intel Haswell Processor (per core)  Infiniband Usage  \\\n",
       "Cycle                                                           \n",
       "First                          0.000000e+00                 0   \n",
       "Second                         2.184649e+11                 0   \n",
       "Third                          5.001884e+11                 0   \n",
       "Fourth                         7.418263e+11                 0   \n",
       "Fifth                          1.046498e+12                 0   \n",
       "Sixth                          1.277884e+12                 0   \n",
       "Seventh                        1.306175e+12                 0   \n",
       "\n",
       "         Lustre Filesystem Usage (per mount)  Lustre Network Usage  \\\n",
       "Cycle                                                                \n",
       "First                           0.000000e+00                     0   \n",
       "Second                          1.055711e+08            1160953426   \n",
       "Third                           3.164712e+08            2382550442   \n",
       "Fourth                          4.254576e+08            3031516248   \n",
       "Fifth                           6.408629e+08            4311930214   \n",
       "Sixth                           1.053455e+09            6310672441   \n",
       "Seventh                         1.165815e+09            6372874127   \n",
       "\n",
       "         Memory Usage  Network Device Usage (per device)  NFS System Usage  \\\n",
       "Cycle                                                                        \n",
       "First     13568753664                                0.0              0.00   \n",
       "Second    14044895744                         63618267.8          17621.00   \n",
       "Third     14916957056                        182035070.8          36200.50   \n",
       "Fourth    15796083840                        269732900.4          49999.00   \n",
       "Fifth     15956769664                        371739373.2          64791.75   \n",
       "Sixth     16010846080                        479776010.4          88660.75   \n",
       "Seventh   15753095168                        483060553.4          93288.50   \n",
       "\n",
       "         NUMA Statistics (per socket)  Process Specific Data  \\\n",
       "Cycle                                                          \n",
       "First                             0.0           3.721382e+05   \n",
       "Second                      1938144.0           5.675921e+17   \n",
       "Third                       4001166.0           5.675921e+17   \n",
       "Fourth                      6570716.5           5.675921e+17   \n",
       "Fifth                       9634908.0           6.621908e+17   \n",
       "Sixth                      12274940.0           6.621908e+17   \n",
       "Seventh                    12567728.5           1.229783e+18   \n",
       "\n",
       "         RAM-Backed Filesystem Usage (per mount)  \\\n",
       "Cycle                                              \n",
       "First                                          2   \n",
       "Second                                         2   \n",
       "Third                                          2   \n",
       "Fourth                                         2   \n",
       "Fifth                                          2   \n",
       "Sixth                                          2   \n",
       "Seventh                                        2   \n",
       "\n",
       "         Scheduler Accounting (per CPU)  System Shared Memory Segment Usage  \n",
       "Cycle                                                                        \n",
       "First                          0.000000                                   0  \n",
       "Second                      8560.125000                                   0  \n",
       "Third                      17116.625000                                   0  \n",
       "Fourth                     25673.416667                                   0  \n",
       "Fifth                      34230.625000                                   0  \n",
       "Sixth                      42791.083333                                   0  \n",
       "Seventh                    43924.708333                                   0  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_time"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}