metrics_and_plots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import load_data, highlight_min\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = './data'\n",
    "\n",
    "targets = os.listdir(data_dir)\n",
    "splits = ['cv', 'bac']\n",
    "split_ids = np.arange(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CV Train Size</th>\n",
       "      <th>CV Test Size</th>\n",
       "      <th>BAC Train Size</th>\n",
       "      <th>BAC Test Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CHEMBL214</th>\n",
       "      <td>3129</td>\n",
       "      <td>782</td>\n",
       "      <td>3192</td>\n",
       "      <td>719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL216</th>\n",
       "      <td>1152</td>\n",
       "      <td>288</td>\n",
       "      <td>1073</td>\n",
       "      <td>367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL217</th>\n",
       "      <td>5536</td>\n",
       "      <td>1383</td>\n",
       "      <td>5068</td>\n",
       "      <td>1851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL224</th>\n",
       "      <td>3194</td>\n",
       "      <td>798</td>\n",
       "      <td>2878</td>\n",
       "      <td>1114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL225</th>\n",
       "      <td>2448</td>\n",
       "      <td>612</td>\n",
       "      <td>2374</td>\n",
       "      <td>686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL226</th>\n",
       "      <td>3347</td>\n",
       "      <td>836</td>\n",
       "      <td>3358</td>\n",
       "      <td>825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL251</th>\n",
       "      <td>3920</td>\n",
       "      <td>980</td>\n",
       "      <td>3879</td>\n",
       "      <td>1021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL264</th>\n",
       "      <td>2750</td>\n",
       "      <td>687</td>\n",
       "      <td>2888</td>\n",
       "      <td>549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL3155</th>\n",
       "      <td>1531</td>\n",
       "      <td>382</td>\n",
       "      <td>1632</td>\n",
       "      <td>281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL3371</th>\n",
       "      <td>2400</td>\n",
       "      <td>599</td>\n",
       "      <td>2104</td>\n",
       "      <td>895</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            CV Train Size  CV Test Size  BAC Train Size  BAC Test Size\n",
       "CHEMBL214            3129           782            3192            719\n",
       "CHEMBL216            1152           288            1073            367\n",
       "CHEMBL217            5536          1383            5068           1851\n",
       "CHEMBL224            3194           798            2878           1114\n",
       "CHEMBL225            2448           612            2374            686\n",
       "CHEMBL226            3347           836            3358            825\n",
       "CHEMBL251            3920           980            3879           1021\n",
       "CHEMBL264            2750           687            2888            549\n",
       "CHEMBL3155           1531           382            1632            281\n",
       "CHEMBL3371           2400           599            2104            895"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = defaultdict(dict)\n",
    "\n",
    "for target, split, split_id in product(targets, splits, split_ids):\n",
    "    \n",
    "    train_dataset, test_dataset, sim = load_data(target, split, split_id)\n",
    "    \n",
    "    y_train = train_dataset.y.flatten()\n",
    "    y_test = test_dataset.y.flatten()\n",
    "    \n",
    "    data[target][split.upper() + ' Train Size'] = y_train.shape[0]\n",
    "    data[target][split.upper() + ' Test Size'] = y_test.shape[0]\n",
    "    \n",
    "pd.DataFrame.from_dict(data, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_names = ['_mse', '_drop_mse', '_unc']\n",
    "big_ol_dict_of_everything = {}\n",
    "\n",
    "for target in targets:\n",
    "\n",
    "    target_dict = defaultdict(list)\n",
    "\n",
    "    for split in splits:\n",
    "        for split_id in split_ids:\n",
    "\n",
    "            _, test_dataset, sim = load_data(target, split, split_id)\n",
    "            y_test = test_dataset.y.flatten()\n",
    "\n",
    "            results_path = os.path.join(data_dir, target, split, f\"large_result_{split_id}.npz\")\n",
    "            results = np.load(results_path)\n",
    "\n",
    "            y_pred = results['y_pred'] \n",
    "            y_drop_pred = results['y_drop_pred']\n",
    "            unc = results['unc']\n",
    "\n",
    "            mse = np.mean((y_test - y_pred)**2)\n",
    "            mse_drop = np.mean((y_test - y_drop_pred)**2)\n",
    "\n",
    "            target_dict[split + '_mse'].append(mse)\n",
    "            target_dict[split + '_drop_mse'].append(mse_drop)\n",
    "            target_dict[split + '_unc'].append(unc.mean())\n",
    "        \n",
    "        for score in score_names:\n",
    "        \n",
    "            mean = np.mean(target_dict[split + score])\n",
    "            std = np.std(target_dict[split + score])\n",
    "                \n",
    "            target_dict[split + score] = f\"{mean:.3f} ± {std:.2f}\"\n",
    "\n",
    "    big_ol_dict_of_everything[target] = target_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CV Split Scores\n",
    "MSE - lower is better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow0_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow1_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow2_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow3_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow4_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow5_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow6_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow7_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow8_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow9_col1 {\n",
       "            background-color:  yellow;\n",
       "        }</style>  \n",
       "<table id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5c\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >cv_mse</th> \n",
       "        <th class=\"col_heading level0 col1\" >cv_drop_mse</th> \n",
       "        <th class=\"col_heading level0 col2\" >cv_unc</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row0\" class=\"row_heading level0 row0\" >CHEMBL214</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow0_col0\" class=\"data row0 col0\" >0.491 ± 0.03</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow0_col1\" class=\"data row0 col1\" >0.477 ± 0.03</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow0_col2\" class=\"data row0 col2\" >0.329 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row1\" class=\"row_heading level0 row1\" >CHEMBL216</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow1_col0\" class=\"data row1 col0\" >0.666 ± 0.10</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow1_col1\" class=\"data row1 col1\" >0.657 ± 0.10</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow1_col2\" class=\"data row1 col2\" >0.376 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row2\" class=\"row_heading level0 row2\" >CHEMBL217</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow2_col0\" class=\"data row2 col0\" >0.387 ± 0.01</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow2_col1\" class=\"data row2 col1\" >0.370 ± 0.00</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow2_col2\" class=\"data row2 col2\" >0.313 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row3\" class=\"row_heading level0 row3\" >CHEMBL224</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow3_col0\" class=\"data row3 col0\" >0.488 ± 0.04</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow3_col1\" class=\"data row3 col1\" >0.483 ± 0.03</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow3_col2\" class=\"data row3 col2\" >0.327 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row4\" class=\"row_heading level0 row4\" >CHEMBL225</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow4_col0\" class=\"data row4 col0\" >0.538 ± 0.03</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow4_col1\" class=\"data row4 col1\" >0.522 ± 0.03</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow4_col2\" class=\"data row4 col2\" >0.327 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row5\" class=\"row_heading level0 row5\" >CHEMBL226</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow5_col0\" class=\"data row5 col0\" >0.430 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow5_col1\" class=\"data row5 col1\" >0.423 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow5_col2\" class=\"data row5 col2\" >0.323 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row6\" class=\"row_heading level0 row6\" >CHEMBL251</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow6_col0\" class=\"data row6 col0\" >0.440 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow6_col1\" class=\"data row6 col1\" >0.434 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow6_col2\" class=\"data row6 col2\" >0.346 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row7\" class=\"row_heading level0 row7\" >CHEMBL264</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow7_col0\" class=\"data row7 col0\" >0.439 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow7_col1\" class=\"data row7 col1\" >0.433 ± 0.02</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow7_col2\" class=\"data row7 col2\" >0.305 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row8\" class=\"row_heading level0 row8\" >CHEMBL3155</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow8_col0\" class=\"data row8 col0\" >0.506 ± 0.04</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow8_col1\" class=\"data row8 col1\" >0.498 ± 0.05</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow8_col2\" class=\"data row8 col2\" >0.303 ± 0.01</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5clevel0_row9\" class=\"row_heading level0 row9\" >CHEMBL3371</th> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow9_col0\" class=\"data row9 col0\" >0.512 ± 0.05</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow9_col1\" class=\"data row9 col1\" >0.505 ± 0.04</td> \n",
       "        <td id=\"T_76aac59e_8866_11e9_b854_ac1f6bac1d5crow9_col2\" class=\"data row9 col2\" >0.317 ± 0.01</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fbe4cf12eb8>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names = ['cv' + score for score in score_names]\n",
    "\n",
    "df = pd.DataFrame.from_dict(big_ol_dict_of_everything, orient='index')\n",
    "cv_df = df[col_names]\n",
    "\n",
    "cv_df.style.apply(highlight_min, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llll}\n",
      "\\toprule\n",
      "{} &        cv\\_mse &   cv\\_drop\\_mse &        cv\\_unc \\\\\n",
      "\\midrule\n",
      "CHEMBL214  &  0.491 ± 0.03 &  0.477 ± 0.03 &  0.329 ± 0.01 \\\\\n",
      "CHEMBL216  &  0.666 ± 0.10 &  0.657 ± 0.10 &  0.376 ± 0.01 \\\\\n",
      "CHEMBL217  &  0.387 ± 0.01 &  0.370 ± 0.00 &  0.313 ± 0.01 \\\\\n",
      "CHEMBL224  &  0.488 ± 0.04 &  0.483 ± 0.03 &  0.327 ± 0.01 \\\\\n",
      "CHEMBL225  &  0.538 ± 0.03 &  0.522 ± 0.03 &  0.327 ± 0.01 \\\\\n",
      "CHEMBL226  &  0.430 ± 0.02 &  0.423 ± 0.02 &  0.323 ± 0.01 \\\\\n",
      "CHEMBL251  &  0.440 ± 0.02 &  0.434 ± 0.02 &  0.346 ± 0.01 \\\\\n",
      "CHEMBL264  &  0.439 ± 0.02 &  0.433 ± 0.02 &  0.305 ± 0.01 \\\\\n",
      "CHEMBL3155 &  0.506 ± 0.04 &  0.498 ± 0.05 &  0.303 ± 0.01 \\\\\n",
      "CHEMBL3371 &  0.512 ± 0.05 &  0.505 ± 0.04 &  0.317 ± 0.01 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cv_df.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BAC Split Scores\n",
    "MSE - lower is better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow0_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow1_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow2_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow3_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow4_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow5_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow6_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow7_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow8_col1 {\n",
       "            background-color:  yellow;\n",
       "        }    #T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow9_col1 {\n",
       "            background-color:  yellow;\n",
       "        }</style>  \n",
       "<table id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5c\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >bac_mse</th> \n",
       "        <th class=\"col_heading level0 col1\" >bac_drop_mse</th> \n",
       "        <th class=\"col_heading level0 col2\" >bac_unc</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row0\" class=\"row_heading level0 row0\" >CHEMBL214</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow0_col0\" class=\"data row0 col0\" >1.424 ± 0.18</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow0_col1\" class=\"data row0 col1\" >1.377 ± 0.17</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow0_col2\" class=\"data row0 col2\" >0.370 ± 0.03</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row1\" class=\"row_heading level0 row1\" >CHEMBL216</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow1_col0\" class=\"data row1 col0\" >2.432 ± 0.81</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow1_col1\" class=\"data row1 col1\" >2.380 ± 0.75</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow1_col2\" class=\"data row1 col2\" >0.391 ± 0.04</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row2\" class=\"row_heading level0 row2\" >CHEMBL217</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow2_col0\" class=\"data row2 col0\" >1.067 ± 0.12</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow2_col1\" class=\"data row2 col1\" >1.008 ± 0.13</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow2_col2\" class=\"data row2 col2\" >0.361 ± 0.03</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row3\" class=\"row_heading level0 row3\" >CHEMBL224</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow3_col0\" class=\"data row3 col0\" >1.834 ± 0.32</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow3_col1\" class=\"data row3 col1\" >1.740 ± 0.34</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow3_col2\" class=\"data row3 col2\" >0.432 ± 0.08</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row4\" class=\"row_heading level0 row4\" >CHEMBL225</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow4_col0\" class=\"data row4 col0\" >1.741 ± 0.38</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow4_col1\" class=\"data row4 col1\" >1.684 ± 0.39</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow4_col2\" class=\"data row4 col2\" >0.363 ± 0.02</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row5\" class=\"row_heading level0 row5\" >CHEMBL226</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow5_col0\" class=\"data row5 col0\" >1.487 ± 0.56</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow5_col1\" class=\"data row5 col1\" >1.473 ± 0.57</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow5_col2\" class=\"data row5 col2\" >0.356 ± 0.03</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row6\" class=\"row_heading level0 row6\" >CHEMBL251</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow6_col0\" class=\"data row6 col0\" >1.529 ± 0.43</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow6_col1\" class=\"data row6 col1\" >1.505 ± 0.43</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow6_col2\" class=\"data row6 col2\" >0.370 ± 0.04</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row7\" class=\"row_heading level0 row7\" >CHEMBL264</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow7_col0\" class=\"data row7 col0\" >1.296 ± 0.24</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow7_col1\" class=\"data row7 col1\" >1.235 ± 0.24</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow7_col2\" class=\"data row7 col2\" >0.377 ± 0.03</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row8\" class=\"row_heading level0 row8\" >CHEMBL3155</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow8_col0\" class=\"data row8 col0\" >1.394 ± 0.70</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow8_col1\" class=\"data row8 col1\" >1.391 ± 0.74</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow8_col2\" class=\"data row8 col2\" >0.349 ± 0.03</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5clevel0_row9\" class=\"row_heading level0 row9\" >CHEMBL3371</th> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow9_col0\" class=\"data row9 col0\" >1.502 ± 0.34</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow9_col1\" class=\"data row9 col1\" >1.435 ± 0.33</td> \n",
       "        <td id=\"T_77e0cca6_8866_11e9_b854_ac1f6bac1d5crow9_col2\" class=\"data row9 col2\" >0.344 ± 0.02</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fbdd8809cc0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names = ['bac' + score for score in score_names]\n",
    "bac_df = df[col_names]\n",
    "bac_df.style.apply(highlight_min, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llll}\n",
      "\\toprule\n",
      "{} &       bac\\_mse &  bac\\_drop\\_mse &       bac\\_unc \\\\\n",
      "\\midrule\n",
      "CHEMBL214  &  1.424 ± 0.18 &  1.377 ± 0.17 &  0.370 ± 0.03 \\\\\n",
      "CHEMBL216  &  2.432 ± 0.81 &  2.380 ± 0.75 &  0.391 ± 0.04 \\\\\n",
      "CHEMBL217  &  1.067 ± 0.12 &  1.008 ± 0.13 &  0.361 ± 0.03 \\\\\n",
      "CHEMBL224  &  1.834 ± 0.32 &  1.740 ± 0.34 &  0.432 ± 0.08 \\\\\n",
      "CHEMBL225  &  1.741 ± 0.38 &  1.684 ± 0.39 &  0.363 ± 0.02 \\\\\n",
      "CHEMBL226  &  1.487 ± 0.56 &  1.473 ± 0.57 &  0.356 ± 0.03 \\\\\n",
      "CHEMBL251  &  1.529 ± 0.43 &  1.505 ± 0.43 &  0.370 ± 0.04 \\\\\n",
      "CHEMBL264  &  1.296 ± 0.24 &  1.235 ± 0.24 &  0.377 ± 0.03 \\\\\n",
      "CHEMBL3155 &  1.394 ± 0.70 &  1.391 ± 0.74 &  0.349 ± 0.03 \\\\\n",
      "CHEMBL3371 &  1.502 ± 0.34 &  1.435 ± 0.33 &  0.344 ± 0.02 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(bac_df.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Uncertainties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cv_unc</th>\n",
       "      <th>bac_unc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CHEMBL214</th>\n",
       "      <td>0.329 ± 0.01</td>\n",
       "      <td>0.370 ± 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL216</th>\n",
       "      <td>0.376 ± 0.01</td>\n",
       "      <td>0.391 ± 0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL217</th>\n",
       "      <td>0.313 ± 0.01</td>\n",
       "      <td>0.361 ± 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL224</th>\n",
       "      <td>0.327 ± 0.01</td>\n",
       "      <td>0.432 ± 0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL225</th>\n",
       "      <td>0.327 ± 0.01</td>\n",
       "      <td>0.363 ± 0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL226</th>\n",
       "      <td>0.323 ± 0.01</td>\n",
       "      <td>0.356 ± 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL251</th>\n",
       "      <td>0.346 ± 0.01</td>\n",
       "      <td>0.370 ± 0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL264</th>\n",
       "      <td>0.305 ± 0.01</td>\n",
       "      <td>0.377 ± 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL3155</th>\n",
       "      <td>0.303 ± 0.01</td>\n",
       "      <td>0.349 ± 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHEMBL3371</th>\n",
       "      <td>0.317 ± 0.01</td>\n",
       "      <td>0.344 ± 0.02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  cv_unc       bac_unc\n",
       "CHEMBL214   0.329 ± 0.01  0.370 ± 0.03\n",
       "CHEMBL216   0.376 ± 0.01  0.391 ± 0.04\n",
       "CHEMBL217   0.313 ± 0.01  0.361 ± 0.03\n",
       "CHEMBL224   0.327 ± 0.01  0.432 ± 0.08\n",
       "CHEMBL225   0.327 ± 0.01  0.363 ± 0.02\n",
       "CHEMBL226   0.323 ± 0.01  0.356 ± 0.03\n",
       "CHEMBL251   0.346 ± 0.01  0.370 ± 0.04\n",
       "CHEMBL264   0.305 ± 0.01  0.377 ± 0.03\n",
       "CHEMBL3155  0.303 ± 0.01  0.349 ± 0.03\n",
       "CHEMBL3371  0.317 ± 0.01  0.344 ± 0.02"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unc_names = ['cv_unc', 'bac_unc']\n",
    "df[unc_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scatter Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "for target in targets:\n",
    "\n",
    "    fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(16, 2 * 6))\n",
    "\n",
    "    for ax_row, split in zip(axes, splits):\n",
    "\n",
    "        mses = []\n",
    "        uncs = []\n",
    "        sims = []\n",
    "\n",
    "        for split_id in split_ids:\n",
    "\n",
    "            _, test_dataset, sim = load_data(target, split, split_id)\n",
    "            y_test = test_dataset.y.flatten()\n",
    "\n",
    "            results_path = os.path.join(data_dir, target, split, f\"large_result_{split_id}.npz\")\n",
    "            results = np.load(results_path)\n",
    "\n",
    "            y_pred = results['y_pred'] \n",
    "            y_drop_pred = results['y_drop_pred']\n",
    "\n",
    "            unc = results['unc']\n",
    "            mse = (y_test - y_pred)**2\n",
    "            mse_drop = (y_test - y_drop_pred)**2\n",
    "\n",
    "            mses.append(mse)\n",
    "            uncs.append(unc)\n",
    "            sims.append(sim)\n",
    "\n",
    "        mses = np.concatenate(mses)\n",
    "        uncs = np.concatenate(uncs)\n",
    "        sims = np.concatenate(sims)\n",
    "\n",
    "        ax_1, ax_2, ax_3 = ax_row\n",
    "\n",
    "        ax_1.scatter(sims, mses, s=15, alpha=0.7)\n",
    "        ax_1.set_xlabel('SIMILARITY')\n",
    "        ax_1.set_ylabel('MSE')\n",
    "\n",
    "        ax_2.scatter(sims, uncs, s=15, alpha=0.7)\n",
    "        ax_2.set_xlabel('SIMILARITY')\n",
    "        ax_2.set_ylabel('UNCERTAINTY')\n",
    "\n",
    "        ax_3.scatter(uncs, mses, s=15, alpha=0.7)\n",
    "        ax_3.set_xlabel('UNCERTAINTY')\n",
    "        ax_3.set_ylabel('MSE')\n",
    "\n",
    "        for ax in ax_row:\n",
    "            ax.set_xticks([])\n",
    "            ax.set_yticks([])\n",
    "\n",
    "        fig.tight_layout()\n",
    "\n",
    "    for ax, row in zip(axes[:,0], splits):\n",
    "        ax.annotate(row, xy=(0, 2.), xytext=(-ax.yaxis.labelpad - 5, 0),\n",
    "                    xycoords=ax.yaxis.label, textcoords='offset points',\n",
    "                    size=25, ha='right', va='center')\n",
    "\n",
    "    _ = fig.suptitle(target, fontsize=30, y=1.02)\n",
    "\n",
    "    plt.savefig(fname=f'./img/large_{target}.png', bbox_inches='tight')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:chem]",
   "language": "python",
   "name": "conda-env-chem-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}