07_deep_learning_end_to_end.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "15015028",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T13:07:52.163408Z",
     "start_time": "2022-09-04T13:07:52.159554Z"
    }
   },
   "source": [
    "# Alternative 3 - End-to-end deep learning on raw features\n",
    "\n",
    "<img src=\"images/feature_set_summary.jpg\" width=\"1000\" height=\"1000\">\n",
    "\n",
    "Get the pre-computed dataset from: https://www.dropbox.com/scl/fo/5llpuwwtcuo22p9jnfxuo/h?dl=0&rlkey=nm0kqrfbk3z9s8qns8hjh4437"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3195be58",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:11:08.965057Z",
     "start_time": "2022-09-04T19:11:06.995541Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "\n",
    "def reset_seed():\n",
    "    tf.random.set_seed(0)\n",
    "    np.random.seed(0)\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "def classification_results(y, yhat):\n",
    "    acc = metrics.accuracy_score(y, yhat)\n",
    "    mcc = metrics.matthews_corrcoef(y, yhat)\n",
    "    f1_weighted = metrics.f1_score(y, yhat, average=\"weighted\")\n",
    "    return acc, mcc, f1_weighted\n",
    "\n",
    "\n",
    "reset_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d12f86a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:11:09.474798Z",
     "start_time": "2022-09-04T19:11:08.966535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>id</th>\n",
       "      <th>sort</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>act_0</th>\n",
       "      <th>act_1</th>\n",
       "      <th>act_2</th>\n",
       "      <th>act_3</th>\n",
       "      <th>act_4</th>\n",
       "      <th>...</th>\n",
       "      <th>hr_1</th>\n",
       "      <th>hr_2</th>\n",
       "      <th>hr_3</th>\n",
       "      <th>hr_4</th>\n",
       "      <th>hr_5</th>\n",
       "      <th>hr_6</th>\n",
       "      <th>hr_7</th>\n",
       "      <th>hr_8</th>\n",
       "      <th>hr_9</th>\n",
       "      <th>hr_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>(0, 2)</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>(0, 3)</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>(0, 4)</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pid  time      id  sort  sleep_phase  act_0  act_1  act_2  act_3  act_4  \\\n",
       "0    0    29  (0, 0)     0          0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "1    0    59  (0, 1)     1          0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "2    0    89  (0, 2)     2          0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "3    0   119  (0, 3)     3          0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "4    0   149  (0, 4)     4          0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "\n",
       "   ...  hr_1  hr_2  hr_3  hr_4  hr_5  hr_6  hr_7  hr_8  hr_9  hr_10  \n",
       "0  ...   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   71.0  \n",
       "1  ...   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  71.0   76.0  \n",
       "2  ...   0.0   0.0   0.0   0.0   0.0   0.0   0.0  71.0  76.0   78.0  \n",
       "3  ...   0.0   0.0   0.0   0.0   0.0   0.0  71.0  76.0  78.0   73.0  \n",
       "4  ...   0.0   0.0   0.0   0.0   0.0  71.0  76.0  78.0  73.0   80.0  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./datasets/df_raw_features.tar.gz\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "335241b4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:11:22.591575Z",
     "start_time": "2022-09-04T19:11:22.583376Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 5., 3., 4.])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"sleep_phase\"].unique()\n",
    "\n",
    "# Type of sleep statging problems:\n",
    "# -------------------------------\n",
    "#\n",
    "#      5-class | 4-class | 3-class | 2-Class\n",
    "# 0 -> Wake    | Wake    | Wake    | Wake\n",
    "# 1 -> N1      | Light   | NREM    | Sleep\n",
    "# 2 -> N2      | Light   | NREM    | Sleep\n",
    "# 3 -> N3      | Deep    | NREM    | Sleep\n",
    "# 4 -> N4      | Deep    | NREM    | Sleep\n",
    "# 5 -> REM     | REM     | REM     | Sleep\n",
    "#\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6a62d658",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:11:23.021644Z",
     "start_time": "2022-09-04T19:11:23.010509Z"
    }
   },
   "outputs": [],
   "source": [
    "df[\"sleep\"] = df[\"sleep_phase\"] > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92e7e8f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:43:18.172150Z",
     "start_time": "2022-09-04T19:43:18.145136Z"
    }
   },
   "outputs": [],
   "source": [
    "MASKING_VALUE = -1000\n",
    "\n",
    "def generate_XY(df, maxdim, ycol=\"sleep\"):\n",
    "\n",
    "    hr_cols = dict([(int(k.split(\"_\")[1]), k) for k in df.keys() if k.startswith(\"hr_\")])\n",
    "    last_hr_key = sorted(hr_cols.items(), key=lambda item: item[0])[-1][1]\n",
    "\n",
    "    act_cols = dict([(int(k.split(\"_\")[1]), k) for k in df.keys() if k.startswith(\"act_\")])\n",
    "    last_act_key = sorted(act_cols.items(), key=lambda item: item[0])[-1][1]\n",
    "\n",
    "    hr = df[last_hr_key].values\n",
    "    act = df[last_act_key].values\n",
    "    \n",
    "    Y  = df[ycol].astype(int).values.reshape(-1, 1)\n",
    "    X = np.stack((act,hr))\n",
    "    X = X.transpose(1,0)\n",
    "    \n",
    "    if maxdim > X.shape[0]:\n",
    "        # Pad sequences (e.g., to the maxium length in the sequence or a constant like 8-10 hours)\n",
    "        X = np.pad(X, ((0, maxdim-X.shape[0]), (0,0)), \"constant\", constant_values=MASKING_VALUE)\n",
    "        Y = np.pad(Y, ((0, maxdim-Y.shape[0]), (0,0)), \"constant\", constant_values=MASKING_VALUE)\n",
    "    else:\n",
    "        # Crop the sequence at some maxium length\n",
    "        X = X[:maxdim]\n",
    "        Y = Y[:maxdim]\n",
    "        \n",
    "    # Expand dims\n",
    "    X = np.expand_dims(X, axis=0)\n",
    "    Y = np.expand_dims(Y, axis=0)\n",
    "    \n",
    "    X = X.astype(float)\n",
    "    Y = Y.astype(int)\n",
    "    \n",
    "    return X, Y\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2bcc8a19",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T19:43:18.541341Z",
     "start_time": "2022-09-04T19:43:18.525625Z"
    }
   },
   "outputs": [],
   "source": [
    "X, Y = generate_XY(df[df[\"pid\"]==1], maxdim=800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "790a18c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 800, 2)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "094c0e8b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:11.980993Z",
     "start_time": "2022-09-04T20:06:11.967205Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The largest dimension is 1615\n",
      "Quantile 0.8 1153\n"
     ]
    }
   ],
   "source": [
    "max_dimension = df.groupby(\"pid\").size().max()\n",
    "print(\"The largest dimension is %d\" % (max_dimension))\n",
    "\n",
    "quantile08 = int(df.groupby(\"pid\").size().quantile(0.8))\n",
    "print(\"Quantile 0.8 %d\" % (quantile08))\n",
    "\n",
    "MAX_DIMENSION = max_dimension\n",
    "# MAX_DIMENSION = 1200 # 1200 = 10 hours\n",
    "# MAX_DIMENSION = 400 # 1200 = 10 hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e4b0427e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:12.719733Z",
     "start_time": "2022-09-04T20:06:12.525421Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pid\n",
       "0    ([[[ 2. 71.], [ 0. 76.], [ 1. 78.], [ 2. 73.],...\n",
       "1    ([[[ 1. 66.], [ 0. 67.], [ 0. 66.], [ 1. 67.],...\n",
       "2    ([[[12. 77.], [27. 62.], [ 1. 64.], [ 0. 64.],...\n",
       "3    ([[[ 1. 73.], [ 0. 69.], [ 0. 70.], [ 0. 70.],...\n",
       "4    ([[[ 0. 51.], [ 1. 49.], [ 1. 53.], [ 1. 50.],...\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_XY = df.groupby(\"pid\").apply(lambda x: generate_XY(x, maxdim=MAX_DIMENSION))\n",
    "df_XY.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "32683352",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:12.883246Z",
     "start_time": "2022-09-04T20:06:12.873898Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1, 1615, 2), (1, 1615, 1))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = 30\n",
    "df_XY.iloc[idx][0].shape, df_XY.iloc[idx][1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "367cb6cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:13.303825Z",
     "start_time": "2022-09-04T20:06:13.291504Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[    1.,    70.],\n",
       "        [    1.,    69.],\n",
       "        [    0.,    71.],\n",
       "        ...,\n",
       "        [-1000., -1000.],\n",
       "        [-1000., -1000.],\n",
       "        [-1000., -1000.]]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_XY.iloc[idx][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2fb00869",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:13.705901Z",
     "start_time": "2022-09-04T20:06:13.698225Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[    1],\n",
       "        [    1],\n",
       "        [    1],\n",
       "        ...,\n",
       "        [-1000],\n",
       "        [-1000],\n",
       "        [-1000]]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_XY.iloc[idx][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c916d626",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:14.126095Z",
     "start_time": "2022-09-04T20:06:14.083422Z"
    }
   },
   "outputs": [],
   "source": [
    "xs, ys = [], []\n",
    "for row_id, (x, y) in df_XY.items():\n",
    "    xs.append(x)\n",
    "    ys.append(y)\n",
    "    \n",
    "xs = np.array(xs, dtype=object)\n",
    "ys = np.array(ys, dtype=object)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "433ef872",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T20:06:14.608988Z",
     "start_time": "2022-09-04T20:06:14.574227Z"
    }
   },
   "outputs": [],
   "source": [
    "subjects_train_idx = range(0, 40)\n",
    "X_train = np.vstack(xs[subjects_train_idx]).astype(float)\n",
    "Y_train = np.vstack(ys[subjects_train_idx]).astype(int)\n",
    "\n",
    "subjects_val_idx = range(40, 50)\n",
    "X_val = np.vstack(xs[subjects_val_idx]).astype(float)\n",
    "Y_val = np.vstack(ys[subjects_val_idx]).astype(int)\n",
    "\n",
    "subjects_test_idx = range(50, 100)\n",
    "X_test = np.vstack(xs[subjects_test_idx]).astype(float)\n",
    "Y_test = np.vstack(ys[subjects_test_idx]).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e48a94c2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-05T07:00:31.417099Z",
     "start_time": "2022-09-05T07:00:31.355206Z"
    }
   },
   "outputs": [],
   "source": [
    "def bilstm_model(cnn_d=32, lstm_d=16):\n",
    "    \n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.layers.Masking(mask_value=MASKING_VALUE))\n",
    "        \n",
    "    model.add(tf.keras.layers.Dropout(0.1))\n",
    "\n",
    "    model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_d, return_sequences=True)))\n",
    "    model.add(tf.keras.layers.Dense(1, activation=\"sigmoid\", name='output'))\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "bilstm_model = bilstm_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5fd24772",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-15 18:44:44.200468: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5/5 [==============================] - 12s 1s/step - loss: 0.4283 - accuracy: 0.6186 - val_loss: 0.4214 - val_accuracy: 0.6748\n",
      "Epoch 2/50\n",
      "5/5 [==============================] - 2s 407ms/step - loss: 0.3902 - accuracy: 0.6870 - val_loss: 0.4066 - val_accuracy: 0.6748\n",
      "Epoch 3/50\n",
      "5/5 [==============================] - 2s 405ms/step - loss: 0.3848 - accuracy: 0.6870 - val_loss: 0.4050 - val_accuracy: 0.6749\n",
      "Epoch 4/50\n",
      "5/5 [==============================] - 2s 402ms/step - loss: 0.3798 - accuracy: 0.6875 - val_loss: 0.3991 - val_accuracy: 0.6788\n",
      "Epoch 5/50\n",
      "5/5 [==============================] - 2s 407ms/step - loss: 0.3727 - accuracy: 0.6912 - val_loss: 0.3908 - val_accuracy: 0.6888\n",
      "Epoch 6/50\n",
      "5/5 [==============================] - 2s 403ms/step - loss: 0.3667 - accuracy: 0.7013 - val_loss: 0.3826 - val_accuracy: 0.7065\n",
      "Epoch 7/50\n",
      "5/5 [==============================] - 2s 412ms/step - loss: 0.3568 - accuracy: 0.7196 - val_loss: 0.3701 - val_accuracy: 0.7282\n",
      "Epoch 8/50\n",
      "5/5 [==============================] - 2s 407ms/step - loss: 0.3429 - accuracy: 0.7383 - val_loss: 0.3518 - val_accuracy: 0.7548\n",
      "Epoch 9/50\n",
      "5/5 [==============================] - 2s 412ms/step - loss: 0.3197 - accuracy: 0.7717 - val_loss: 0.3206 - val_accuracy: 0.8015\n",
      "Epoch 10/50\n",
      "5/5 [==============================] - 2s 404ms/step - loss: 0.2973 - accuracy: 0.8125 - val_loss: 0.2927 - val_accuracy: 0.8265\n",
      "Epoch 11/50\n",
      "5/5 [==============================] - 2s 405ms/step - loss: 0.2865 - accuracy: 0.8000 - val_loss: 0.2915 - val_accuracy: 0.8206\n",
      "Epoch 12/50\n",
      "5/5 [==============================] - 2s 402ms/step - loss: 0.2665 - accuracy: 0.8277 - val_loss: 0.2694 - val_accuracy: 0.8339\n",
      "Epoch 13/50\n",
      "5/5 [==============================] - 2s 399ms/step - loss: 0.2599 - accuracy: 0.8265 - val_loss: 0.2708 - val_accuracy: 0.8346\n",
      "Epoch 14/50\n",
      "5/5 [==============================] - 2s 400ms/step - loss: 0.2507 - accuracy: 0.8397 - val_loss: 0.2652 - val_accuracy: 0.8327\n",
      "Epoch 15/50\n",
      "5/5 [==============================] - 2s 399ms/step - loss: 0.2564 - accuracy: 0.8333 - val_loss: 0.2648 - val_accuracy: 0.8308\n",
      "Epoch 16/50\n",
      "5/5 [==============================] - 2s 403ms/step - loss: 0.2685 - accuracy: 0.8134 - val_loss: 0.2614 - val_accuracy: 0.8307\n",
      "Epoch 17/50\n",
      "5/5 [==============================] - 2s 407ms/step - loss: 0.2612 - accuracy: 0.8277 - val_loss: 0.2679 - val_accuracy: 0.8268\n",
      "Epoch 18/50\n",
      "5/5 [==============================] - 2s 405ms/step - loss: 0.2515 - accuracy: 0.8325 - val_loss: 0.2696 - val_accuracy: 0.8308\n",
      "Epoch 19/50\n",
      "5/5 [==============================] - 2s 403ms/step - loss: 0.2542 - accuracy: 0.8347 - val_loss: 0.2615 - val_accuracy: 0.8334\n"
     ]
    }
   ],
   "source": [
    "reset_seed()\n",
    "early_stop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n",
    "history = bilstm_model.fit(X_train, Y_train,\n",
    "                           validation_data=(X_val, Y_val), \n",
    "                           epochs=50, \n",
    "                           batch_size=8,\n",
    "                           shuffle=True,\n",
    "                           callbacks=[early_stop_callback])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8ac0a0b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 77ms/step - loss: 0.2231 - accuracy: 0.8579\n"
     ]
    }
   ],
   "source": [
    "with tf.device('/cpu:0'):\n",
    "    bilstm_model.evaluate(x=X_test.astype(float), y=Y_test.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8933436f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 2s 77ms/step\n",
      "Have a look at the shape now: (50, 1615, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]],\n",
       "\n",
       "       [[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]],\n",
       "\n",
       "       [[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]],\n",
       "\n",
       "       [[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]],\n",
       "\n",
       "       [[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        ...,\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]]], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.device('/cpu:0'):\n",
    "    p = bilstm_model.predict(x=X_test.astype(float)).round()\n",
    "    \n",
    "print(\"Have a look at the shape now:\", p.shape)\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "df7330fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>51982</th>\n",
       "      <td>50</td>\n",
       "      <td>5099</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51983</th>\n",
       "      <td>50</td>\n",
       "      <td>5129</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51984</th>\n",
       "      <td>50</td>\n",
       "      <td>5159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51985</th>\n",
       "      <td>50</td>\n",
       "      <td>5189</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51986</th>\n",
       "      <td>50</td>\n",
       "      <td>5219</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50877 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time  sleep_phase  sleep\n",
       "51982    50   5099          0.0  False\n",
       "51983    50   5129          0.0  False\n",
       "51984    50   5159          0.0  False\n",
       "51985    50   5189          0.0  False\n",
       "51986    50   5219          0.0  False\n",
       "...     ...    ...          ...    ...\n",
       "102854   99  27569          0.0  False\n",
       "102855   99  27599          0.0  False\n",
       "102856   99  27629          0.0  False\n",
       "102857   99  27659          0.0  False\n",
       "102858   99  27689          0.0  False\n",
       "\n",
       "[50877 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_range = range(50, 100)\n",
    "\n",
    "df_held_out_test = df[df[\"pid\"].isin(test_range)][[\"pid\", \"time\", \"sleep_phase\", \"sleep\"]].copy()\n",
    "df_held_out_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "aee65a34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38972.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_test.flatten().shape, p.flatten().shape\n",
    "\n",
    "p[Y_test != MASKING_VALUE].round().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "79a54427",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50</td>\n",
       "      <td>26519</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>26549</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>26579</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>26609</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50</td>\n",
       "      <td>26639</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pid   time  sleep_phase  sleep\n",
       "0       50  26519          1.0   True\n",
       "1       50  26549          2.0   True\n",
       "2       50  26579          1.0   True\n",
       "3       50  26609          1.0   True\n",
       "4       50  26639          1.0   True\n",
       "...    ...    ...          ...    ...\n",
       "19995   99  27569          0.0  False\n",
       "19996   99  27599          0.0  False\n",
       "19997   99  27629          0.0  False\n",
       "19998   99  27659          0.0  False\n",
       "19999   99  27689          0.0  False\n",
       "\n",
       "[20000 rows x 4 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_held_out_test.groupby(\"pid\").tail(400).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4222b42a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_per_pid(df, test_range, maxdim, nnmodel, x_test, y_test, masking_value, name):\n",
    "    \n",
    "    df_held_out_test = df[df[\"pid\"].isin(test_range)][[\"pid\", \"time\", \"sleep_phase\", \"sleep\"]].copy()\n",
    "    df_held_out_test = df_held_out_test.groupby(\"pid\").head(maxdim).reset_index(drop=True)\n",
    "    \n",
    "    p = nnmodel.predict(x_test).round()\n",
    "    df_held_out_test[\"yhat\"] = p[y_test != masking_value].round()\n",
    "    \n",
    "    df_held_out_test.to_csv(f\"results/{name}.csv.tar.gz\", index=False)\n",
    "        \n",
    "    final_results = df_held_out_test.groupby([\"pid\"])[[\"sleep\", \"yhat\"]].apply(lambda x:\n",
    "                                                                          classification_results(x[\"sleep\"].values, \n",
    "                                                                                                 x[\"yhat\"].values))\n",
    "\n",
    "    final_results = final_results.apply(pd.Series).rename(columns={0: \"Accuracy\", 1: \"MCC\", 2: \"F1_weighted\"})\n",
    "\n",
    "    return final_results.agg([\"mean\", \"std\"]).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9efc829e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 79ms/step\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>MCC</th>\n",
       "      <th>F1_weighted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.858</td>\n",
       "      <td>0.634</td>\n",
       "      <td>0.848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.076</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Accuracy    MCC  F1_weighted\n",
       "mean     0.858  0.634        0.848\n",
       "std      0.076  0.166        0.080"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_per_pid(df, range(50, 100), MAX_DIMENSION, bilstm_model,\n",
    "                 X_test.astype(float), Y_test, MASKING_VALUE, name=\"cnnlstm_model_end2end\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd72d696",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}