03_windowing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56f19279",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:08.342659Z",
     "start_time": "2022-09-04T09:19:07.419394Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b8278a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:22.593787Z",
     "start_time": "2022-09-04T09:19:21.418581Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./datasets/tutorial_sleep_training_data.csv.gz\")\n",
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37b1fc4d",
   "metadata": {},
   "source": [
    "# Visual Intuition:\n",
    "\n",
    "<img src=\"images/sliding_win.jpg\" width=\"400\" height=\"200\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a76dba72",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:23.151462Z",
     "start_time": "2022-09-04T09:19:23.100449Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>act</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>hr</th>\n",
       "      <th>pid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289664</th>\n",
       "      <td>29</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289694</th>\n",
       "      <td>59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289724</th>\n",
       "      <td>89</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289754</th>\n",
       "      <td>119</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289784</th>\n",
       "      <td>149</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323654</th>\n",
       "      <td>34019</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323684</th>\n",
       "      <td>34049</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323714</th>\n",
       "      <td>34079</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323744</th>\n",
       "      <td>34109</td>\n",
       "      <td>105.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323774</th>\n",
       "      <td>34139</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1138 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         time    act  sleep_phase     hr  pid\n",
       "289664     29   38.0          0.0   71.0   10\n",
       "289694     59    0.0          0.0   69.0   10\n",
       "289724     89   23.0          0.0   73.0   10\n",
       "289754    119   17.0          0.0   59.0   10\n",
       "289784    149    8.0          0.0   71.0   10\n",
       "...       ...    ...          ...    ...  ...\n",
       "323654  34019   64.0          0.0  100.0   10\n",
       "323684  34049   42.0          0.0   31.0   10\n",
       "323714  34079   45.0          0.0   34.0   10\n",
       "323744  34109  105.0          0.0   33.0   10\n",
       "323774  34139    7.0          0.0   32.0   10\n",
       "\n",
       "[1138 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = df[df[\"pid\"] == 10]\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54ea5803",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:23.834067Z",
     "start_time": "2022-09-04T09:19:23.821027Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.06414762741652"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[\"act\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b2de28d5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:24.421044Z",
     "start_time": "2022-09-04T09:19:24.404895Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10.6, 8.8, 13.2)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[0:10][\"act\"].mean(), df1[1:11][\"act\"].mean(), df1[2:12][\"act\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "aef4c2d9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:24.927799Z",
     "start_time": "2022-09-04T09:19:24.900121Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape:  (11, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>act</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>hr</th>\n",
       "      <th>pid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289664</th>\n",
       "      <td>29</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289694</th>\n",
       "      <td>59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289724</th>\n",
       "      <td>89</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289754</th>\n",
       "      <td>119</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289784</th>\n",
       "      <td>149</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289814</th>\n",
       "      <td>179</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289844</th>\n",
       "      <td>209</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289874</th>\n",
       "      <td>239</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289904</th>\n",
       "      <td>269</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289934</th>\n",
       "      <td>299</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289964</th>\n",
       "      <td>329</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        time   act  sleep_phase    hr  pid\n",
       "289664    29  38.0          0.0  71.0   10\n",
       "289694    59   0.0          0.0  69.0   10\n",
       "289724    89  23.0          0.0  73.0   10\n",
       "289754   119  17.0          0.0  59.0   10\n",
       "289784   149   8.0          0.0  71.0   10\n",
       "289814   179  12.0          0.0  61.0   10\n",
       "289844   209   0.0          0.0  73.0   10\n",
       "289874   239   0.0          0.0  73.0   10\n",
       "289904   269   0.0          0.0  68.0   10\n",
       "289934   299   8.0          0.0  80.0   10\n",
       "289964   329  20.0          0.0  72.0   10"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Centered window of size 11 (5 + 1 + 5)\n",
    "i = 5\n",
    "winsize = 10\n",
    "df_slice = df1[i-(winsize//2):i+(winsize//2)+1]\n",
    "\n",
    "print(\"Shape: \", df_slice.shape)\n",
    "df_slice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "77349839",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:25.700390Z",
     "start_time": "2022-09-04T09:19:25.573852Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31.3 ms ± 305 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "# %%timeit\n",
    "\n",
    "n = df1.shape[0]\n",
    "winsize = 11\n",
    "\n",
    "result_loop = []\n",
    "for i in range(0, n):\n",
    "    r = df1[i-(winsize//2):i+(winsize//2)+1][\"act\"].mean()\n",
    "    result_loop.append(r)\n",
    "    \n",
    "result_loop = pd.Series(result_loop)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1786d347",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           NaN\n",
       "1           NaN\n",
       "2           NaN\n",
       "3           NaN\n",
       "4           NaN\n",
       "5     11.454545\n",
       "6     12.000000\n",
       "7     12.181818\n",
       "8     12.818182\n",
       "9     11.727273\n",
       "10    11.000000\n",
       "11    11.363636\n",
       "12    13.545455\n",
       "13    25.909091\n",
       "14    39.181818\n",
       "15    45.454545\n",
       "16    52.818182\n",
       "17    67.727273\n",
       "18    72.363636\n",
       "19    83.727273\n",
       "dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_loop[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cbeb5c4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T08:18:00.755597Z",
     "start_time": "2022-08-28T08:18:00.751342Z"
    }
   },
   "source": [
    "# Pandas windowing\n",
    "Alternatively, you could use the native version from pandas:\n",
    "\n",
    "https://pandas.pydata.org/docs/user_guide/window.html\n",
    "\n",
    "It also has interesting properties to deal with time (e.g., rolling windows per hour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a4105efc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:27.797445Z",
     "start_time": "2022-09-04T09:19:27.779378Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65.2 µs ± 698 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n"
     ]
    }
   ],
   "source": [
    "# %%timeit \n",
    "\n",
    "result_pdw = df1[\"act\"].rolling(window=11, center=True, min_periods=11).mean()\n",
    "result_pdw[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9f1222cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:28.544228Z",
     "start_time": "2022-09-04T09:19:28.526823Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "289664          NaN\n",
       "289694          NaN\n",
       "289724          NaN\n",
       "289754          NaN\n",
       "289784          NaN\n",
       "289814          NaN\n",
       "289844          NaN\n",
       "289874          NaN\n",
       "289904          NaN\n",
       "289934          NaN\n",
       "289964    11.454545\n",
       "289994    12.000000\n",
       "290024    12.181818\n",
       "290054    12.818182\n",
       "290084    11.727273\n",
       "290114    11.000000\n",
       "290144    11.363636\n",
       "290174    13.545455\n",
       "290204    25.909091\n",
       "290234    39.181818\n",
       "Name: act, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_pdw = df1[\"act\"].rolling(window=11, center=False, min_periods=11).mean()\n",
    "result_pdw[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "19e9480b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:29.124397Z",
     "start_time": "2022-09-04T09:19:29.106700Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "289664    38.000000\n",
       "289694    19.000000\n",
       "289724    20.333333\n",
       "289754    19.500000\n",
       "289784    17.200000\n",
       "289814    16.333333\n",
       "289844    14.000000\n",
       "289874    12.250000\n",
       "289904    10.888889\n",
       "289934    10.600000\n",
       "289964    11.454545\n",
       "289994    12.000000\n",
       "290024    12.181818\n",
       "290054    12.818182\n",
       "290084    11.727273\n",
       "290114    11.000000\n",
       "290144    11.363636\n",
       "290174    13.545455\n",
       "290204    25.909091\n",
       "290234    39.181818\n",
       "Name: act, dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_pdw = df1[\"act\"].rolling(window=11, center=False, min_periods=1).mean()\n",
    "result_pdw[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0087c811",
   "metadata": {},
   "source": [
    "## A Few Question:\n",
    "\n",
    "- How about the last 20 numbers? I.e.,: result_pdw[-20:] Vs result_loop[-20:]\n",
    "- How about the execution time? (use %%timeit to check it)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e9281eaa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-09-04T09:19:30.425874Z",
     "start_time": "2022-09-04T09:19:30.391748Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>mysum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289664</th>\n",
       "      <td>16.333333</td>\n",
       "      <td>14.5</td>\n",
       "      <td>13.185851</td>\n",
       "      <td>173.866667</td>\n",
       "      <td>0.713228</td>\n",
       "      <td>0.666706</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289694</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>12.0</td>\n",
       "      <td>13.527749</td>\n",
       "      <td>183.000000</td>\n",
       "      <td>0.835845</td>\n",
       "      <td>0.404760</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289724</th>\n",
       "      <td>12.250000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>13.466891</td>\n",
       "      <td>181.357143</td>\n",
       "      <td>1.000748</td>\n",
       "      <td>0.535173</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289754</th>\n",
       "      <td>10.888889</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13.242398</td>\n",
       "      <td>175.361111</td>\n",
       "      <td>1.158859</td>\n",
       "      <td>0.824273</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289784</th>\n",
       "      <td>10.600000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.518431</td>\n",
       "      <td>156.711111</td>\n",
       "      <td>1.270328</td>\n",
       "      <td>1.344534</td>\n",
       "      <td>106.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323654</th>\n",
       "      <td>61.300000</td>\n",
       "      <td>54.0</td>\n",
       "      <td>43.415435</td>\n",
       "      <td>1884.900000</td>\n",
       "      <td>0.938059</td>\n",
       "      <td>1.207229</td>\n",
       "      <td>613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323684</th>\n",
       "      <td>62.111111</td>\n",
       "      <td>54.0</td>\n",
       "      <td>45.968588</td>\n",
       "      <td>2113.111111</td>\n",
       "      <td>0.847039</td>\n",
       "      <td>0.740940</td>\n",
       "      <td>559.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323714</th>\n",
       "      <td>68.625000</td>\n",
       "      <td>59.0</td>\n",
       "      <td>44.480935</td>\n",
       "      <td>1978.553571</td>\n",
       "      <td>0.823750</td>\n",
       "      <td>1.031684</td>\n",
       "      <td>549.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323744</th>\n",
       "      <td>56.571429</td>\n",
       "      <td>54.0</td>\n",
       "      <td>30.859127</td>\n",
       "      <td>952.285714</td>\n",
       "      <td>-0.009530</td>\n",
       "      <td>0.702244</td>\n",
       "      <td>396.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323774</th>\n",
       "      <td>52.833333</td>\n",
       "      <td>49.5</td>\n",
       "      <td>32.021347</td>\n",
       "      <td>1025.366667</td>\n",
       "      <td>0.414128</td>\n",
       "      <td>1.640862</td>\n",
       "      <td>317.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1138 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             mean  median        std          var      skew      kurt  mysum\n",
       "289664  16.333333    14.5  13.185851   173.866667  0.713228  0.666706   98.0\n",
       "289694  14.000000    12.0  13.527749   183.000000  0.835845  0.404760   98.0\n",
       "289724  12.250000    10.0  13.466891   181.357143  1.000748  0.535173   98.0\n",
       "289754  10.888889     8.0  13.242398   175.361111  1.158859  0.824273   98.0\n",
       "289784  10.600000     8.0  12.518431   156.711111  1.270328  1.344534  106.0\n",
       "...           ...     ...        ...          ...       ...       ...    ...\n",
       "323654  61.300000    54.0  43.415435  1884.900000  0.938059  1.207229  613.0\n",
       "323684  62.111111    54.0  45.968588  2113.111111  0.847039  0.740940  559.0\n",
       "323714  68.625000    59.0  44.480935  1978.553571  0.823750  1.031684  549.0\n",
       "323744  56.571429    54.0  30.859127   952.285714 -0.009530  0.702244  396.0\n",
       "323774  52.833333    49.5  32.021347  1025.366667  0.414128  1.640862  317.0\n",
       "\n",
       "[1138 rows x 7 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Centered = True\n",
    "Wsize = 11\n",
    "\n",
    "df1[\"act\"].rolling(window=Wsize, center=Centered, min_periods=1).agg([\"mean\", \"median\", \"std\", \"var\", \"skew\", \"kurt\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "780143b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>mymean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289664</th>\n",
       "      <td>16.333333</td>\n",
       "      <td>14.5</td>\n",
       "      <td>13.185851</td>\n",
       "      <td>173.866667</td>\n",
       "      <td>0.713228</td>\n",
       "      <td>0.666706</td>\n",
       "      <td>16.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289694</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>12.0</td>\n",
       "      <td>13.527749</td>\n",
       "      <td>183.000000</td>\n",
       "      <td>0.835845</td>\n",
       "      <td>0.404760</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289724</th>\n",
       "      <td>12.250000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>13.466891</td>\n",
       "      <td>181.357143</td>\n",
       "      <td>1.000748</td>\n",
       "      <td>0.535173</td>\n",
       "      <td>12.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289754</th>\n",
       "      <td>10.888889</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13.242398</td>\n",
       "      <td>175.361111</td>\n",
       "      <td>1.158859</td>\n",
       "      <td>0.824273</td>\n",
       "      <td>10.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289784</th>\n",
       "      <td>10.600000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.518431</td>\n",
       "      <td>156.711111</td>\n",
       "      <td>1.270328</td>\n",
       "      <td>1.344534</td>\n",
       "      <td>10.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323654</th>\n",
       "      <td>61.300000</td>\n",
       "      <td>54.0</td>\n",
       "      <td>43.415435</td>\n",
       "      <td>1884.900000</td>\n",
       "      <td>0.938059</td>\n",
       "      <td>1.207229</td>\n",
       "      <td>61.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323684</th>\n",
       "      <td>62.111111</td>\n",
       "      <td>54.0</td>\n",
       "      <td>45.968588</td>\n",
       "      <td>2113.111111</td>\n",
       "      <td>0.847039</td>\n",
       "      <td>0.740940</td>\n",
       "      <td>62.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323714</th>\n",
       "      <td>68.625000</td>\n",
       "      <td>59.0</td>\n",
       "      <td>44.480935</td>\n",
       "      <td>1978.553571</td>\n",
       "      <td>0.823750</td>\n",
       "      <td>1.031684</td>\n",
       "      <td>68.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323744</th>\n",
       "      <td>56.571429</td>\n",
       "      <td>54.0</td>\n",
       "      <td>30.859127</td>\n",
       "      <td>952.285714</td>\n",
       "      <td>-0.009530</td>\n",
       "      <td>0.702244</td>\n",
       "      <td>56.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323774</th>\n",
       "      <td>52.833333</td>\n",
       "      <td>49.5</td>\n",
       "      <td>32.021347</td>\n",
       "      <td>1025.366667</td>\n",
       "      <td>0.414128</td>\n",
       "      <td>1.640862</td>\n",
       "      <td>52.833333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1138 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             mean  median        std          var      skew      kurt  \\\n",
       "289664  16.333333    14.5  13.185851   173.866667  0.713228  0.666706   \n",
       "289694  14.000000    12.0  13.527749   183.000000  0.835845  0.404760   \n",
       "289724  12.250000    10.0  13.466891   181.357143  1.000748  0.535173   \n",
       "289754  10.888889     8.0  13.242398   175.361111  1.158859  0.824273   \n",
       "289784  10.600000     8.0  12.518431   156.711111  1.270328  1.344534   \n",
       "...           ...     ...        ...          ...       ...       ...   \n",
       "323654  61.300000    54.0  43.415435  1884.900000  0.938059  1.207229   \n",
       "323684  62.111111    54.0  45.968588  2113.111111  0.847039  0.740940   \n",
       "323714  68.625000    59.0  44.480935  1978.553571  0.823750  1.031684   \n",
       "323744  56.571429    54.0  30.859127   952.285714 -0.009530  0.702244   \n",
       "323774  52.833333    49.5  32.021347  1025.366667  0.414128  1.640862   \n",
       "\n",
       "           mymean  \n",
       "289664  16.333333  \n",
       "289694  14.000000  \n",
       "289724  12.250000  \n",
       "289754  10.888889  \n",
       "289784  10.600000  \n",
       "...           ...  \n",
       "323654  61.300000  \n",
       "323684  62.111111  \n",
       "323714  68.625000  \n",
       "323744  56.571429  \n",
       "323774  52.833333  \n",
       "\n",
       "[1138 rows x 7 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def mymean(x):\n",
    "    return np.mean(x)\n",
    "\n",
    "df1[\"act\"].rolling(window=Wsize, center=Centered, min_periods=1).agg([\"mean\", \"median\", \"std\", \"var\", \"skew\", \"kurt\", mymean])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b702eadf",
   "metadata": {},
   "source": [
    "## How to apply rolling windows to the whole dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f507f409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>hr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>29</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">99</th>\n",
       "      <th>27569</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27599</th>\n",
       "      <td>29.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27629</th>\n",
       "      <td>53.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27659</th>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27689</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            act  sleep_phase    hr\n",
       "pid time                          \n",
       "0   29      2.0          0.0  71.0\n",
       "    59      0.0          0.0  76.0\n",
       "    89      1.0          0.0  78.0\n",
       "    119     2.0          0.0  73.0\n",
       "    149    87.0          0.0  80.0\n",
       "...         ...          ...   ...\n",
       "99  27569   9.0          0.0  67.0\n",
       "    27599  29.0          0.0  65.0\n",
       "    27629  53.0          0.0  64.0\n",
       "    27659  74.0          0.0  67.0\n",
       "    27689  41.0          0.0  69.0\n",
       "\n",
       "[102859 rows x 3 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_indexed = df.set_index([\"pid\", \"time\"])\n",
    "df_indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0fddd587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>29</th>\n",
       "      <td>15.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>35.120744</td>\n",
       "      <td>1233.466667</td>\n",
       "      <td>2.445916</td>\n",
       "      <td>5.986385</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>13.285714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.515198</td>\n",
       "      <td>1057.238095</td>\n",
       "      <td>2.642249</td>\n",
       "      <td>6.985881</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>11.625000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.467489</td>\n",
       "      <td>928.267857</td>\n",
       "      <td>2.824207</td>\n",
       "      <td>7.982049</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>10.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.761954</td>\n",
       "      <td>827.250000</td>\n",
       "      <td>2.995208</td>\n",
       "      <td>8.978607</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>9.300000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>27.313204</td>\n",
       "      <td>746.011111</td>\n",
       "      <td>3.157003</td>\n",
       "      <td>9.975393</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">99</th>\n",
       "      <th>27569</th>\n",
       "      <td>32.600000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.336078</td>\n",
       "      <td>802.933333</td>\n",
       "      <td>0.034822</td>\n",
       "      <td>-1.788971</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>326.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27599</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>41.0</td>\n",
       "      <td>27.807373</td>\n",
       "      <td>773.250000</td>\n",
       "      <td>-0.217459</td>\n",
       "      <td>-1.604856</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>324.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27629</th>\n",
       "      <td>33.125000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.261218</td>\n",
       "      <td>798.696429</td>\n",
       "      <td>0.060339</td>\n",
       "      <td>-1.596417</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27659</th>\n",
       "      <td>29.428571</td>\n",
       "      <td>29.0</td>\n",
       "      <td>28.359931</td>\n",
       "      <td>804.285714</td>\n",
       "      <td>0.446974</td>\n",
       "      <td>-1.109392</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27689</th>\n",
       "      <td>34.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>27.623661</td>\n",
       "      <td>763.066667</td>\n",
       "      <td>0.175520</td>\n",
       "      <td>-0.958776</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                mean  median        std          var      skew      kurt  \\\n",
       "pid time                                                                   \n",
       "0   29     15.333333     1.5  35.120744  1233.466667  2.445916  5.986385   \n",
       "    59     13.285714     1.0  32.515198  1057.238095  2.642249  6.985881   \n",
       "    89     11.625000     1.0  30.467489   928.267857  2.824207  7.982049   \n",
       "    119    10.333333     1.0  28.761954   827.250000  2.995208  8.978607   \n",
       "    149     9.300000     0.5  27.313204   746.011111  3.157003  9.975393   \n",
       "...              ...     ...        ...          ...       ...       ...   \n",
       "99  27569  32.600000    35.0  28.336078   802.933333  0.034822 -1.788971   \n",
       "    27599  36.000000    41.0  27.807373   773.250000 -0.217459 -1.604856   \n",
       "    27629  33.125000    35.0  28.261218   798.696429  0.060339 -1.596417   \n",
       "    27659  29.428571    29.0  28.359931   804.285714  0.446974 -1.109392   \n",
       "    27689  34.333333    35.0  27.623661   763.066667  0.175520 -0.958776   \n",
       "\n",
       "            max  min  count    sum  \n",
       "pid time                            \n",
       "0   29     87.0  0.0    6.0   92.0  \n",
       "    59     87.0  0.0    7.0   93.0  \n",
       "    89     87.0  0.0    8.0   93.0  \n",
       "    119    87.0  0.0    9.0   93.0  \n",
       "    149    87.0  0.0   10.0   93.0  \n",
       "...         ...  ...    ...    ...  \n",
       "99  27569  74.0  0.0   10.0  326.0  \n",
       "    27599  74.0  0.0    9.0  324.0  \n",
       "    27629  74.0  0.0    8.0  265.0  \n",
       "    27659  74.0  0.0    7.0  206.0  \n",
       "    27689  74.0  0.0    6.0  206.0  \n",
       "\n",
       "[102859 rows x 10 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_grped = df_indexed[\"act\"].rolling(window=Wsize, center=Centered, min_periods=1).agg([\"mean\", \"median\", \"std\", \"var\", \"skew\", \"kurt\", \"max\", \"min\", \"count\", \"sum\"])\n",
    "X_grped"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eebea655",
   "metadata": {},
   "source": [
    "<hr>\n",
    "<h3> That is it!!! </h3>\n",
    "<h3> We already have the minimal minimal minimal necessary to run our first ML model! </h3>\n",
    "\n",
    "So lets do it....\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e24440b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:49.332628Z",
     "start_time": "2022-08-28T13:50:49.313418Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>35.120744</td>\n",
       "      <td>1233.466667</td>\n",
       "      <td>2.445916</td>\n",
       "      <td>5.986385</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>13.285714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.515198</td>\n",
       "      <td>1057.238095</td>\n",
       "      <td>2.642249</td>\n",
       "      <td>6.985881</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11.625000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.467489</td>\n",
       "      <td>928.267857</td>\n",
       "      <td>2.824207</td>\n",
       "      <td>7.982049</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>10.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.761954</td>\n",
       "      <td>827.250000</td>\n",
       "      <td>2.995208</td>\n",
       "      <td>8.978607</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>9.300000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>27.313204</td>\n",
       "      <td>746.011111</td>\n",
       "      <td>3.157003</td>\n",
       "      <td>9.975393</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>32.600000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.336078</td>\n",
       "      <td>802.933333</td>\n",
       "      <td>0.034822</td>\n",
       "      <td>-1.788971</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>326.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>41.0</td>\n",
       "      <td>27.807373</td>\n",
       "      <td>773.250000</td>\n",
       "      <td>-0.217459</td>\n",
       "      <td>-1.604856</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>324.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>33.125000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.261218</td>\n",
       "      <td>798.696429</td>\n",
       "      <td>0.060339</td>\n",
       "      <td>-1.596417</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>29.428571</td>\n",
       "      <td>29.0</td>\n",
       "      <td>28.359931</td>\n",
       "      <td>804.285714</td>\n",
       "      <td>0.446974</td>\n",
       "      <td>-1.109392</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>34.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>27.623661</td>\n",
       "      <td>763.066667</td>\n",
       "      <td>0.175520</td>\n",
       "      <td>-0.958776</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time       mean  median        std          var      skew  \\\n",
       "0         0     29  15.333333     1.5  35.120744  1233.466667  2.445916   \n",
       "1         0     59  13.285714     1.0  32.515198  1057.238095  2.642249   \n",
       "2         0     89  11.625000     1.0  30.467489   928.267857  2.824207   \n",
       "3         0    119  10.333333     1.0  28.761954   827.250000  2.995208   \n",
       "4         0    149   9.300000     0.5  27.313204   746.011111  3.157003   \n",
       "...     ...    ...        ...     ...        ...          ...       ...   \n",
       "102854   99  27569  32.600000    35.0  28.336078   802.933333  0.034822   \n",
       "102855   99  27599  36.000000    41.0  27.807373   773.250000 -0.217459   \n",
       "102856   99  27629  33.125000    35.0  28.261218   798.696429  0.060339   \n",
       "102857   99  27659  29.428571    29.0  28.359931   804.285714  0.446974   \n",
       "102858   99  27689  34.333333    35.0  27.623661   763.066667  0.175520   \n",
       "\n",
       "            kurt   max  min  count    sum  \n",
       "0       5.986385  87.0  0.0    6.0   92.0  \n",
       "1       6.985881  87.0  0.0    7.0   93.0  \n",
       "2       7.982049  87.0  0.0    8.0   93.0  \n",
       "3       8.978607  87.0  0.0    9.0   93.0  \n",
       "4       9.975393  87.0  0.0   10.0   93.0  \n",
       "...          ...   ...  ...    ...    ...  \n",
       "102854 -1.788971  74.0  0.0   10.0  326.0  \n",
       "102855 -1.604856  74.0  0.0    9.0  324.0  \n",
       "102856 -1.596417  74.0  0.0    8.0  265.0  \n",
       "102857 -1.109392  74.0  0.0    7.0  206.0  \n",
       "102858 -0.958776  74.0  0.0    6.0  206.0  \n",
       "\n",
       "[102859 rows x 12 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset index to get pid and time cols back to the dataframe\n",
    "X_reseted = X_grped.reset_index()\n",
    "X_reseted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "85835507",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>35.120744</td>\n",
       "      <td>1233.466667</td>\n",
       "      <td>2.445916</td>\n",
       "      <td>5.986385</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>13.285714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.515198</td>\n",
       "      <td>1057.238095</td>\n",
       "      <td>2.642249</td>\n",
       "      <td>6.985881</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11.625000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.467489</td>\n",
       "      <td>928.267857</td>\n",
       "      <td>2.824207</td>\n",
       "      <td>7.982049</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>10.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.761954</td>\n",
       "      <td>827.250000</td>\n",
       "      <td>2.995208</td>\n",
       "      <td>8.978607</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>9.300000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>27.313204</td>\n",
       "      <td>746.011111</td>\n",
       "      <td>3.157003</td>\n",
       "      <td>9.975393</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>87.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>32.600000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.336078</td>\n",
       "      <td>802.933333</td>\n",
       "      <td>0.034822</td>\n",
       "      <td>-1.788971</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>41.0</td>\n",
       "      <td>27.807373</td>\n",
       "      <td>773.250000</td>\n",
       "      <td>-0.217459</td>\n",
       "      <td>-1.604856</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>33.125000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.261218</td>\n",
       "      <td>798.696429</td>\n",
       "      <td>0.060339</td>\n",
       "      <td>-1.596417</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>53.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>29.428571</td>\n",
       "      <td>29.0</td>\n",
       "      <td>28.359931</td>\n",
       "      <td>804.285714</td>\n",
       "      <td>0.446974</td>\n",
       "      <td>-1.109392</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>34.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>27.623661</td>\n",
       "      <td>763.066667</td>\n",
       "      <td>0.175520</td>\n",
       "      <td>-0.958776</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time       mean  median        std          var      skew  \\\n",
       "0         0     29  15.333333     1.5  35.120744  1233.466667  2.445916   \n",
       "1         0     59  13.285714     1.0  32.515198  1057.238095  2.642249   \n",
       "2         0     89  11.625000     1.0  30.467489   928.267857  2.824207   \n",
       "3         0    119  10.333333     1.0  28.761954   827.250000  2.995208   \n",
       "4         0    149   9.300000     0.5  27.313204   746.011111  3.157003   \n",
       "...     ...    ...        ...     ...        ...          ...       ...   \n",
       "102854   99  27569  32.600000    35.0  28.336078   802.933333  0.034822   \n",
       "102855   99  27599  36.000000    41.0  27.807373   773.250000 -0.217459   \n",
       "102856   99  27629  33.125000    35.0  28.261218   798.696429  0.060339   \n",
       "102857   99  27659  29.428571    29.0  28.359931   804.285714  0.446974   \n",
       "102858   99  27689  34.333333    35.0  27.623661   763.066667  0.175520   \n",
       "\n",
       "            kurt   max  min  count    sum   act  \n",
       "0       5.986385  87.0  0.0    6.0   92.0   2.0  \n",
       "1       6.985881  87.0  0.0    7.0   93.0   0.0  \n",
       "2       7.982049  87.0  0.0    8.0   93.0   1.0  \n",
       "3       8.978607  87.0  0.0    9.0   93.0   2.0  \n",
       "4       9.975393  87.0  0.0   10.0   93.0  87.0  \n",
       "...          ...   ...  ...    ...    ...   ...  \n",
       "102854 -1.788971  74.0  0.0   10.0  326.0   9.0  \n",
       "102855 -1.604856  74.0  0.0    9.0  324.0  29.0  \n",
       "102856 -1.596417  74.0  0.0    8.0  265.0  53.0  \n",
       "102857 -1.109392  74.0  0.0    7.0  206.0  74.0  \n",
       "102858 -0.958776  74.0  0.0    6.0  206.0  41.0  \n",
       "\n",
       "[102859 rows x 13 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge X to the original dataframe to append the original activity values\n",
    "X = pd.merge(X_reseted, df[[\"time\", \"pid\", \"act\"]], on=[\"time\", \"pid\"])\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "dbaeb673",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:50.138917Z",
     "start_time": "2022-08-28T13:50:50.095010Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pid</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>149</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>3116246</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>3116276</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>3116306</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>3116336</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>3116366</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           time  pid  sleep_phase  sleep\n",
       "0            29    0          0.0  False\n",
       "1            59    0          0.0  False\n",
       "2            89    0          0.0  False\n",
       "3           119    0          0.0  False\n",
       "4           149    0          0.0  False\n",
       "...         ...  ...          ...    ...\n",
       "102854  3116246   99          0.0  False\n",
       "102855  3116276   99          0.0  False\n",
       "102856  3116306   99          0.0  False\n",
       "102857  3116336   99          0.0  False\n",
       "102858  3116366   99          0.0  False\n",
       "\n",
       "[102859 rows x 4 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get Ys (i.e., sleep labels)\n",
    "Y = df[[\"pid\", \"sleep_phase\"]].reset_index().rename(columns={\"index\": \"time\"})\n",
    "Y[\"sleep\"] = Y[\"sleep_phase\"] > 0\n",
    "Y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fb8ac95",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T12:34:29.468688Z",
     "start_time": "2022-08-28T12:34:29.462139Z"
    }
   },
   "source": [
    "### Are there NAs?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "15a3951b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:50.729970Z",
     "start_time": "2022-08-28T13:50:50.692617Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>0</td>\n",
       "      <td>18059</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602</th>\n",
       "      <td>0</td>\n",
       "      <td>18089</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>0</td>\n",
       "      <td>18119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>0</td>\n",
       "      <td>18149</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>0</td>\n",
       "      <td>18179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102274</th>\n",
       "      <td>99</td>\n",
       "      <td>10169</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>5.025142e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102471</th>\n",
       "      <td>99</td>\n",
       "      <td>16079</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.232615e-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102490</th>\n",
       "      <td>99</td>\n",
       "      <td>16649</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.232629e-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102491</th>\n",
       "      <td>99</td>\n",
       "      <td>16679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.232629e-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102492</th>\n",
       "      <td>99</td>\n",
       "      <td>16709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.232629e-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>802 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time  mean  median       std           var  skew  kurt  max  \\\n",
       "601       0  18059   0.0     0.0  0.000002  3.219017e-12   NaN   NaN  0.0   \n",
       "602       0  18089   0.0     0.0  0.000002  3.219017e-12   NaN   NaN  0.0   \n",
       "603       0  18119   0.0     0.0  0.000002  3.219017e-12   NaN   NaN  0.0   \n",
       "604       0  18149   0.0     0.0  0.000002  3.219017e-12   NaN   NaN  0.0   \n",
       "605       0  18179   0.0     0.0  0.000002  3.219017e-12   NaN   NaN  0.0   \n",
       "...     ...    ...   ...     ...       ...           ...   ...   ...  ...   \n",
       "102274   99  10169   0.0     0.0  0.000002  5.025142e-12   NaN   NaN  0.0   \n",
       "102471   99  16079   0.0     0.0  0.000004  1.232615e-11   NaN   NaN  0.0   \n",
       "102490   99  16649   0.0     0.0  0.000004  1.232629e-11   NaN   NaN  0.0   \n",
       "102491   99  16679   0.0     0.0  0.000004  1.232629e-11   NaN   NaN  0.0   \n",
       "102492   99  16709   0.0     0.0  0.000004  1.232629e-11   NaN   NaN  0.0   \n",
       "\n",
       "        min  count  sum  act  \n",
       "601     0.0   11.0  0.0  0.0  \n",
       "602     0.0   11.0  0.0  0.0  \n",
       "603     0.0   11.0  0.0  0.0  \n",
       "604     0.0   11.0  0.0  0.0  \n",
       "605     0.0   11.0  0.0  0.0  \n",
       "...     ...    ...  ...  ...  \n",
       "102274  0.0   11.0  0.0  0.0  \n",
       "102471  0.0   11.0  0.0  0.0  \n",
       "102490  0.0   11.0  0.0  0.0  \n",
       "102491  0.0   11.0  0.0  0.0  \n",
       "102492  0.0   11.0  0.0  0.0  \n",
       "\n",
       "[802 rows x 13 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[X.isna().any(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f70f83b",
   "metadata": {},
   "source": [
    "Lets have a look at them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5a6612d8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:51.154182Z",
     "start_time": "2022-08-28T13:50:51.123027Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>0</td>\n",
       "      <td>17729</td>\n",
       "      <td>50.181818</td>\n",
       "      <td>17.0</td>\n",
       "      <td>73.730344</td>\n",
       "      <td>5.436164e+03</td>\n",
       "      <td>1.703638</td>\n",
       "      <td>1.619478</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>552.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>0</td>\n",
       "      <td>17759</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>74.486240</td>\n",
       "      <td>5.548200e+03</td>\n",
       "      <td>1.688307</td>\n",
       "      <td>1.572494</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>539.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>0</td>\n",
       "      <td>17789</td>\n",
       "      <td>47.454545</td>\n",
       "      <td>5.0</td>\n",
       "      <td>75.387484</td>\n",
       "      <td>5.683273e+03</td>\n",
       "      <td>1.679443</td>\n",
       "      <td>1.531653</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>522.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>0</td>\n",
       "      <td>17819</td>\n",
       "      <td>28.636364</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.529392</td>\n",
       "      <td>2.973455e+03</td>\n",
       "      <td>2.616569</td>\n",
       "      <td>7.399291</td>\n",
       "      <td>182.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>315.0</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>0</td>\n",
       "      <td>17849</td>\n",
       "      <td>12.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.057191</td>\n",
       "      <td>4.022909e+02</td>\n",
       "      <td>1.363299</td>\n",
       "      <td>0.100988</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>0</td>\n",
       "      <td>17879</td>\n",
       "      <td>12.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.057191</td>\n",
       "      <td>4.022909e+02</td>\n",
       "      <td>1.363299</td>\n",
       "      <td>0.100988</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>0</td>\n",
       "      <td>17909</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.114671</td>\n",
       "      <td>4.046000e+02</td>\n",
       "      <td>1.362160</td>\n",
       "      <td>0.096118</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>0</td>\n",
       "      <td>17939</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.114671</td>\n",
       "      <td>4.046000e+02</td>\n",
       "      <td>1.362160</td>\n",
       "      <td>0.096118</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>0</td>\n",
       "      <td>17969</td>\n",
       "      <td>11.545455</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.343862</td>\n",
       "      <td>4.138727e+02</td>\n",
       "      <td>1.378474</td>\n",
       "      <td>0.106173</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>0</td>\n",
       "      <td>17999</td>\n",
       "      <td>7.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.533712</td>\n",
       "      <td>2.733636e+02</td>\n",
       "      <td>2.208707</td>\n",
       "      <td>4.059921</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>0</td>\n",
       "      <td>18029</td>\n",
       "      <td>4.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.774056</td>\n",
       "      <td>2.182727e+02</td>\n",
       "      <td>3.316625</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>0</td>\n",
       "      <td>18059</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602</th>\n",
       "      <td>0</td>\n",
       "      <td>18089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>0</td>\n",
       "      <td>18119</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>0</td>\n",
       "      <td>18149</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>0</td>\n",
       "      <td>18179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606</th>\n",
       "      <td>0</td>\n",
       "      <td>18209</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>607</th>\n",
       "      <td>0</td>\n",
       "      <td>18239</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>9.090909e-02</td>\n",
       "      <td>3.316625</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>608</th>\n",
       "      <td>0</td>\n",
       "      <td>18269</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>609</th>\n",
       "      <td>0</td>\n",
       "      <td>18299</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>610</th>\n",
       "      <td>0</td>\n",
       "      <td>18329</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>0</td>\n",
       "      <td>18359</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>0</td>\n",
       "      <td>18389</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>0</td>\n",
       "      <td>18419</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.674200</td>\n",
       "      <td>4.545455e-01</td>\n",
       "      <td>1.799664</td>\n",
       "      <td>2.610667</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>0</td>\n",
       "      <td>18449</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.674200</td>\n",
       "      <td>4.545455e-01</td>\n",
       "      <td>1.799664</td>\n",
       "      <td>2.610667</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>0</td>\n",
       "      <td>18479</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.687552</td>\n",
       "      <td>4.727273e-01</td>\n",
       "      <td>1.323910</td>\n",
       "      <td>0.976331</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>0</td>\n",
       "      <td>18509</td>\n",
       "      <td>4.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.229621</td>\n",
       "      <td>1.495636e+02</td>\n",
       "      <td>3.298547</td>\n",
       "      <td>10.911555</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>0</td>\n",
       "      <td>18539</td>\n",
       "      <td>4.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.229621</td>\n",
       "      <td>1.495636e+02</td>\n",
       "      <td>3.298547</td>\n",
       "      <td>10.911555</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>0</td>\n",
       "      <td>18569</td>\n",
       "      <td>4.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.259319</td>\n",
       "      <td>1.502909e+02</td>\n",
       "      <td>3.299109</td>\n",
       "      <td>10.913811</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>0</td>\n",
       "      <td>18599</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.288206</td>\n",
       "      <td>1.510000e+02</td>\n",
       "      <td>3.300460</td>\n",
       "      <td>10.919927</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     pid   time       mean  median        std           var      skew  \\\n",
       "590    0  17729  50.181818    17.0  73.730344  5.436164e+03  1.703638   \n",
       "591    0  17759  49.000000    17.0  74.486240  5.548200e+03  1.688307   \n",
       "592    0  17789  47.454545     5.0  75.387484  5.683273e+03  1.679443   \n",
       "593    0  17819  28.636364     1.0  54.529392  2.973455e+03  2.616569   \n",
       "594    0  17849  12.090909     0.0  20.057191  4.022909e+02  1.363299   \n",
       "595    0  17879  12.090909     0.0  20.057191  4.022909e+02  1.363299   \n",
       "596    0  17909  12.000000     0.0  20.114671  4.046000e+02  1.362160   \n",
       "597    0  17939  12.000000     0.0  20.114671  4.046000e+02  1.362160   \n",
       "598    0  17969  11.545455     0.0  20.343862  4.138727e+02  1.378474   \n",
       "599    0  17999   7.181818     0.0  16.533712  2.733636e+02  2.208707   \n",
       "600    0  18029   4.454545     0.0  14.774056  2.182727e+02  3.316625   \n",
       "601    0  18059   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "602    0  18089   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "603    0  18119   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "604    0  18149   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "605    0  18179   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "606    0  18209   0.000000     0.0   0.000002  3.219017e-12       NaN   \n",
       "607    0  18239   0.090909     0.0   0.301511  9.090909e-02  3.316625   \n",
       "608    0  18269   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "609    0  18299   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "610    0  18329   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "611    0  18359   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "612    0  18389   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "613    0  18419   0.363636     0.0   0.674200  4.545455e-01  1.799664   \n",
       "614    0  18449   0.363636     0.0   0.674200  4.545455e-01  1.799664   \n",
       "615    0  18479   0.454545     0.0   0.687552  4.727273e-01  1.323910   \n",
       "616    0  18509   4.181818     0.0  12.229621  1.495636e+02  3.298547   \n",
       "617    0  18539   4.181818     0.0  12.229621  1.495636e+02  3.298547   \n",
       "618    0  18569   4.090909     0.0  12.259319  1.502909e+02  3.299109   \n",
       "619    0  18599   4.000000     0.0  12.288206  1.510000e+02  3.300460   \n",
       "\n",
       "          kurt    max  min  count    sum   act  \n",
       "590   1.619478  207.0  0.0   11.0  552.0   1.0  \n",
       "591   1.572494  207.0  0.0   11.0  539.0   0.0  \n",
       "592   1.531653  207.0  0.0   11.0  522.0   5.0  \n",
       "593   7.399291  182.0  0.0   11.0  315.0  48.0  \n",
       "594   0.100988   49.0  0.0   11.0  133.0  30.0  \n",
       "595   0.100988   49.0  0.0   11.0  133.0  49.0  \n",
       "596   0.096118   49.0  0.0   11.0  132.0   0.0  \n",
       "597   0.096118   49.0  0.0   11.0  132.0   0.0  \n",
       "598   0.106173   49.0  0.0   11.0  127.0   0.0  \n",
       "599   4.059921   49.0  0.0   11.0   79.0   0.0  \n",
       "600  11.000000   49.0  0.0   11.0   49.0   0.0  \n",
       "601        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "602        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "603        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "604        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "605        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "606        NaN    0.0  0.0   11.0    0.0   0.0  \n",
       "607  11.000000    1.0  0.0   11.0    1.0   0.0  \n",
       "608   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "609   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "610   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "611   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "612   2.037037    1.0  0.0   11.0    2.0   1.0  \n",
       "613   2.610667    2.0  0.0   11.0    4.0   1.0  \n",
       "614   2.610667    2.0  0.0   11.0    4.0   0.0  \n",
       "615   0.976331    2.0  0.0   11.0    5.0   0.0  \n",
       "616  10.911555   41.0  0.0   11.0   46.0   0.0  \n",
       "617  10.911555   41.0  0.0   11.0   46.0   0.0  \n",
       "618  10.913811   41.0  0.0   11.0   45.0   2.0  \n",
       "619  10.919927   41.0  0.0   11.0   44.0   0.0  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[590:620]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ab1f839d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:51.566861Z",
     "start_time": "2022-08-28T13:50:51.547577Z"
    }
   },
   "outputs": [],
   "source": [
    "X = X.fillna(0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "58149784",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>0</td>\n",
       "      <td>17729</td>\n",
       "      <td>50.181818</td>\n",
       "      <td>17.0</td>\n",
       "      <td>73.730344</td>\n",
       "      <td>5.436164e+03</td>\n",
       "      <td>1.703638</td>\n",
       "      <td>1.619478</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>552.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>0</td>\n",
       "      <td>17759</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>74.486240</td>\n",
       "      <td>5.548200e+03</td>\n",
       "      <td>1.688307</td>\n",
       "      <td>1.572494</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>539.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>0</td>\n",
       "      <td>17789</td>\n",
       "      <td>47.454545</td>\n",
       "      <td>5.0</td>\n",
       "      <td>75.387484</td>\n",
       "      <td>5.683273e+03</td>\n",
       "      <td>1.679443</td>\n",
       "      <td>1.531653</td>\n",
       "      <td>207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>522.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>0</td>\n",
       "      <td>17819</td>\n",
       "      <td>28.636364</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.529392</td>\n",
       "      <td>2.973455e+03</td>\n",
       "      <td>2.616569</td>\n",
       "      <td>7.399291</td>\n",
       "      <td>182.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>315.0</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>0</td>\n",
       "      <td>17849</td>\n",
       "      <td>12.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.057191</td>\n",
       "      <td>4.022909e+02</td>\n",
       "      <td>1.363299</td>\n",
       "      <td>0.100988</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>0</td>\n",
       "      <td>17879</td>\n",
       "      <td>12.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.057191</td>\n",
       "      <td>4.022909e+02</td>\n",
       "      <td>1.363299</td>\n",
       "      <td>0.100988</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>0</td>\n",
       "      <td>17909</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.114671</td>\n",
       "      <td>4.046000e+02</td>\n",
       "      <td>1.362160</td>\n",
       "      <td>0.096118</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>0</td>\n",
       "      <td>17939</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.114671</td>\n",
       "      <td>4.046000e+02</td>\n",
       "      <td>1.362160</td>\n",
       "      <td>0.096118</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>0</td>\n",
       "      <td>17969</td>\n",
       "      <td>11.545455</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.343862</td>\n",
       "      <td>4.138727e+02</td>\n",
       "      <td>1.378474</td>\n",
       "      <td>0.106173</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>0</td>\n",
       "      <td>17999</td>\n",
       "      <td>7.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.533712</td>\n",
       "      <td>2.733636e+02</td>\n",
       "      <td>2.208707</td>\n",
       "      <td>4.059921</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>0</td>\n",
       "      <td>18029</td>\n",
       "      <td>4.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.774056</td>\n",
       "      <td>2.182727e+02</td>\n",
       "      <td>3.316625</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>0</td>\n",
       "      <td>18059</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602</th>\n",
       "      <td>0</td>\n",
       "      <td>18089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>0</td>\n",
       "      <td>18119</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>0</td>\n",
       "      <td>18149</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>0</td>\n",
       "      <td>18179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606</th>\n",
       "      <td>0</td>\n",
       "      <td>18209</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>3.219017e-12</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>607</th>\n",
       "      <td>0</td>\n",
       "      <td>18239</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>9.090909e-02</td>\n",
       "      <td>3.316625</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>608</th>\n",
       "      <td>0</td>\n",
       "      <td>18269</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>609</th>\n",
       "      <td>0</td>\n",
       "      <td>18299</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>610</th>\n",
       "      <td>0</td>\n",
       "      <td>18329</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>0</td>\n",
       "      <td>18359</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>0</td>\n",
       "      <td>18389</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.404520</td>\n",
       "      <td>1.636364e-01</td>\n",
       "      <td>1.922718</td>\n",
       "      <td>2.037037</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>0</td>\n",
       "      <td>18419</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.674200</td>\n",
       "      <td>4.545455e-01</td>\n",
       "      <td>1.799664</td>\n",
       "      <td>2.610667</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>0</td>\n",
       "      <td>18449</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.674200</td>\n",
       "      <td>4.545455e-01</td>\n",
       "      <td>1.799664</td>\n",
       "      <td>2.610667</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>0</td>\n",
       "      <td>18479</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.687552</td>\n",
       "      <td>4.727273e-01</td>\n",
       "      <td>1.323910</td>\n",
       "      <td>0.976331</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>0</td>\n",
       "      <td>18509</td>\n",
       "      <td>4.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.229621</td>\n",
       "      <td>1.495636e+02</td>\n",
       "      <td>3.298547</td>\n",
       "      <td>10.911555</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>0</td>\n",
       "      <td>18539</td>\n",
       "      <td>4.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.229621</td>\n",
       "      <td>1.495636e+02</td>\n",
       "      <td>3.298547</td>\n",
       "      <td>10.911555</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>0</td>\n",
       "      <td>18569</td>\n",
       "      <td>4.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.259319</td>\n",
       "      <td>1.502909e+02</td>\n",
       "      <td>3.299109</td>\n",
       "      <td>10.913811</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>0</td>\n",
       "      <td>18599</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.288206</td>\n",
       "      <td>1.510000e+02</td>\n",
       "      <td>3.300460</td>\n",
       "      <td>10.919927</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     pid   time       mean  median        std           var      skew  \\\n",
       "590    0  17729  50.181818    17.0  73.730344  5.436164e+03  1.703638   \n",
       "591    0  17759  49.000000    17.0  74.486240  5.548200e+03  1.688307   \n",
       "592    0  17789  47.454545     5.0  75.387484  5.683273e+03  1.679443   \n",
       "593    0  17819  28.636364     1.0  54.529392  2.973455e+03  2.616569   \n",
       "594    0  17849  12.090909     0.0  20.057191  4.022909e+02  1.363299   \n",
       "595    0  17879  12.090909     0.0  20.057191  4.022909e+02  1.363299   \n",
       "596    0  17909  12.000000     0.0  20.114671  4.046000e+02  1.362160   \n",
       "597    0  17939  12.000000     0.0  20.114671  4.046000e+02  1.362160   \n",
       "598    0  17969  11.545455     0.0  20.343862  4.138727e+02  1.378474   \n",
       "599    0  17999   7.181818     0.0  16.533712  2.733636e+02  2.208707   \n",
       "600    0  18029   4.454545     0.0  14.774056  2.182727e+02  3.316625   \n",
       "601    0  18059   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "602    0  18089   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "603    0  18119   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "604    0  18149   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "605    0  18179   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "606    0  18209   0.000000     0.0   0.000002  3.219017e-12  0.000000   \n",
       "607    0  18239   0.090909     0.0   0.301511  9.090909e-02  3.316625   \n",
       "608    0  18269   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "609    0  18299   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "610    0  18329   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "611    0  18359   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "612    0  18389   0.181818     0.0   0.404520  1.636364e-01  1.922718   \n",
       "613    0  18419   0.363636     0.0   0.674200  4.545455e-01  1.799664   \n",
       "614    0  18449   0.363636     0.0   0.674200  4.545455e-01  1.799664   \n",
       "615    0  18479   0.454545     0.0   0.687552  4.727273e-01  1.323910   \n",
       "616    0  18509   4.181818     0.0  12.229621  1.495636e+02  3.298547   \n",
       "617    0  18539   4.181818     0.0  12.229621  1.495636e+02  3.298547   \n",
       "618    0  18569   4.090909     0.0  12.259319  1.502909e+02  3.299109   \n",
       "619    0  18599   4.000000     0.0  12.288206  1.510000e+02  3.300460   \n",
       "\n",
       "          kurt    max  min  count    sum   act  \n",
       "590   1.619478  207.0  0.0   11.0  552.0   1.0  \n",
       "591   1.572494  207.0  0.0   11.0  539.0   0.0  \n",
       "592   1.531653  207.0  0.0   11.0  522.0   5.0  \n",
       "593   7.399291  182.0  0.0   11.0  315.0  48.0  \n",
       "594   0.100988   49.0  0.0   11.0  133.0  30.0  \n",
       "595   0.100988   49.0  0.0   11.0  133.0  49.0  \n",
       "596   0.096118   49.0  0.0   11.0  132.0   0.0  \n",
       "597   0.096118   49.0  0.0   11.0  132.0   0.0  \n",
       "598   0.106173   49.0  0.0   11.0  127.0   0.0  \n",
       "599   4.059921   49.0  0.0   11.0   79.0   0.0  \n",
       "600  11.000000   49.0  0.0   11.0   49.0   0.0  \n",
       "601   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "602   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "603   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "604   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "605   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "606   0.000000    0.0  0.0   11.0    0.0   0.0  \n",
       "607  11.000000    1.0  0.0   11.0    1.0   0.0  \n",
       "608   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "609   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "610   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "611   2.037037    1.0  0.0   11.0    2.0   0.0  \n",
       "612   2.037037    1.0  0.0   11.0    2.0   1.0  \n",
       "613   2.610667    2.0  0.0   11.0    4.0   1.0  \n",
       "614   2.610667    2.0  0.0   11.0    4.0   0.0  \n",
       "615   0.976331    2.0  0.0   11.0    5.0   0.0  \n",
       "616  10.911555   41.0  0.0   11.0   46.0   0.0  \n",
       "617  10.911555   41.0  0.0   11.0   46.0   0.0  \n",
       "618  10.913811   41.0  0.0   11.0   45.0   2.0  \n",
       "619  10.919927   41.0  0.0   11.0   44.0   0.0  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[590:620]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cbe2a3a",
   "metadata": {},
   "source": [
    "# First Simple ML model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f6cb1e0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:52.429481Z",
     "start_time": "2022-08-28T13:50:52.332889Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import f1_score, matthews_corrcoef, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6f24142a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:52.764553Z",
     "start_time": "2022-08-28T13:50:52.670887Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7496629213483146"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression()\n",
    "model.fit(X.values[:1000], Y[\"sleep\"].values[:1000])\n",
    "\n",
    "pred = model.predict(X.values[5000:10000])\n",
    "f1_score(Y[\"sleep\"].values[5000:10000], pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d97d9d50",
   "metadata": {},
   "source": [
    "---\n",
    "# Open Parenthesis\n",
    "- Is F1 a good metric to use here?\n",
    "- Is F1 score a good metric in general?\n",
    "\n",
    "See https://bmcgenomics.biomedcentral.com/articles/10.1186/s12864-019-6413-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0b20ef9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7496629213483146"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(Y[\"sleep\"].values[5000:10000], pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b9cfdbe1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:54.171352Z",
     "start_time": "2022-08-28T13:50:54.155824Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49744360902255647"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(~Y[\"sleep\"].values[5000:10000], ~pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "eeba953a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:53.742934Z",
     "start_time": "2022-08-28T13:50:53.682442Z"
    }
   },
   "outputs": [],
   "source": [
    "f1_score?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c4a80dcd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:58.974102Z",
     "start_time": "2022-08-28T13:50:58.966275Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.28078064048670404, 0.28078064048670404)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matthews_corrcoef(~Y[\"sleep\"].values[5000:10000], ~pred), matthews_corrcoef(Y[\"sleep\"].values[5000:10000], pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "543075dc",
   "metadata": {},
   "source": [
    "<img src=\"images/f1.jpg\" width=\"600\" height=\"200\">\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1231c4d",
   "metadata": {},
   "source": [
    "The usual $f_1$ score is soly based on FP, TP and FN (implicitly coming from Precision and Recall):\n",
    "\n",
    "$$ F_1 = \\frac{2 \\times P \\times R}{P + R} $$\n",
    "\n",
    "Matthew's coefficient correlation (Phi correlation)\n",
    "https://en.wikipedia.org/wiki/Phi_coefficient\n",
    "\n",
    "\n",
    "$$ MCC = \\frac{TP \\times TN - FP \\times FN}{\\sqrt{(TP + FP)\\times(TP + FN)\\times(TN + FP)\\times(TN + FN)}} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "354f210b",
   "metadata": {},
   "source": [
    "# Close parenthesis\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "cd0518a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:59.355710Z",
     "start_time": "2022-08-28T13:50:59.345559Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " 1: 1,\n",
       " 2: 2,\n",
       " 3: 3,\n",
       " 4: 4,\n",
       " 5: 0,\n",
       " 6: 1,\n",
       " 7: 2,\n",
       " 8: 3,\n",
       " 9: 4,\n",
       " 10: 0,\n",
       " 11: 1,\n",
       " 12: 2,\n",
       " 13: 3,\n",
       " 14: 4,\n",
       " 15: 0,\n",
       " 16: 1,\n",
       " 17: 2,\n",
       " 18: 3,\n",
       " 19: 4,\n",
       " 20: 0,\n",
       " 21: 1,\n",
       " 22: 2,\n",
       " 23: 3,\n",
       " 24: 4,\n",
       " 25: 0,\n",
       " 26: 1,\n",
       " 27: 2,\n",
       " 28: 3,\n",
       " 29: 4,\n",
       " 30: 0,\n",
       " 31: 1,\n",
       " 32: 2,\n",
       " 33: 3,\n",
       " 34: 4,\n",
       " 35: 0,\n",
       " 36: 1,\n",
       " 37: 2,\n",
       " 38: 3,\n",
       " 39: 4,\n",
       " 40: 0,\n",
       " 41: 1,\n",
       " 42: 2,\n",
       " 43: 3,\n",
       " 44: 4,\n",
       " 45: 0,\n",
       " 46: 1,\n",
       " 47: 2,\n",
       " 48: 3,\n",
       " 49: 4,\n",
       " 50: 0,\n",
       " 51: 1,\n",
       " 52: 2,\n",
       " 53: 3,\n",
       " 54: 4,\n",
       " 55: 0,\n",
       " 56: 1,\n",
       " 57: 2,\n",
       " 58: 3,\n",
       " 59: 4,\n",
       " 60: 0,\n",
       " 61: 1,\n",
       " 62: 2,\n",
       " 63: 3,\n",
       " 64: 4,\n",
       " 65: 0,\n",
       " 66: 1,\n",
       " 67: 2,\n",
       " 68: 3,\n",
       " 69: 4,\n",
       " 70: 0,\n",
       " 71: 1,\n",
       " 72: 2,\n",
       " 73: 3,\n",
       " 74: 4,\n",
       " 75: 0,\n",
       " 76: 1,\n",
       " 77: 2,\n",
       " 78: 3,\n",
       " 79: 4,\n",
       " 80: 0,\n",
       " 81: 1,\n",
       " 82: 2,\n",
       " 83: 3,\n",
       " 84: 4,\n",
       " 85: 0,\n",
       " 86: 1,\n",
       " 87: 2,\n",
       " 88: 3,\n",
       " 89: 4,\n",
       " 90: 0,\n",
       " 91: 1,\n",
       " 92: 2,\n",
       " 93: 3,\n",
       " 94: 4,\n",
       " 95: 0,\n",
       " 96: 1,\n",
       " 97: 2,\n",
       " 98: 3,\n",
       " 99: 4}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrps = 5\n",
    "pid_grp = {}\n",
    "\n",
    "i = 0\n",
    "for pid in X[\"pid\"].unique():\n",
    "    pid_grp[pid] = i\n",
    "    i = (i+1) % ngrps\n",
    "\n",
    "pid_grp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "60adee61",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:50:59.979844Z",
     "start_time": "2022-08-28T13:50:59.872991Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "      <th>grp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>35.120744</td>\n",
       "      <td>1233.466667</td>\n",
       "      <td>2.445916</td>\n",
       "      <td>5.986385</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>13.285714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.515198</td>\n",
       "      <td>1057.238095</td>\n",
       "      <td>2.642249</td>\n",
       "      <td>6.985881</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11.625000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.467489</td>\n",
       "      <td>928.267857</td>\n",
       "      <td>2.824207</td>\n",
       "      <td>7.982049</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>10.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.761954</td>\n",
       "      <td>827.250000</td>\n",
       "      <td>2.995208</td>\n",
       "      <td>8.978607</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>9.300000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>27.313204</td>\n",
       "      <td>746.011111</td>\n",
       "      <td>3.157003</td>\n",
       "      <td>9.975393</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>32.600000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.336078</td>\n",
       "      <td>802.933333</td>\n",
       "      <td>0.034822</td>\n",
       "      <td>-1.788971</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>41.0</td>\n",
       "      <td>27.807373</td>\n",
       "      <td>773.250000</td>\n",
       "      <td>-0.217459</td>\n",
       "      <td>-1.604856</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>33.125000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.261218</td>\n",
       "      <td>798.696429</td>\n",
       "      <td>0.060339</td>\n",
       "      <td>-1.596417</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>29.428571</td>\n",
       "      <td>29.0</td>\n",
       "      <td>28.359931</td>\n",
       "      <td>804.285714</td>\n",
       "      <td>0.446974</td>\n",
       "      <td>-1.109392</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>34.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>27.623661</td>\n",
       "      <td>763.066667</td>\n",
       "      <td>0.175520</td>\n",
       "      <td>-0.958776</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102859 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time       mean  median        std          var      skew  \\\n",
       "0         0     29  15.333333     1.5  35.120744  1233.466667  2.445916   \n",
       "1         0     59  13.285714     1.0  32.515198  1057.238095  2.642249   \n",
       "2         0     89  11.625000     1.0  30.467489   928.267857  2.824207   \n",
       "3         0    119  10.333333     1.0  28.761954   827.250000  2.995208   \n",
       "4         0    149   9.300000     0.5  27.313204   746.011111  3.157003   \n",
       "...     ...    ...        ...     ...        ...          ...       ...   \n",
       "102854   99  27569  32.600000    35.0  28.336078   802.933333  0.034822   \n",
       "102855   99  27599  36.000000    41.0  27.807373   773.250000 -0.217459   \n",
       "102856   99  27629  33.125000    35.0  28.261218   798.696429  0.060339   \n",
       "102857   99  27659  29.428571    29.0  28.359931   804.285714  0.446974   \n",
       "102858   99  27689  34.333333    35.0  27.623661   763.066667  0.175520   \n",
       "\n",
       "            kurt   max  min  count    sum   act  grp  \n",
       "0       5.986385  87.0  0.0    6.0   92.0   2.0    0  \n",
       "1       6.985881  87.0  0.0    7.0   93.0   0.0    0  \n",
       "2       7.982049  87.0  0.0    8.0   93.0   1.0    0  \n",
       "3       8.978607  87.0  0.0    9.0   93.0   2.0    0  \n",
       "4       9.975393  87.0  0.0   10.0   93.0  87.0    0  \n",
       "...          ...   ...  ...    ...    ...   ...  ...  \n",
       "102854 -1.788971  74.0  0.0   10.0  326.0   9.0    4  \n",
       "102855 -1.604856  74.0  0.0    9.0  324.0  29.0    4  \n",
       "102856 -1.596417  74.0  0.0    8.0  265.0  53.0    4  \n",
       "102857 -1.109392  74.0  0.0    7.0  206.0  74.0    4  \n",
       "102858 -0.958776  74.0  0.0    6.0  206.0  41.0    4  \n",
       "\n",
       "[102859 rows x 14 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[\"grp\"] = X[\"pid\"].apply(lambda x: pid_grp[x])\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e63d5996",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:51:01.609705Z",
     "start_time": "2022-08-28T13:51:00.279077Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8078489342928084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.79357922, 0.78114769, 0.81538766, 0.81446029, 0.83466981])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(LogisticRegression(), \n",
    "                         X.values, \n",
    "                         Y[\"sleep\"].values, \n",
    "                         groups=X[\"grp\"], \n",
    "                         scoring=\"f1_weighted\")\n",
    "print(scores.mean())\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "c5481636",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:51:03.177201Z",
     "start_time": "2022-08-28T13:51:01.611484Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5563373079766306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.50685162, 0.49896126, 0.57166581, 0.57312517, 0.63108267])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(LogisticRegression(), \n",
    "                         X.values, \n",
    "                         Y[\"sleep\"].values,\n",
    "                         groups=X[\"grp\"], \n",
    "                         scoring=\"matthews_corrcoef\")\n",
    "print(scores.mean())\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e20f6ff2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:21:16.442381Z",
     "start_time": "2022-08-28T13:21:16.434451Z"
    }
   },
   "source": [
    "# Ideas to improve??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "910541e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-28T13:51:07.400339Z",
     "start_time": "2022-08-28T13:51:03.179137Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5504742449396713\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.49670038, 0.50062669, 0.54590421, 0.57525503, 0.63388492])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "\n",
    "\n",
    "pipe = make_pipeline(StandardScaler(), LogisticRegression())\n",
    "scores = cross_val_score(pipe, X.values, Y[\"sleep\"].values, groups=X[\"grp\"], scoring=\"matthews_corrcoef\")\n",
    "print(scores.mean())\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e68639a",
   "metadata": {},
   "source": [
    "# Comparison with different ML techniques -- hand-out test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "95c0f54f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ids = range(50, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "69355ca2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "      <th>grp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>51982</th>\n",
       "      <td>50</td>\n",
       "      <td>5099</td>\n",
       "      <td>25.454545</td>\n",
       "      <td>7.0</td>\n",
       "      <td>34.737195</td>\n",
       "      <td>1206.672727</td>\n",
       "      <td>1.189953</td>\n",
       "      <td>-0.005073</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51983</th>\n",
       "      <td>50</td>\n",
       "      <td>5129</td>\n",
       "      <td>28.545455</td>\n",
       "      <td>13.0</td>\n",
       "      <td>33.744225</td>\n",
       "      <td>1138.672727</td>\n",
       "      <td>1.015676</td>\n",
       "      <td>-0.202428</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>314.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51984</th>\n",
       "      <td>50</td>\n",
       "      <td>5159</td>\n",
       "      <td>29.090909</td>\n",
       "      <td>13.0</td>\n",
       "      <td>33.281991</td>\n",
       "      <td>1107.690909</td>\n",
       "      <td>1.037608</td>\n",
       "      <td>-0.143160</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>320.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51985</th>\n",
       "      <td>50</td>\n",
       "      <td>5189</td>\n",
       "      <td>34.545455</td>\n",
       "      <td>33.0</td>\n",
       "      <td>32.952583</td>\n",
       "      <td>1085.872727</td>\n",
       "      <td>0.640964</td>\n",
       "      <td>-0.888777</td>\n",
       "      <td>94.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>380.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51986</th>\n",
       "      <td>50</td>\n",
       "      <td>5219</td>\n",
       "      <td>36.727273</td>\n",
       "      <td>33.0</td>\n",
       "      <td>31.183620</td>\n",
       "      <td>972.418182</td>\n",
       "      <td>0.660783</td>\n",
       "      <td>-0.684294</td>\n",
       "      <td>94.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>404.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>99</td>\n",
       "      <td>27569</td>\n",
       "      <td>32.600000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.336078</td>\n",
       "      <td>802.933333</td>\n",
       "      <td>0.034822</td>\n",
       "      <td>-1.788971</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>99</td>\n",
       "      <td>27599</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>41.0</td>\n",
       "      <td>27.807373</td>\n",
       "      <td>773.250000</td>\n",
       "      <td>-0.217459</td>\n",
       "      <td>-1.604856</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>99</td>\n",
       "      <td>27629</td>\n",
       "      <td>33.125000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>28.261218</td>\n",
       "      <td>798.696429</td>\n",
       "      <td>0.060339</td>\n",
       "      <td>-1.596417</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>99</td>\n",
       "      <td>27659</td>\n",
       "      <td>29.428571</td>\n",
       "      <td>29.0</td>\n",
       "      <td>28.359931</td>\n",
       "      <td>804.285714</td>\n",
       "      <td>0.446974</td>\n",
       "      <td>-1.109392</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>99</td>\n",
       "      <td>27689</td>\n",
       "      <td>34.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>27.623661</td>\n",
       "      <td>763.066667</td>\n",
       "      <td>0.175520</td>\n",
       "      <td>-0.958776</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50877 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pid   time       mean  median        std          var      skew  \\\n",
       "51982    50   5099  25.454545     7.0  34.737195  1206.672727  1.189953   \n",
       "51983    50   5129  28.545455    13.0  33.744225  1138.672727  1.015676   \n",
       "51984    50   5159  29.090909    13.0  33.281991  1107.690909  1.037608   \n",
       "51985    50   5189  34.545455    33.0  32.952583  1085.872727  0.640964   \n",
       "51986    50   5219  36.727273    33.0  31.183620   972.418182  0.660783   \n",
       "...     ...    ...        ...     ...        ...          ...       ...   \n",
       "102854   99  27569  32.600000    35.0  28.336078   802.933333  0.034822   \n",
       "102855   99  27599  36.000000    41.0  27.807373   773.250000 -0.217459   \n",
       "102856   99  27629  33.125000    35.0  28.261218   798.696429  0.060339   \n",
       "102857   99  27659  29.428571    29.0  28.359931   804.285714  0.446974   \n",
       "102858   99  27689  34.333333    35.0  27.623661   763.066667  0.175520   \n",
       "\n",
       "            kurt   max  min  count    sum   act  grp  \n",
       "51982  -0.005073  94.0  0.0   11.0  280.0  33.0    0  \n",
       "51983  -0.202428  94.0  0.0   11.0  314.0  13.0    0  \n",
       "51984  -0.143160  94.0  0.0   11.0  320.0   7.0    0  \n",
       "51985  -0.888777  94.0  0.0   11.0  380.0  53.0    0  \n",
       "51986  -0.684294  94.0  1.0   11.0  404.0  79.0    0  \n",
       "...          ...   ...  ...    ...    ...   ...  ...  \n",
       "102854 -1.788971  74.0  0.0   10.0  326.0   9.0    4  \n",
       "102855 -1.604856  74.0  0.0    9.0  324.0  29.0    4  \n",
       "102856 -1.596417  74.0  0.0    8.0  265.0  53.0    4  \n",
       "102857 -1.109392  74.0  0.0    7.0  206.0  74.0    4  \n",
       "102858 -0.958776  74.0  0.0    6.0  206.0  41.0    4  \n",
       "\n",
       "[50877 rows x 14 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = X[X[\"pid\"].isin(test_ids)]\n",
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f975b409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pid</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>51982</th>\n",
       "      <td>1578416</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51983</th>\n",
       "      <td>1578446</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51984</th>\n",
       "      <td>1578476</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51985</th>\n",
       "      <td>1578506</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51986</th>\n",
       "      <td>1578536</td>\n",
       "      <td>50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102854</th>\n",
       "      <td>3116246</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102855</th>\n",
       "      <td>3116276</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102856</th>\n",
       "      <td>3116306</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102857</th>\n",
       "      <td>3116336</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102858</th>\n",
       "      <td>3116366</td>\n",
       "      <td>99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50877 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           time  pid  sleep_phase  sleep\n",
       "51982   1578416   50          0.0  False\n",
       "51983   1578446   50          0.0  False\n",
       "51984   1578476   50          0.0  False\n",
       "51985   1578506   50          0.0  False\n",
       "51986   1578536   50          0.0  False\n",
       "...         ...  ...          ...    ...\n",
       "102854  3116246   99          0.0  False\n",
       "102855  3116276   99          0.0  False\n",
       "102856  3116306   99          0.0  False\n",
       "102857  3116336   99          0.0  False\n",
       "102858  3116366   99          0.0  False\n",
       "\n",
       "[50877 rows x 4 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_test = Y[Y[\"pid\"].isin(test_ids)]\n",
    "Y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "0d1a4dc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>time</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>std</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurt</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>act</th>\n",
       "      <th>grp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>35.120744</td>\n",
       "      <td>1233.466667</td>\n",
       "      <td>2.445916</td>\n",
       "      <td>5.986385</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>13.285714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.515198</td>\n",
       "      <td>1057.238095</td>\n",
       "      <td>2.642249</td>\n",
       "      <td>6.985881</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>11.625000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.467489</td>\n",
       "      <td>928.267857</td>\n",
       "      <td>2.824207</td>\n",
       "      <td>7.982049</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>10.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.761954</td>\n",
       "      <td>827.250000</td>\n",
       "      <td>2.995208</td>\n",
       "      <td>8.978607</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>149</td>\n",
       "      <td>9.300000</td>\n",
       "      <td>0.5</td>\n",
       "      <td>27.313204</td>\n",
       "      <td>746.011111</td>\n",
       "      <td>3.157003</td>\n",
       "      <td>9.975393</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51977</th>\n",
       "      <td>49</td>\n",
       "      <td>31379</td>\n",
       "      <td>3.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.832229</td>\n",
       "      <td>96.672727</td>\n",
       "      <td>3.276594</td>\n",
       "      <td>10.802229</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51978</th>\n",
       "      <td>49</td>\n",
       "      <td>31409</td>\n",
       "      <td>4.454545</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.221190</td>\n",
       "      <td>104.472727</td>\n",
       "      <td>2.656492</td>\n",
       "      <td>7.124992</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51979</th>\n",
       "      <td>49</td>\n",
       "      <td>31439</td>\n",
       "      <td>5.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.133652</td>\n",
       "      <td>102.690909</td>\n",
       "      <td>2.507661</td>\n",
       "      <td>6.525873</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51980</th>\n",
       "      <td>49</td>\n",
       "      <td>31469</td>\n",
       "      <td>9.727273</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.539047</td>\n",
       "      <td>307.618182</td>\n",
       "      <td>1.981308</td>\n",
       "      <td>3.310513</td>\n",
       "      <td>53.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51981</th>\n",
       "      <td>49</td>\n",
       "      <td>31499</td>\n",
       "      <td>16.909091</td>\n",
       "      <td>1.0</td>\n",
       "      <td>26.856860</td>\n",
       "      <td>721.290909</td>\n",
       "      <td>1.648773</td>\n",
       "      <td>1.862526</td>\n",
       "      <td>79.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51982 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pid   time       mean  median        std          var      skew  \\\n",
       "0        0     29  15.333333     1.5  35.120744  1233.466667  2.445916   \n",
       "1        0     59  13.285714     1.0  32.515198  1057.238095  2.642249   \n",
       "2        0     89  11.625000     1.0  30.467489   928.267857  2.824207   \n",
       "3        0    119  10.333333     1.0  28.761954   827.250000  2.995208   \n",
       "4        0    149   9.300000     0.5  27.313204   746.011111  3.157003   \n",
       "...    ...    ...        ...     ...        ...          ...       ...   \n",
       "51977   49  31379   3.454545     0.0   9.832229    96.672727  3.276594   \n",
       "51978   49  31409   4.454545     0.0  10.221190   104.472727  2.656492   \n",
       "51979   49  31439   5.090909     0.0  10.133652   102.690909  2.507661   \n",
       "51980   49  31469   9.727273     0.0  17.539047   307.618182  1.981308   \n",
       "51981   49  31499  16.909091     1.0  26.856860   721.290909  1.648773   \n",
       "\n",
       "            kurt   max  min  count    sum   act  grp  \n",
       "0       5.986385  87.0  0.0    6.0   92.0   2.0    0  \n",
       "1       6.985881  87.0  0.0    7.0   93.0   0.0    0  \n",
       "2       7.982049  87.0  0.0    8.0   93.0   1.0    0  \n",
       "3       8.978607  87.0  0.0    9.0   93.0   2.0    0  \n",
       "4       9.975393  87.0  0.0   10.0   93.0  87.0    0  \n",
       "...          ...   ...  ...    ...    ...   ...  ...  \n",
       "51977  10.802229  33.0  0.0   11.0   38.0   0.0    4  \n",
       "51978   7.124992  33.0  0.0   11.0   49.0   0.0    4  \n",
       "51979   6.525873  33.0  0.0   11.0   56.0   0.0    4  \n",
       "51980   3.310513  53.0  0.0   11.0  107.0   0.0    4  \n",
       "51981   1.862526  79.0  0.0   11.0  186.0   1.0    4  \n",
       "\n",
       "[51982 rows x 14 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The ones that are NOT in the training set are in the test set\n",
    "X_train = X[~X[\"pid\"].isin(test_ids)]\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "498ff94a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pid</th>\n",
       "      <th>sleep_phase</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>149</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51977</th>\n",
       "      <td>1573108</td>\n",
       "      <td>49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51978</th>\n",
       "      <td>1573138</td>\n",
       "      <td>49</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51979</th>\n",
       "      <td>1573168</td>\n",
       "      <td>49</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51980</th>\n",
       "      <td>1573198</td>\n",
       "      <td>49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51981</th>\n",
       "      <td>1573228</td>\n",
       "      <td>49</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51982 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          time  pid  sleep_phase  sleep\n",
       "0           29    0          0.0  False\n",
       "1           59    0          0.0  False\n",
       "2           89    0          0.0  False\n",
       "3          119    0          0.0  False\n",
       "4          149    0          0.0  False\n",
       "...        ...  ...          ...    ...\n",
       "51977  1573108   49          0.0  False\n",
       "51978  1573138   49          2.0   True\n",
       "51979  1573168   49          2.0   True\n",
       "51980  1573198   49          0.0  False\n",
       "51981  1573228   49          1.0   True\n",
       "\n",
       "[51982 rows x 4 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train = Y[~Y[\"pid\"].isin(test_ids)]\n",
    "Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e4413046",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;logisticregression&#x27;, LogisticRegression())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;logisticregression&#x27;, LogisticRegression())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('logisticregression', LogisticRegression())])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = make_pipeline(StandardScaler(), LogisticRegression())\n",
    "pipe = make_pipeline(LogisticRegression())\n",
    "pipe.fit(X_train[['mean', 'median', 'std', 'var', 'skew', 'kurt', 'max', 'min', 'count', 'sum', 'act']].values,\n",
    "         Y_train[\"sleep\"].values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0beb5cd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScale + LR: Acc: 0.832, F1 0.816, MCC: 0.571\n"
     ]
    }
   ],
   "source": [
    "y_hat = pipe.predict(X_test[['mean', 'median', 'std', 'var', 'skew', 'kurt', 'max', 'min', 'count', 'sum', 'act']].values)\n",
    "acc = accuracy_score(Y_test[\"sleep\"].values, y_hat)\n",
    "f1 = f1_score(Y_test[\"sleep\"].values, y_hat, average=\"weighted\")\n",
    "mcc = matthews_corrcoef(Y_test[\"sleep\"].values, y_hat)\n",
    "\n",
    "print(\"StandardScale + LR: Acc: %.3f, F1 %.3f, MCC: %.3f\" % (acc, f1, mcc))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7979788c",
   "metadata": {},
   "source": [
    "#### What if we use only the Logistic Regression model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ae181991",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Only LR: Acc: 0.831, F1 0.815, MCC: 0.568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/joao/miniforge3/envs/tutorial/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "pipe = LogisticRegression()\n",
    "pipe.fit(X_train[['mean', 'median', 'std', 'var', 'skew', 'kurt', 'max', 'min', 'count', 'sum', 'act']].values,\n",
    "         Y_train[\"sleep\"].values)\n",
    "y_hat = pipe.predict(X_test[['mean', 'median', 'std', 'var', 'skew', 'kurt', 'max', 'min', 'count', 'sum', 'act']].values)\n",
    "acc = accuracy_score(Y_test[\"sleep\"].values, y_hat)\n",
    "f1 = f1_score(Y_test[\"sleep\"].values, y_hat, average=\"weighted\")\n",
    "mcc = matthews_corrcoef(Y_test[\"sleep\"].values, y_hat)\n",
    "print(\"Only LR: Acc: %.3f, F1 %.3f, MCC: %.3f\" % (acc, f1, mcc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7488facc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}