approach 2 - best scoring approach - optuna tuned xgboost column transformer data preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports and configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d19392cd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-09T23:18:21.372953Z",
     "iopub.status.busy": "2023-06-09T23:18:21.372578Z",
     "iopub.status.idle": "2023-06-09T23:18:25.356673Z",
     "shell.execute_reply": "2023-06-09T23:18:25.355252Z"
    },
    "papermill": {
     "duration": 3.993706,
     "end_time": "2023-06-09T23:18:25.359557",
     "exception": false,
     "start_time": "2023-06-09T23:18:21.365851",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd, xgboost as xgb, lightgbm as lgbm, sklearn, numpy as np\n",
    "import optuna\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.utils.class_weight import compute_sample_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a61b3404",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-09T23:18:25.372608Z",
     "iopub.status.busy": "2023-06-09T23:18:25.372152Z",
     "iopub.status.idle": "2023-06-09T23:18:25.377233Z",
     "shell.execute_reply": "2023-06-09T23:18:25.376211Z"
    },
    "papermill": {
     "duration": 0.01435,
     "end_time": "2023-06-09T23:18:25.379611",
     "exception": false,
     "start_time": "2023-06-09T23:18:25.365261",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Allow visualizing as many columns and rows as requested.\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data retrieval and preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9c977b09",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-09T23:18:25.393051Z",
     "iopub.status.busy": "2023-06-09T23:18:25.392686Z",
     "iopub.status.idle": "2023-06-09T23:18:25.398096Z",
     "shell.execute_reply": "2023-06-09T23:18:25.397022Z"
    },
    "papermill": {
     "duration": 0.014767,
     "end_time": "2023-06-09T23:18:25.400369",
     "exception": false,
     "start_time": "2023-06-09T23:18:25.385602",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_train_and_prediction_request():\n",
    "    train = pd.read_csv('../input/icr-identify-age-related-conditions/train.csv', index_col = 'Id')\n",
    "    prediction_request = pd.read_csv('../input/icr-identify-age-related-conditions/test.csv', index_col = 'Id')\n",
    "    return train, prediction_request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6fc89ecd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-09T23:18:25.452457Z",
     "iopub.status.busy": "2023-06-09T23:18:25.452065Z",
     "iopub.status.idle": "2023-06-09T23:18:25.656986Z",
     "shell.execute_reply": "2023-06-09T23:18:25.655739Z"
    },
    "papermill": {
     "duration": 0.215137,
     "end_time": "2023-06-09T23:18:25.659871",
     "exception": false,
     "start_time": "2023-06-09T23:18:25.444734",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EJ_A</th>\n",
       "      <th>EJ_B</th>\n",
       "      <th>AB</th>\n",
       "      <th>AF</th>\n",
       "      <th>AH</th>\n",
       "      <th>AM</th>\n",
       "      <th>AR</th>\n",
       "      <th>AX</th>\n",
       "      <th>AY</th>\n",
       "      <th>AZ</th>\n",
       "      <th>BC</th>\n",
       "      <th>BD</th>\n",
       "      <th>BN</th>\n",
       "      <th>BP</th>\n",
       "      <th>BQ</th>\n",
       "      <th>BR</th>\n",
       "      <th>BZ</th>\n",
       "      <th>CB</th>\n",
       "      <th>CC</th>\n",
       "      <th>CD</th>\n",
       "      <th>CF</th>\n",
       "      <th>CH</th>\n",
       "      <th>CL</th>\n",
       "      <th>CR</th>\n",
       "      <th>CS</th>\n",
       "      <th>CU</th>\n",
       "      <th>CW</th>\n",
       "      <th>DA</th>\n",
       "      <th>DE</th>\n",
       "      <th>DF</th>\n",
       "      <th>DH</th>\n",
       "      <th>DI</th>\n",
       "      <th>DL</th>\n",
       "      <th>DN</th>\n",
       "      <th>DU</th>\n",
       "      <th>DV</th>\n",
       "      <th>DY</th>\n",
       "      <th>EB</th>\n",
       "      <th>EE</th>\n",
       "      <th>EG</th>\n",
       "      <th>EH</th>\n",
       "      <th>EL</th>\n",
       "      <th>EP</th>\n",
       "      <th>EU</th>\n",
       "      <th>FC</th>\n",
       "      <th>FD</th>\n",
       "      <th>FE</th>\n",
       "      <th>FI</th>\n",
       "      <th>FL</th>\n",
       "      <th>FR</th>\n",
       "      <th>FS</th>\n",
       "      <th>GB</th>\n",
       "      <th>GE</th>\n",
       "      <th>GF</th>\n",
       "      <th>GH</th>\n",
       "      <th>GI</th>\n",
       "      <th>GL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>000ff2bfdfe9</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.021082</td>\n",
       "      <td>0.102347</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030632</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.180337</td>\n",
       "      <td>0.002958</td>\n",
       "      <td>0.047364</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>0.043240</td>\n",
       "      <td>0.440929</td>\n",
       "      <td>0.004312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015372</td>\n",
       "      <td>0.098469</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021656</td>\n",
       "      <td>0.091892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.241834</td>\n",
       "      <td>0.507476</td>\n",
       "      <td>0.305651</td>\n",
       "      <td>0.125554</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.238606</td>\n",
       "      <td>0.029338</td>\n",
       "      <td>0.234167</td>\n",
       "      <td>0.412931</td>\n",
       "      <td>0.032880</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.147697</td>\n",
       "      <td>0.026299</td>\n",
       "      <td>0.094302</td>\n",
       "      <td>0.041505</td>\n",
       "      <td>0.022225</td>\n",
       "      <td>0.245682</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001939</td>\n",
       "      <td>0.006316</td>\n",
       "      <td>0.052697</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.051720</td>\n",
       "      <td>0.000998</td>\n",
       "      <td>0.000866</td>\n",
       "      <td>0.054959</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013846</td>\n",
       "      <td>0.176983</td>\n",
       "      <td>0.362261</td>\n",
       "      <td>0.005425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>007255e47698</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.010541</td>\n",
       "      <td>0.027589</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.053864</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.078048</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.284500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.074042</td>\n",
       "      <td>0.490909</td>\n",
       "      <td>0.034915</td>\n",
       "      <td>0.039100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007873</td>\n",
       "      <td>0.078406</td>\n",
       "      <td>0.044646</td>\n",
       "      <td>0.027807</td>\n",
       "      <td>0.127928</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>0.353002</td>\n",
       "      <td>0.057157</td>\n",
       "      <td>0.253295</td>\n",
       "      <td>0.529577</td>\n",
       "      <td>0.314082</td>\n",
       "      <td>0.068953</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.316354</td>\n",
       "      <td>0.050913</td>\n",
       "      <td>0.207033</td>\n",
       "      <td>0.552387</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.108335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031732</td>\n",
       "      <td>0.030797</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.017144</td>\n",
       "      <td>0.007454</td>\n",
       "      <td>0.003189</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.036862</td>\n",
       "      <td>0.209978</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.016014</td>\n",
       "      <td>0.039418</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.194527</td>\n",
       "      <td>0.274495</td>\n",
       "      <td>0.164135</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>013f2bd269f5</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.063949</td>\n",
       "      <td>0.085715</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.046519</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.160575</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.265013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.067011</td>\n",
       "      <td>0.854545</td>\n",
       "      <td>0.023597</td>\n",
       "      <td>0.634957</td>\n",
       "      <td>0.002405</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008882</td>\n",
       "      <td>0.081244</td>\n",
       "      <td>0.102545</td>\n",
       "      <td>0.024273</td>\n",
       "      <td>0.149550</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.212468</td>\n",
       "      <td>0.100648</td>\n",
       "      <td>0.181089</td>\n",
       "      <td>0.250979</td>\n",
       "      <td>0.314187</td>\n",
       "      <td>0.138061</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166220</td>\n",
       "      <td>0.060493</td>\n",
       "      <td>0.174504</td>\n",
       "      <td>0.384532</td>\n",
       "      <td>0.007959</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.237922</td>\n",
       "      <td>0.032069</td>\n",
       "      <td>0.435754</td>\n",
       "      <td>0.043532</td>\n",
       "      <td>0.008790</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000240</td>\n",
       "      <td>0.071672</td>\n",
       "      <td>0.005353</td>\n",
       "      <td>0.047831</td>\n",
       "      <td>0.249274</td>\n",
       "      <td>0.054706</td>\n",
       "      <td>0.000385</td>\n",
       "      <td>0.036139</td>\n",
       "      <td>0.250424</td>\n",
       "      <td>0.011229</td>\n",
       "      <td>0.095035</td>\n",
       "      <td>0.258994</td>\n",
       "      <td>0.180218</td>\n",
       "      <td>0.008910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              EJ_A  EJ_B        AB        AF   AH        AM   AR        AX  \\\n",
       "Id                                                                           \n",
       "000ff2bfdfe9     0     1  0.021082  0.102347  0.0  0.030632  0.0  0.000000   \n",
       "007255e47698     1     0  0.010541  0.027589  0.0  0.053864  0.0  0.078048   \n",
       "013f2bd269f5     0     1  0.063949  0.085715  0.0  0.046519  0.0  0.160575   \n",
       "\n",
       "               AY        AZ        BC       BD         BN        BP        BQ  \\\n",
       "Id                                                                              \n",
       "000ff2bfdfe9  0.0  0.180337  0.002958  0.047364  0.654545  0.043240  0.440929   \n",
       "007255e47698  0.0  0.284500  0.000000  0.074042  0.490909  0.034915  0.039100   \n",
       "013f2bd269f5  0.0  0.265013  0.000000  0.067011  0.854545  0.023597  0.634957   \n",
       "\n",
       "                    BR   BZ        CB        CC       CD         CF        CH  \\\n",
       "Id                                                                              \n",
       "000ff2bfdfe9  0.004312  0.0  0.015372  0.098469  0.000000  0.021656  0.091892   \n",
       "007255e47698  0.000000  0.0  0.007873  0.078406  0.044646  0.027807  0.127928   \n",
       "013f2bd269f5  0.002405  0.0  0.008882  0.081244  0.102545  0.024273  0.149550   \n",
       "\n",
       "                    CL        CR        CS        CU       CW         DA  \\\n",
       "Id                                                                         \n",
       "000ff2bfdfe9  0.000000  0.000000  0.000000  0.241834  0.507476  0.305651   \n",
       "007255e47698  0.002077  0.353002  0.057157  0.253295  0.529577  0.314082   \n",
       "013f2bd269f5  0.000000  0.212468  0.100648  0.181089  0.250979  0.314187   \n",
       "\n",
       "                    DE   DF        DH        DI        DL        DN        DU  \\\n",
       "Id                                                                              \n",
       "000ff2bfdfe9  0.125554  0.0  0.238606  0.029338  0.234167  0.412931  0.032880   \n",
       "007255e47698  0.068953  0.0  0.316354  0.050913  0.207033  0.552387  0.000000   \n",
       "013f2bd269f5  0.138061  0.0  0.166220  0.060493  0.174504  0.384532  0.007959   \n",
       "\n",
       "               DV        DY        EB        EE        EG        EH        EL  \\\n",
       "Id                                                                              \n",
       "000ff2bfdfe9  0.0  0.147697  0.026299  0.094302  0.041505  0.022225  0.245682   \n",
       "007255e47698  0.0  0.108335  0.000000  0.031732  0.030797  0.000000  1.000000   \n",
       "013f2bd269f5  0.0  0.237922  0.032069  0.435754  0.043532  0.008790  1.000000   \n",
       "\n",
       "                    EP        EU        FC       FD         FE        FI  \\\n",
       "Id                                                                         \n",
       "000ff2bfdfe9  0.000000  0.000000  0.001939  0.006316  0.052697  0.000000   \n",
       "007255e47698  0.017144  0.007454  0.003189  0.000000  0.036862  0.209978   \n",
       "013f2bd269f5  0.000000  0.000240  0.071672  0.005353  0.047831  0.249274   \n",
       "\n",
       "                    FL        FR        FS        GB        GE        GF  \\\n",
       "Id                                                                         \n",
       "000ff2bfdfe9  0.051720  0.000998  0.000866  0.054959  0.000000  0.013846   \n",
       "007255e47698  0.000000  0.000000  0.016014  0.039418  0.000000  0.194527   \n",
       "013f2bd269f5  0.054706  0.000385  0.036139  0.250424  0.011229  0.095035   \n",
       "\n",
       "                    GH        GI        GL  \n",
       "Id                                          \n",
       "000ff2bfdfe9  0.176983  0.362261  0.005425  \n",
       "007255e47698  0.274495  0.164135  1.000000  \n",
       "013f2bd269f5  0.258994  0.180218  0.008910  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EJ_A</th>\n",
       "      <th>EJ_B</th>\n",
       "      <th>AB</th>\n",
       "      <th>AF</th>\n",
       "      <th>AH</th>\n",
       "      <th>AM</th>\n",
       "      <th>AR</th>\n",
       "      <th>AX</th>\n",
       "      <th>AY</th>\n",
       "      <th>AZ</th>\n",
       "      <th>BC</th>\n",
       "      <th>BD</th>\n",
       "      <th>BN</th>\n",
       "      <th>BP</th>\n",
       "      <th>BQ</th>\n",
       "      <th>BR</th>\n",
       "      <th>BZ</th>\n",
       "      <th>CB</th>\n",
       "      <th>CC</th>\n",
       "      <th>CD</th>\n",
       "      <th>CF</th>\n",
       "      <th>CH</th>\n",
       "      <th>CL</th>\n",
       "      <th>CR</th>\n",
       "      <th>CS</th>\n",
       "      <th>CU</th>\n",
       "      <th>CW</th>\n",
       "      <th>DA</th>\n",
       "      <th>DE</th>\n",
       "      <th>DF</th>\n",
       "      <th>DH</th>\n",
       "      <th>DI</th>\n",
       "      <th>DL</th>\n",
       "      <th>DN</th>\n",
       "      <th>DU</th>\n",
       "      <th>DV</th>\n",
       "      <th>DY</th>\n",
       "      <th>EB</th>\n",
       "      <th>EE</th>\n",
       "      <th>EG</th>\n",
       "      <th>EH</th>\n",
       "      <th>EL</th>\n",
       "      <th>EP</th>\n",
       "      <th>EU</th>\n",
       "      <th>FC</th>\n",
       "      <th>FD</th>\n",
       "      <th>FE</th>\n",
       "      <th>FI</th>\n",
       "      <th>FL</th>\n",
       "      <th>FR</th>\n",
       "      <th>FS</th>\n",
       "      <th>GB</th>\n",
       "      <th>GE</th>\n",
       "      <th>GF</th>\n",
       "      <th>GH</th>\n",
       "      <th>GI</th>\n",
       "      <th>GL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00eed32682bb</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.013352</td>\n",
       "      <td>-0.006759</td>\n",
       "      <td>-0.046687</td>\n",
       "      <td>-0.005065</td>\n",
       "      <td>-0.047649</td>\n",
       "      <td>-0.018628</td>\n",
       "      <td>-0.002486</td>\n",
       "      <td>-0.095483</td>\n",
       "      <td>-0.000841</td>\n",
       "      <td>-0.032971</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.030717</td>\n",
       "      <td>-0.003877</td>\n",
       "      <td>-0.000286</td>\n",
       "      <td>-0.005166</td>\n",
       "      <td>-0.005533</td>\n",
       "      <td>-0.04505</td>\n",
       "      <td>-0.038331</td>\n",
       "      <td>-0.002549</td>\n",
       "      <td>-0.014414</td>\n",
       "      <td>-0.034279</td>\n",
       "      <td>-0.023305</td>\n",
       "      <td>-0.054234</td>\n",
       "      <td>-0.028653</td>\n",
       "      <td>-0.122291</td>\n",
       "      <td>-0.033951</td>\n",
       "      <td>-0.017413</td>\n",
       "      <td>-0.006338</td>\n",
       "      <td>-0.040214</td>\n",
       "      <td>-0.060906</td>\n",
       "      <td>-0.032751</td>\n",
       "      <td>-0.112266</td>\n",
       "      <td>-0.000034</td>\n",
       "      <td>-0.074332</td>\n",
       "      <td>-0.005306</td>\n",
       "      <td>-0.054718</td>\n",
       "      <td>-0.015866</td>\n",
       "      <td>-0.006174</td>\n",
       "      <td>-0.000071</td>\n",
       "      <td>-0.052007</td>\n",
       "      <td>-0.079717</td>\n",
       "      <td>-0.000589</td>\n",
       "      <td>-0.002492</td>\n",
       "      <td>-0.000188</td>\n",
       "      <td>-0.011034</td>\n",
       "      <td>-0.111054</td>\n",
       "      <td>-0.001257</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>-0.002164</td>\n",
       "      <td>-0.031153</td>\n",
       "      <td>-0.050964</td>\n",
       "      <td>-0.000091</td>\n",
       "      <td>-0.131415</td>\n",
       "      <td>-0.004717</td>\n",
       "      <td>-0.000051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>010ebe33f668</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.013352</td>\n",
       "      <td>-0.006759</td>\n",
       "      <td>-0.046687</td>\n",
       "      <td>-0.005065</td>\n",
       "      <td>-0.047649</td>\n",
       "      <td>-0.018628</td>\n",
       "      <td>-0.002486</td>\n",
       "      <td>-0.095483</td>\n",
       "      <td>-0.000841</td>\n",
       "      <td>-0.032971</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.030717</td>\n",
       "      <td>-0.003877</td>\n",
       "      <td>-0.000286</td>\n",
       "      <td>-0.005166</td>\n",
       "      <td>-0.005533</td>\n",
       "      <td>-0.04505</td>\n",
       "      <td>-0.038331</td>\n",
       "      <td>-0.002549</td>\n",
       "      <td>-0.014414</td>\n",
       "      <td>-0.034279</td>\n",
       "      <td>-0.023305</td>\n",
       "      <td>-0.054234</td>\n",
       "      <td>-0.028653</td>\n",
       "      <td>-0.122291</td>\n",
       "      <td>-0.033951</td>\n",
       "      <td>-0.017413</td>\n",
       "      <td>-0.006338</td>\n",
       "      <td>-0.040214</td>\n",
       "      <td>-0.060906</td>\n",
       "      <td>-0.032751</td>\n",
       "      <td>-0.112266</td>\n",
       "      <td>-0.000034</td>\n",
       "      <td>-0.074332</td>\n",
       "      <td>-0.005306</td>\n",
       "      <td>-0.054718</td>\n",
       "      <td>-0.015866</td>\n",
       "      <td>-0.006174</td>\n",
       "      <td>-0.000071</td>\n",
       "      <td>-0.052007</td>\n",
       "      <td>-0.079717</td>\n",
       "      <td>-0.000589</td>\n",
       "      <td>-0.002492</td>\n",
       "      <td>-0.000188</td>\n",
       "      <td>-0.011034</td>\n",
       "      <td>-0.111054</td>\n",
       "      <td>-0.001257</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>-0.002164</td>\n",
       "      <td>-0.031153</td>\n",
       "      <td>-0.050964</td>\n",
       "      <td>-0.000091</td>\n",
       "      <td>-0.131415</td>\n",
       "      <td>-0.004717</td>\n",
       "      <td>-0.000051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02fa521e1838</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.013352</td>\n",
       "      <td>-0.006759</td>\n",
       "      <td>-0.046687</td>\n",
       "      <td>-0.005065</td>\n",
       "      <td>-0.047649</td>\n",
       "      <td>-0.018628</td>\n",
       "      <td>-0.002486</td>\n",
       "      <td>-0.095483</td>\n",
       "      <td>-0.000841</td>\n",
       "      <td>-0.032971</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.030717</td>\n",
       "      <td>-0.003877</td>\n",
       "      <td>-0.000286</td>\n",
       "      <td>-0.005166</td>\n",
       "      <td>-0.005533</td>\n",
       "      <td>-0.04505</td>\n",
       "      <td>-0.038331</td>\n",
       "      <td>-0.002549</td>\n",
       "      <td>-0.014414</td>\n",
       "      <td>-0.034279</td>\n",
       "      <td>-0.023305</td>\n",
       "      <td>-0.054234</td>\n",
       "      <td>-0.028653</td>\n",
       "      <td>-0.122291</td>\n",
       "      <td>-0.033951</td>\n",
       "      <td>-0.017413</td>\n",
       "      <td>-0.006338</td>\n",
       "      <td>-0.040214</td>\n",
       "      <td>-0.060906</td>\n",
       "      <td>-0.032751</td>\n",
       "      <td>-0.112266</td>\n",
       "      <td>-0.000034</td>\n",
       "      <td>-0.074332</td>\n",
       "      <td>-0.005306</td>\n",
       "      <td>-0.054718</td>\n",
       "      <td>-0.015866</td>\n",
       "      <td>-0.006174</td>\n",
       "      <td>-0.000071</td>\n",
       "      <td>-0.052007</td>\n",
       "      <td>-0.079717</td>\n",
       "      <td>-0.000589</td>\n",
       "      <td>-0.002492</td>\n",
       "      <td>-0.000188</td>\n",
       "      <td>-0.011034</td>\n",
       "      <td>-0.111054</td>\n",
       "      <td>-0.001257</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>-0.002164</td>\n",
       "      <td>-0.031153</td>\n",
       "      <td>-0.050964</td>\n",
       "      <td>-0.000091</td>\n",
       "      <td>-0.131415</td>\n",
       "      <td>-0.004717</td>\n",
       "      <td>-0.000051</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              EJ_A  EJ_B        AB        AF        AH        AM        AR  \\\n",
       "Id                                                                           \n",
       "00eed32682bb     1     0 -0.013352 -0.006759 -0.046687 -0.005065 -0.047649   \n",
       "010ebe33f668     1     0 -0.013352 -0.006759 -0.046687 -0.005065 -0.047649   \n",
       "02fa521e1838     1     0 -0.013352 -0.006759 -0.046687 -0.005065 -0.047649   \n",
       "\n",
       "                    AX        AY        AZ        BC       BD         BN  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.018628 -0.002486 -0.095483 -0.000841 -0.032971 -0.509091   \n",
       "010ebe33f668 -0.018628 -0.002486 -0.095483 -0.000841 -0.032971 -0.509091   \n",
       "02fa521e1838 -0.018628 -0.002486 -0.095483 -0.000841 -0.032971 -0.509091   \n",
       "\n",
       "                    BP        BQ        BR        BZ        CB       CC  \\\n",
       "Id                                                                        \n",
       "00eed32682bb -0.030717 -0.003877 -0.000286 -0.005166 -0.005533 -0.04505   \n",
       "010ebe33f668 -0.030717 -0.003877 -0.000286 -0.005166 -0.005533 -0.04505   \n",
       "02fa521e1838 -0.030717 -0.003877 -0.000286 -0.005166 -0.005533 -0.04505   \n",
       "\n",
       "                   CD         CF        CH        CL        CR        CS  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.038331 -0.002549 -0.014414 -0.034279 -0.023305 -0.054234   \n",
       "010ebe33f668 -0.038331 -0.002549 -0.014414 -0.034279 -0.023305 -0.054234   \n",
       "02fa521e1838 -0.038331 -0.002549 -0.014414 -0.034279 -0.023305 -0.054234   \n",
       "\n",
       "                    CU       CW         DA        DE        DF        DH  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.028653 -0.122291 -0.033951 -0.017413 -0.006338 -0.040214   \n",
       "010ebe33f668 -0.028653 -0.122291 -0.033951 -0.017413 -0.006338 -0.040214   \n",
       "02fa521e1838 -0.028653 -0.122291 -0.033951 -0.017413 -0.006338 -0.040214   \n",
       "\n",
       "                    DI        DL        DN        DU        DV        DY  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.060906 -0.032751 -0.112266 -0.000034 -0.074332 -0.005306   \n",
       "010ebe33f668 -0.060906 -0.032751 -0.112266 -0.000034 -0.074332 -0.005306   \n",
       "02fa521e1838 -0.060906 -0.032751 -0.112266 -0.000034 -0.074332 -0.005306   \n",
       "\n",
       "                    EB        EE        EG        EH        EL        EP  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.054718 -0.015866 -0.006174 -0.000071 -0.052007 -0.079717   \n",
       "010ebe33f668 -0.054718 -0.015866 -0.006174 -0.000071 -0.052007 -0.079717   \n",
       "02fa521e1838 -0.054718 -0.015866 -0.006174 -0.000071 -0.052007 -0.079717   \n",
       "\n",
       "                    EU        FC       FD         FE        FI        FL  \\\n",
       "Id                                                                         \n",
       "00eed32682bb -0.000589 -0.002492 -0.000188 -0.011034 -0.111054 -0.001257   \n",
       "010ebe33f668 -0.000589 -0.002492 -0.000188 -0.011034 -0.111054 -0.001257   \n",
       "02fa521e1838 -0.000589 -0.002492 -0.000188 -0.011034 -0.111054 -0.001257   \n",
       "\n",
       "                  FR        FS        GB        GE        GF        GH  \\\n",
       "Id                                                                       \n",
       "00eed32682bb -0.0004 -0.002164 -0.031153 -0.050964 -0.000091 -0.131415   \n",
       "010ebe33f668 -0.0004 -0.002164 -0.031153 -0.050964 -0.000091 -0.131415   \n",
       "02fa521e1838 -0.0004 -0.002164 -0.031153 -0.050964 -0.000091 -0.131415   \n",
       "\n",
       "                    GI        GL  \n",
       "Id                                \n",
       "00eed32682bb -0.004717 -0.000051  \n",
       "010ebe33f668 -0.004717 -0.000051  \n",
       "02fa521e1838 -0.004717 -0.000051  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train, prediction_request = read_train_and_prediction_request()\n",
    "\n",
    "x, y = train.drop(columns='Class'), train['Class'] #Labels cannot be handled by column transformers. https://scikit-learn.org/stable/modules/preprocessing_targets.html https://scikit-learn.org/stable/modules/generated/sklearn.compose.TransformedTargetRegressor.html\n",
    "numerical_column_names = x.select_dtypes(include=np.number).columns.tolist()\n",
    "\n",
    "column_transformer = ColumnTransformer(\n",
    "    transformers=[\n",
    "        #One hot encodes using one column per category. Doesn't need handling of missing values prior, but will error if unknown values are passed.\n",
    "        ('one_hot_encode_categories', OneHotEncoder(sparse_output=False, dtype='int'), ['EJ']),\n",
    "        ('rescale_numericals_from_0_to_1', sklearn.preprocessing.MinMaxScaler(), numerical_column_names),\n",
    "    ],\n",
    "    remainder='passthrough',\n",
    "    verbose_feature_names_out = False #Prevent the transformer from pointlessly prefixing all column names.\n",
    ")\n",
    "column_transformer.set_output(transform='pandas')\n",
    "\n",
    "x, prediction_request = column_transformer.fit_transform(x), column_transformer.transform(prediction_request)\n",
    "\n",
    "display(x.head(3))\n",
    "display(prediction_request.head(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model creation, training, and submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8d6a555",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-06-09T23:18:25.675682Z",
     "iopub.status.busy": "2023-06-09T23:18:25.675293Z",
     "iopub.status.idle": "2023-06-09T23:18:26.070657Z",
     "shell.execute_reply": "2023-06-09T23:18:26.069299Z"
    },
    "papermill": {
     "duration": 0.407861,
     "end_time": "2023-06-09T23:18:26.074561",
     "exception": false,
     "start_time": "2023-06-09T23:18:25.666700",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class_0</th>\n",
       "      <th>class_1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00eed32682bb</th>\n",
       "      <td>0.588577</td>\n",
       "      <td>0.411423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>010ebe33f668</th>\n",
       "      <td>0.588577</td>\n",
       "      <td>0.411423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02fa521e1838</th>\n",
       "      <td>0.588577</td>\n",
       "      <td>0.411423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>040e15f562a2</th>\n",
       "      <td>0.588577</td>\n",
       "      <td>0.411423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>046e85c7cc7f</th>\n",
       "      <td>0.588577</td>\n",
       "      <td>0.411423</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               class_0   class_1\n",
       "Id                              \n",
       "00eed32682bb  0.588577  0.411423\n",
       "010ebe33f668  0.588577  0.411423\n",
       "02fa521e1838  0.588577  0.411423\n",
       "040e15f562a2  0.588577  0.411423\n",
       "046e85c7cc7f  0.588577  0.411423"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#These parameters were obtained by optuna hyperparameter tuning with stratified k-fold crossvalidation\n",
    "model_creation_params= {\n",
    "   'verbosity': 0,\n",
    "   'nthread': None,\n",
    "   'objective': 'binary:logistic',\n",
    "   'eval_metric': 'logloss',\n",
    "   'booster': 'gbtree',\n",
    "   'alpha': 0.00040510453065869997,\n",
    "   'lambda': 3.198576520498575e-05,\n",
    "   'learning_rate': 0.05538695716630914,\n",
    "   'n_estimators': 250,\n",
    "   'random_state': 42,\n",
    "   'eta': 0.017670343976603234,\n",
    "   'gamma': 6.307111169468931e-07,\n",
    "   'max_depth': 5,\n",
    "   'min_child_weight': 5.734858377948545,\n",
    "   'max_delta_step': 3.2806533102798117,\n",
    "   'subsample': 0.4245697631042182,\n",
    "   'sampling_method': 'uniform',\n",
    "   'colsample_bytree': 0.9279384240066151,\n",
    "   'colsample_bylevel': 0.9559142848417146,\n",
    "   'colsample_bynode': 0.6555824396664807,\n",
    "   'tree_method': 'exact',\n",
    "   'scale_pos_weight': 0.9547224624671352\n",
    "}\n",
    "\n",
    "clf1 = xgb.XGBClassifier(**model_creation_params)\n",
    "clf1.fit(x,y,sample_weight=compute_sample_weight('balanced', y))\n",
    "predictions1 = pd.DataFrame(clf1.predict_proba(prediction_request))\n",
    "\n",
    "submission=pd.DataFrame(\n",
    "                    {'class_0':predictions1[0],\n",
    "                    'class_1':predictions1[1]},\n",
    "                  ).set_index(prediction_request.index)\n",
    "display(submission)\n",
    "submission.to_csv('submission.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 20.019802,
   "end_time": "2023-06-09T23:18:27.423284",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-06-09T23:18:07.403482",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}