train-from-file-generic-clean-pixel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: efficientnet in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (1.1.0)\n",
      "Requirement already satisfied: tqdm in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (4.47.0)\n",
      "Requirement already satisfied: keras-applications<=1.0.8,>=1.0.7 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from efficientnet) (1.0.8)\n",
      "Requirement already satisfied: scikit-image in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from efficientnet) (0.17.2)\n",
      "Requirement already satisfied: h5py in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (2.10.0)\n",
      "Requirement already satisfied: numpy>=1.9.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (1.18.5)\n",
      "Requirement already satisfied: scipy>=1.0.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (1.4.1)\n",
      "Requirement already satisfied: networkx>=2.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (2.4)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (3.2.1)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (1.1.1)\n",
      "Requirement already satisfied: tifffile>=2019.7.26 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (2020.6.3)\n",
      "Requirement already satisfied: pillow!=7.1.0,!=7.1.1,>=4.3.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (7.1.2)\n",
      "Requirement already satisfied: imageio>=2.3.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from scikit-image->efficientnet) (2.8.0)\n",
      "Requirement already satisfied: six in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from h5py->keras-applications<=1.0.8,>=1.0.7->efficientnet) (1.15.0)\n",
      "Requirement already satisfied: decorator>=4.3.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from networkx>=2.0->scikit-image->efficientnet) (4.4.2)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (1.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (2.8.1)\n",
      "Requirement already satisfied: cycler>=0.10 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (0.10.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (2.4.7)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 20.1.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install efficientnet tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.071891Z",
     "start_time": "2020-05-11T10:42:15.046485Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\\nimport os\\nfrom pathlib import Path\\n\\n# from tensorflow.keras.applications import (\\n#     MobileNetV2,\\n# )\\nfrom efficientnet.tfkeras import EfficientNetB0\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\\nimport os\\nfrom pathlib import Path\\n\\n# from tensorflow.keras.applications import (\\n#     MobileNetV2,\\n# )\\nfrom efficientnet.tfkeras import EfficientNetB0\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "# from tensorflow.keras.applications import (\n",
    "#     MobileNetV2,\n",
    "# )\n",
    "from efficientnet.tfkeras import EfficientNetB0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.094693Z",
     "start_time": "2020-05-11T10:42:17.073979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"# dataset_name = \\\"santander-customer-satisfaction\\\"\\n# IMAGE_SIZE = 640\\ndataset_name = \\\"census-income\\\"\\nIMAGE_SIZE = 224\\ntarget = \\\"taxable income amount\\\"\\n# dataset_name = \\\"springleaf-marketing-response\\\"\\n# IMAGE_SIZE = 1408\\n# dataset_name = \\\"segment\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"rl\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"open-payments\\\"\\n# target = \\\"status\\\"\\n# IMAGE_SIZE = 96\\n# dataset_name = \\\"bank-marketing\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"springleaf-marketing-response\\\"\\n# IMAGE_SIZE = 1408\\n# dataset_name = \\\"bnp-cardif\\\"\\n# IMAGE_SIZE = 384\\n# dataset_name = \\\"albert\\\"\\n# IMAGE_SIZE = 288\\n# dataset_name = \\\"titanic\\\"\\n# IMAGE_SIZE = 128\\n# target = \\\"Survived\\\"\\n\\n# IMAGE_SIZE = 24\\n# dataset_name = \\\"rl\\\"\\n# target = \\\"target\\\"\\n# dataset_name = \\\"bank-marketing\\\"\\n# target = \\\"y\\\"\\n\\n# dataset_name = \\\"cat-in-the-dat-ii\\\"\\n# IMAGE_SIZE = 96\\n# dataset_name = \\\"give-me-some-credit\\\"\\n# IMAGE_SIZE = 128\\n\\n# IMAGE_SIZE = 160\\n\\nDATASET_FOLDER = Path(os.getcwd()) / f\\\"data/{dataset_name}\\\"\\n\\nBATCH_SIZE = 32\\nPREFETCH = 50\\n\\n\\nONE_CHANNEL = True\\nNB_CHANNEL = 1 if ONE_CHANNEL else 3\\n\\n# Target size for model to use\\n# TARGET_SIZE = IMAGE_SIZE\\nPRETRAINED_MODEL = EfficientNetB0  # MobileNetV2  # EfficientNetB0  # EfficientNetB4  # MobileNetV2  # EfficientNetB2  # EfficientNetB0  # MobileNetV2  # NASNetMobile\\n\\n# Training params for only new layers\\n\\n# Training params for refit\\nepochs = 200\\npatience = 20\\n\\n# Model name => image size, last_block retrain\\n# NASNetMobile => 224,\\n# NASNetLarge => 331,\\n# MobileNetV2 => 96, ... 160..... 224, last layer => 128 (3 blocks), 137(2 blocks), 146(1 block)\\n# InceptionV3 => 299,\\n# Xception => 299,\";\n",
       "                var nbb_formatted_code = \"# dataset_name = \\\"santander-customer-satisfaction\\\"\\n# IMAGE_SIZE = 640\\ndataset_name = \\\"census-income\\\"\\nIMAGE_SIZE = 224\\ntarget = \\\"taxable income amount\\\"\\n# dataset_name = \\\"springleaf-marketing-response\\\"\\n# IMAGE_SIZE = 1408\\n# dataset_name = \\\"segment\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"rl\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"open-payments\\\"\\n# target = \\\"status\\\"\\n# IMAGE_SIZE = 96\\n# dataset_name = \\\"bank-marketing\\\"\\n# IMAGE_SIZE = 160\\n# dataset_name = \\\"springleaf-marketing-response\\\"\\n# IMAGE_SIZE = 1408\\n# dataset_name = \\\"bnp-cardif\\\"\\n# IMAGE_SIZE = 384\\n# dataset_name = \\\"albert\\\"\\n# IMAGE_SIZE = 288\\n# dataset_name = \\\"titanic\\\"\\n# IMAGE_SIZE = 128\\n# target = \\\"Survived\\\"\\n\\n# IMAGE_SIZE = 24\\n# dataset_name = \\\"rl\\\"\\n# target = \\\"target\\\"\\n# dataset_name = \\\"bank-marketing\\\"\\n# target = \\\"y\\\"\\n\\n# dataset_name = \\\"cat-in-the-dat-ii\\\"\\n# IMAGE_SIZE = 96\\n# dataset_name = \\\"give-me-some-credit\\\"\\n# IMAGE_SIZE = 128\\n\\n# IMAGE_SIZE = 160\\n\\nDATASET_FOLDER = Path(os.getcwd()) / f\\\"data/{dataset_name}\\\"\\n\\nBATCH_SIZE = 32\\nPREFETCH = 50\\n\\n\\nONE_CHANNEL = True\\nNB_CHANNEL = 1 if ONE_CHANNEL else 3\\n\\n# Target size for model to use\\n# TARGET_SIZE = IMAGE_SIZE\\nPRETRAINED_MODEL = EfficientNetB0  # MobileNetV2  # EfficientNetB0  # EfficientNetB4  # MobileNetV2  # EfficientNetB2  # EfficientNetB0  # MobileNetV2  # NASNetMobile\\n\\n# Training params for only new layers\\n\\n# Training params for refit\\nepochs = 200\\npatience = 20\\n\\n# Model name => image size, last_block retrain\\n# NASNetMobile => 224,\\n# NASNetLarge => 331,\\n# MobileNetV2 => 96, ... 160..... 224, last layer => 128 (3 blocks), 137(2 blocks), 146(1 block)\\n# InceptionV3 => 299,\\n# Xception => 299,\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dataset_name = \"santander-customer-satisfaction\"\n",
    "# IMAGE_SIZE = 640\n",
    "dataset_name = \"census-income\"\n",
    "IMAGE_SIZE = 224\n",
    "target = \"taxable income amount\"\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"segment\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"rl\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"open-payments\"\n",
    "# target = \"status\"\n",
    "# IMAGE_SIZE = 96\n",
    "# dataset_name = \"bank-marketing\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"bnp-cardif\"\n",
    "# IMAGE_SIZE = 384\n",
    "# dataset_name = \"albert\"\n",
    "# IMAGE_SIZE = 288\n",
    "# dataset_name = \"titanic\"\n",
    "# IMAGE_SIZE = 128\n",
    "# target = \"Survived\"\n",
    "\n",
    "# IMAGE_SIZE = 24\n",
    "# dataset_name = \"rl\"\n",
    "# target = \"target\"\n",
    "# dataset_name = \"bank-marketing\"\n",
    "# target = \"y\"\n",
    "\n",
    "# dataset_name = \"cat-in-the-dat-ii\"\n",
    "# IMAGE_SIZE = 96\n",
    "# dataset_name = \"give-me-some-credit\"\n",
    "# IMAGE_SIZE = 128\n",
    "\n",
    "# IMAGE_SIZE = 160\n",
    "\n",
    "DATASET_FOLDER = Path(os.getcwd()) / f\"data/{dataset_name}\"\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "PREFETCH = 50\n",
    "\n",
    "\n",
    "ONE_CHANNEL = True\n",
    "NB_CHANNEL = 1 if ONE_CHANNEL else 3\n",
    "\n",
    "# Target size for model to use\n",
    "# TARGET_SIZE = IMAGE_SIZE\n",
    "PRETRAINED_MODEL = EfficientNetB0  # MobileNetV2  # EfficientNetB0  # EfficientNetB4  # MobileNetV2  # EfficientNetB2  # EfficientNetB0  # MobileNetV2  # NASNetMobile\n",
    "\n",
    "# Training params for only new layers\n",
    "\n",
    "# Training params for refit\n",
    "epochs = 200\n",
    "patience = 20\n",
    "\n",
    "# Model name => image size, last_block retrain\n",
    "# NASNetMobile => 224,\n",
    "# NASNetLarge => 331,\n",
    "# MobileNetV2 => 96, ... 160..... 224, last layer => 128 (3 blocks), 137(2 blocks), 146(1 block)\n",
    "# InceptionV3 => 299,\n",
    "# Xception => 299,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:04:39.963383Z",
     "start_time": "2020-04-17T13:04:39.955914Z"
    }
   },
   "source": [
    "## Import + utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.383330Z",
     "start_time": "2020-05-11T10:42:17.097163Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"import json\\nimport numpy as np\\nimport matplotlib.pyplot as plt\\nfrom matplotlib.pyplot import imshow\\nfrom sklearn.metrics import roc_auc_score, accuracy_score\\n\\nfrom thc_net.image.pretrained_model import (\\n    build_dataset,\\n    build_process_path,\\n    build_model,\\n)\\n\\nfrom thc_net.image.tabular_preproc import csv_to_pixel\\n\\n%matplotlib inline\";\n",
       "                var nbb_formatted_code = \"import json\\nimport numpy as np\\nimport matplotlib.pyplot as plt\\nfrom matplotlib.pyplot import imshow\\nfrom sklearn.metrics import roc_auc_score, accuracy_score\\n\\nfrom thc_net.image.pretrained_model import (\\n    build_dataset,\\n    build_process_path,\\n    build_model,\\n)\\n\\nfrom thc_net.image.tabular_preproc import csv_to_pixel\\n\\n%matplotlib inline\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import imshow\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "\n",
    "from thc_net.image.pretrained_model import (\n",
    "    build_dataset,\n",
    "    build_process_path,\n",
    "    build_model,\n",
    ")\n",
    "\n",
    "from thc_net.image.tabular_preproc import csv_to_pixel\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.400914Z",
     "start_time": "2020-05-11T10:42:17.385251Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"def plot_metric(history, metric):\\n    # Plot training & validation loss values\\n    plt.plot(history.history[metric])\\n    plt.plot(history.history[f\\\"val_{metric}\\\"])\\n    plt.title(f\\\"Model {metric}\\\")\\n    plt.ylabel(f\\\"{metric}\\\")\\n    plt.xlabel(\\\"Epoch\\\")\\n    plt.legend([\\\"Train\\\", \\\"Test\\\"], loc=\\\"upper left\\\")\\n    plt.show()\";\n",
       "                var nbb_formatted_code = \"def plot_metric(history, metric):\\n    # Plot training & validation loss values\\n    plt.plot(history.history[metric])\\n    plt.plot(history.history[f\\\"val_{metric}\\\"])\\n    plt.title(f\\\"Model {metric}\\\")\\n    plt.ylabel(f\\\"{metric}\\\")\\n    plt.xlabel(\\\"Epoch\\\")\\n    plt.legend([\\\"Train\\\", \\\"Test\\\"], loc=\\\"upper left\\\")\\n    plt.show()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_metric(history, metric):\n",
    "    # Plot training & validation loss values\n",
    "    plt.plot(history.history[metric])\n",
    "    plt.plot(history.history[f\"val_{metric}\"])\n",
    "    plt.title(f\"Model {metric}\")\n",
    "    plt.ylabel(f\"{metric}\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Test\"], loc=\"upper left\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:09:05.896414Z",
     "start_time": "2020-04-17T13:09:05.860854Z"
    }
   },
   "source": [
    "## Preparing datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.429158Z",
     "start_time": "2020-05-11T10:42:17.403519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['age',\n",
       " 'class_of_worker',\n",
       " 'industry_code',\n",
       " 'occupation_code',\n",
       " 'education',\n",
       " 'adjusted gross income',\n",
       " 'wage per hour',\n",
       " 'enrolled in edu inst last wk',\n",
       " 'marital status',\n",
       " 'major industry code',\n",
       " 'major occupation code',\n",
       " 'mace',\n",
       " 'hispanic Origin',\n",
       " 'sex',\n",
       " 'member of a labor union',\n",
       " 'reason for unemployment',\n",
       " 'full or part time employment stat',\n",
       " 'capital gains',\n",
       " 'capital losses',\n",
       " 'divdends from stocks',\n",
       " 'federal income tax liability',\n",
       " 'tax filer status',\n",
       " 'region of previous residence',\n",
       " 'state of previous residence',\n",
       " 'detailed household and family stat',\n",
       " 'detailed household summary in household',\n",
       " 'instance weight',\n",
       " 'migration code-change in msa',\n",
       " 'migration code-change in reg',\n",
       " 'migration code-move within reg',\n",
       " 'live in this house 1 year ago',\n",
       " 'migration prev res in sunbelt',\n",
       " 'num persons worked for employer',\n",
       " 'family members under 18',\n",
       " 'total person earnings',\n",
       " 'country of birth father',\n",
       " 'country of birth mother',\n",
       " 'country of birth self',\n",
       " 'citizenship',\n",
       " 'total person income',\n",
       " 'own business or self employed',\n",
       " 'taxable income amount',\n",
       " 'Set']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"import pandas as pd\\n\\npanda_kwargs = {}\\nDATASET_FILENAME = \\\"train_bench.csv\\\"\\ndataset_path = DATASET_FOLDER / DATASET_FILENAME\\nTAILORED_COLUMN = \\\"Set\\\"\\n\\ncolumns = pd.read_csv(dataset_path, **panda_kwargs, nrows=1).columns.tolist()\\ncolumns\";\n",
       "                var nbb_formatted_code = \"import pandas as pd\\n\\npanda_kwargs = {}\\nDATASET_FILENAME = \\\"train_bench.csv\\\"\\ndataset_path = DATASET_FOLDER / DATASET_FILENAME\\nTAILORED_COLUMN = \\\"Set\\\"\\n\\ncolumns = pd.read_csv(dataset_path, **panda_kwargs, nrows=1).columns.tolist()\\ncolumns\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "panda_kwargs = {}\n",
    "DATASET_FILENAME = \"train_bench.csv\"\n",
    "dataset_path = DATASET_FOLDER / DATASET_FILENAME\n",
    "TAILORED_COLUMN = \"Set\"\n",
    "\n",
    "columns = pd.read_csv(dataset_path, **panda_kwargs, nrows=1).columns.tolist()\n",
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.527710Z",
     "start_time": "2020-05-11T10:42:17.430882Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"split = None\\npanda_kwargs = {}\\nif TAILORED_COLUMN not in columns:\\n    split = np.random.choice(\\n        [\\\"train\\\", \\\"valid\\\", \\\"test\\\"], p=[0.8, 0.1, 0.1], size=(NB_LINES,)\\n    )\\nelse:\\n    split = pd.read_csv(\\n        dataset_path, **panda_kwargs, usecols=[TAILORED_COLUMN]\\n    ).values.reshape(-1)\\n\\n\\ntrain_indices = np.argwhere(split == \\\"train\\\").reshape(-1)\\nnp.random.shuffle(train_indices)\\nvalid_indices = np.argwhere(split == \\\"valid\\\").reshape(-1)\\ntest_indices = np.argwhere(split == \\\"test\\\").reshape(-1)\";\n",
       "                var nbb_formatted_code = \"split = None\\npanda_kwargs = {}\\nif TAILORED_COLUMN not in columns:\\n    split = np.random.choice(\\n        [\\\"train\\\", \\\"valid\\\", \\\"test\\\"], p=[0.8, 0.1, 0.1], size=(NB_LINES,)\\n    )\\nelse:\\n    split = pd.read_csv(\\n        dataset_path, **panda_kwargs, usecols=[TAILORED_COLUMN]\\n    ).values.reshape(-1)\\n\\n\\ntrain_indices = np.argwhere(split == \\\"train\\\").reshape(-1)\\nnp.random.shuffle(train_indices)\\nvalid_indices = np.argwhere(split == \\\"valid\\\").reshape(-1)\\ntest_indices = np.argwhere(split == \\\"test\\\").reshape(-1)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "split = None\n",
    "panda_kwargs = {}\n",
    "if TAILORED_COLUMN not in columns:\n",
    "    split = np.random.choice(\n",
    "        [\"train\", \"valid\", \"test\"], p=[0.8, 0.1, 0.1], size=(NB_LINES,)\n",
    "    )\n",
    "else:\n",
    "    split = pd.read_csv(\n",
    "        dataset_path, **panda_kwargs, usecols=[TAILORED_COLUMN]\n",
    "    ).values.reshape(-1)\n",
    "\n",
    "\n",
    "train_indices = np.argwhere(split == \"train\").reshape(-1)\n",
    "np.random.shuffle(train_indices)\n",
    "valid_indices = np.argwhere(split == \"valid\").reshape(-1)\n",
    "test_indices = np.argwhere(split == \"test\").reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.539346Z",
     "start_time": "2020-05-11T10:42:17.529338Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['education',\n",
       " 'wage per hour',\n",
       " 'divdends from stocks',\n",
       " 'reason for unemployment',\n",
       " 'country of birth self',\n",
       " 'adjusted gross income',\n",
       " 'detailed household summary in household',\n",
       " 'major industry code',\n",
       " 'federal income tax liability',\n",
       " 'migration code-move within reg',\n",
       " 'migration prev res in sunbelt',\n",
       " 'instance weight',\n",
       " 'migration code-change in reg',\n",
       " 'total person earnings',\n",
       " 'citizenship',\n",
       " 'full or part time employment stat',\n",
       " 'capital gains',\n",
       " 'region of previous residence',\n",
       " 'occupation_code',\n",
       " 'mace',\n",
       " 'own business or self employed',\n",
       " 'live in this house 1 year ago',\n",
       " 'detailed household and family stat',\n",
       " 'num persons worked for employer',\n",
       " 'hispanic Origin',\n",
       " 'major occupation code',\n",
       " 'migration code-change in msa',\n",
       " 'total person income',\n",
       " 'age',\n",
       " 'enrolled in edu inst last wk',\n",
       " 'industry_code',\n",
       " 'marital status',\n",
       " 'country of birth mother',\n",
       " 'sex',\n",
       " 'tax filer status',\n",
       " 'state of previous residence',\n",
       " 'country of birth father',\n",
       " 'class_of_worker',\n",
       " 'member of a labor union',\n",
       " 'family members under 18',\n",
       " 'capital losses']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"used_columns = list(set(columns) - set([TAILORED_COLUMN, target]))\\nused_columns\";\n",
       "                var nbb_formatted_code = \"used_columns = list(set(columns) - set([TAILORED_COLUMN, target]))\\nused_columns\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "used_columns = list(set(columns) - set([TAILORED_COLUMN, target]))\n",
    "used_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.674344Z",
     "start_time": "2020-05-11T10:42:17.543391Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"df = pd.read_csv(dataset_path, low_memory=False)\";\n",
       "                var nbb_formatted_code = \"df = pd.read_csv(dataset_path, low_memory=False)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.read_csv(dataset_path, low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.709403Z",
     "start_time": "2020-05-11T10:42:17.679046Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"X = df[used_columns].values\\nY = df[target].values.reshape(-1)\\n\\nextract_df = df[used_columns]\\nnb_idx = (extract_df.dtypes == \\\"int64\\\") | (extract_df.dtypes == \\\"float64\\\")\\nnb_idx = extract_df.columns[nb_idx]\\nis_numeric = np.isin(extract_df.columns.values, nb_idx.values)\";\n",
       "                var nbb_formatted_code = \"X = df[used_columns].values\\nY = df[target].values.reshape(-1)\\n\\nextract_df = df[used_columns]\\nnb_idx = (extract_df.dtypes == \\\"int64\\\") | (extract_df.dtypes == \\\"float64\\\")\\nnb_idx = extract_df.columns[nb_idx]\\nis_numeric = np.isin(extract_df.columns.values, nb_idx.values)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = df[used_columns].values\n",
    "Y = df[target].values.reshape(-1)\n",
    "\n",
    "extract_df = df[used_columns]\n",
    "nb_idx = (extract_df.dtypes == \"int64\") | (extract_df.dtypes == \"float64\")\n",
    "nb_idx = extract_df.columns[nb_idx]\n",
    "is_numeric = np.isin(extract_df.columns.values, nb_idx.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:17.739582Z",
     "start_time": "2020-05-11T10:42:17.711367Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "education                                   True\n",
       "wage per hour                               True\n",
       "divdends from stocks                        True\n",
       "reason for unemployment                     True\n",
       "country of birth self                       True\n",
       "adjusted gross income                       True\n",
       "detailed household summary in household     True\n",
       "major industry code                         True\n",
       "federal income tax liability                True\n",
       "migration code-move within reg              True\n",
       "migration prev res in sunbelt               True\n",
       "instance weight                             True\n",
       "migration code-change in reg                True\n",
       "total person earnings                       True\n",
       "citizenship                                 True\n",
       "full or part time employment stat           True\n",
       "capital gains                               True\n",
       "region of previous residence                True\n",
       "occupation_code                             True\n",
       "mace                                        True\n",
       "own business or self employed               True\n",
       "live in this house 1 year ago               True\n",
       "detailed household and family stat         False\n",
       "num persons worked for employer             True\n",
       "hispanic Origin                             True\n",
       "major occupation code                       True\n",
       "migration code-change in msa                True\n",
       "total person income                         True\n",
       "age                                         True\n",
       "enrolled in edu inst last wk                True\n",
       "industry_code                               True\n",
       "marital status                              True\n",
       "country of birth mother                     True\n",
       "sex                                         True\n",
       "tax filer status                            True\n",
       "state of previous residence                 True\n",
       "country of birth father                     True\n",
       "class_of_worker                             True\n",
       "member of a labor union                     True\n",
       "family members under 18                     True\n",
       "capital losses                              True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"(extract_df.nunique() / extract_df.shape[0]) < 0.02\";\n",
       "                var nbb_formatted_code = \"(extract_df.nunique() / extract_df.shape[0]) < 0.02\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(extract_df.nunique() / extract_df.shape[0]) < 0.02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.613360Z",
     "start_time": "2020-05-11T10:42:17.741785Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process ForkProcess-16:\n",
      "Process ForkProcess-1:\n",
      "Process ForkProcess-12:\n",
      "Process ForkProcess-14:\n",
      "Process ForkProcess-6:\n",
      "Process ForkProcess-7:\n",
      "Process ForkProcess-2:\n",
      "Process ForkProcess-3:\n",
      "Process ForkProcess-11:\n",
      "Process ForkProcess-9:\n",
      "Process ForkProcess-13:\n",
      "Traceback (most recent call last):\n",
      "Process ForkProcess-8:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 364, in put\n",
      "    self._writer.send_bytes(obj)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "KeyboardInterrupt\n",
      "Traceback (most recent call last):\n",
      "KeyboardInterrupt\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "KeyboardInterrupt\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"# https://www.tensorflow.org/tutorials/load_data/images\\nX, Y = csv_to_pixel(df, used_columns, target, ascii_only=ONE_CHANNEL)\\nIMAGE_SIZE = X.shape[1]\\nTARGET_SIZE = IMAGE_SIZE if IMAGE_SIZE > 32 else 32\";\n",
       "                var nbb_formatted_code = \"# https://www.tensorflow.org/tutorials/load_data/images\\nX, Y = csv_to_pixel(df, used_columns, target, ascii_only=ONE_CHANNEL)\\nIMAGE_SIZE = X.shape[1]\\nTARGET_SIZE = IMAGE_SIZE if IMAGE_SIZE > 32 else 32\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 244, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, result=r)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "Process ForkProcess-15:\n",
      "Process ForkProcess-5:\n",
      "Process ForkProcess-4:\n",
      "Process ForkProcess-10:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 239, in _process_worker\n",
      "    r = call_item.fn(*call_item.args, **call_item.kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 239, in _process_worker\n",
      "    r = call_item.fn(*call_item.args, **call_item.kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 239, in _process_worker\n",
      "    r = call_item.fn(*call_item.args, **call_item.kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 239, in _process_worker\n",
      "    r = call_item.fn(*call_item.args, **call_item.kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in _process_chunk\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in _process_chunk\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in _process_chunk\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in _process_chunk\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in <listcomp>\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in <listcomp>\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in <listcomp>\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 198, in <listcomp>\n",
      "    return [fn(*args) for args in chunk]\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 380, in build_image\n",
      "    ] = feature_to_square_image(features[idx], is_numeric[idx], ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 380, in build_image\n",
      "    ] = feature_to_square_image(features[idx], is_numeric[idx], ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 380, in build_image\n",
      "    ] = feature_to_square_image(features[idx], is_numeric[idx], ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 380, in build_image\n",
      "    ] = feature_to_square_image(features[idx], is_numeric[idx], ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 304, in feature_to_square_image\n",
      "    return word_to_img(feature, ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 304, in feature_to_square_image\n",
      "    return word_to_img(feature, ascii_only=ascii_only)\n",
      "KeyboardInterrupt\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 304, in feature_to_square_image\n",
      "    return word_to_img(feature, ascii_only=ascii_only)\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 297, in word_to_img\n",
      "    res[: word.shape[0]] = word\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 296, in word_to_img\n",
      "    res = np.zeros(shape=(64, nb_channel), dtype=\"uint8\")\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 294, in word_to_img\n",
      "    word_to_unicode(normalize(str(word))[:64], ascii_only=ascii_only), dtype=\"uint8\"\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/work/thc-net/src/thc_net/image/tabular_preproc.py\", line 268, in word_to_unicode\n",
      "    int_letter = ord(letter)\n",
      "Traceback (most recent call last):\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 205, in _sendback_result\n",
      "    exception=exception))\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 242, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, exception=exc)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 242, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, exception=exc)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 242, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, exception=exc)\n",
      "Traceback (most recent call last):\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 297, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/process.py\", line 99, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 242, in _process_worker\n",
      "    _sendback_result(result_queue, call_item.work_id, exception=exc)\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/lib/python3.7/concurrent/futures/process.py\", line 208, in _sendback_result\n",
      "    result_queue.put(_ResultItem(work_id, exception=exc))\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/queues.py\", line 363, in put\n",
      "    with self._wlock:\n",
      "  File \"/usr/local/lib/python3.7/multiprocessing/synchronize.py\", line 95, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "# https://www.tensorflow.org/tutorials/load_data/images\n",
    "X, Y = csv_to_pixel(df, used_columns, target, ascii_only=ONE_CHANNEL)\n",
    "IMAGE_SIZE = X.shape[1]\n",
    "TARGET_SIZE = IMAGE_SIZE if IMAGE_SIZE > 32 else 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.621590Z",
     "start_time": "2020-05-11T10:42:33.615407Z"
    }
   },
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.633713Z",
     "start_time": "2020-05-11T10:42:33.623336Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "Y = LabelEncoder().fit_transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.649488Z",
     "start_time": "2020-05-11T10:42:33.636541Z"
    }
   },
   "outputs": [],
   "source": [
    "Y = to_categorical(Y.reshape(-1, 1))\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.660717Z",
     "start_time": "2020-05-11T10:42:33.651088Z"
    }
   },
   "outputs": [],
   "source": [
    "OUTPUT_DIM = len(np.unique(Y))\n",
    "OUTPUT_DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.672210Z",
     "start_time": "2020-05-11T10:42:33.663005Z"
    }
   },
   "outputs": [],
   "source": [
    "X.nbytes / 1024 ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.682806Z",
     "start_time": "2020-05-11T10:42:33.674439Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.data import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.952353Z",
     "start_time": "2020-05-11T10:42:33.685054Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_train = (\n",
    "    Dataset.from_tensor_slices((X[train_indices], Y[train_indices]))\n",
    "    #     .repeat()\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "dataset_valid = (\n",
    "    Dataset.from_tensor_slices((X[valid_indices], Y[valid_indices]))\n",
    "    #     .repeat()\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "dataset_test = Dataset.from_tensor_slices((X[test_indices], Y[test_indices])).batch(\n",
    "    BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.986806Z",
     "start_time": "2020-05-11T10:42:33.953989Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_train.take(1):\n",
    "    print(\"Image shape: \", image.numpy().shape)\n",
    "    print(\"Label: \", label.numpy().shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Images Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:33.998423Z",
     "start_time": "2020-05-11T10:42:33.988829Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_image(image, image_size, one_channel=False):\n",
    "    if one_channel:\n",
    "        imshow(image.reshape(IMAGE_SIZE, IMAGE_SIZE))\n",
    "    else:\n",
    "        imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:34.675090Z",
     "start_time": "2020-05-11T10:42:34.000018Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_train.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-09T14:27:57.747818Z",
     "start_time": "2020-05-09T14:27:55.622Z"
    }
   },
   "source": [
    "for image, label in dataset_valid.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model, using existing one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:37.905818Z",
     "start_time": "2020-05-11T10:42:34.677301Z"
    }
   },
   "outputs": [],
   "source": [
    "model, callbacks = build_model(\n",
    "    EfficientNetB0,\n",
    "    image_size=IMAGE_SIZE,\n",
    "    nb_channel=NB_CHANNEL,\n",
    "    input_size=TARGET_SIZE,  # IMAGE_SIZE,\n",
    "    output_dim=OUTPUT_DIM,\n",
    "    weights=None,\n",
    "    patience=5,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:38.049167Z",
     "start_time": "2020-05-11T10:42:37.912842Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit new layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:42:38.162944Z",
     "start_time": "2020-05-11T10:42:38.054441Z"
    }
   },
   "outputs": [],
   "source": [
    "truth_test = []\n",
    "for _, labels in dataset_test:\n",
    "    truth_test.append(np.argmax(labels, axis=1))\n",
    "truth_test = np.hstack(truth_test)\n",
    "truth_test\n",
    "\n",
    "truth_valid = []\n",
    "for i, (_, labels) in enumerate(dataset_valid):\n",
    "    truth_valid.append(np.argmax(labels, axis=1))\n",
    "#     if i >= steps_per_epoch_val - 1:\n",
    "#         break\n",
    "truth_valid = np.hstack(truth_valid)\n",
    "truth_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unfreeze and fit more/all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:33.280331Z",
     "start_time": "2020-05-11T10:42:38.164677Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# we train our model again (this time fine-tuning the top 2 inception blocks\n",
    "# alongside the top Dense layers\n",
    "history = model.fit(\n",
    "    dataset_train,\n",
    "    callbacks=callbacks,\n",
    "    epochs=epochs,\n",
    "    # steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    # validation_steps=steps_per_epoch_val,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:33.479974Z",
     "start_time": "2020-05-11T10:54:33.282229Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_metric(history, \"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:16:20.349217Z",
     "start_time": "2020-04-17T13:16:16.612Z"
    }
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.437437Z",
     "start_time": "2020-05-11T10:54:33.481718Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_valid = model.predict(dataset_valid)  # , steps=steps_per_epoch_val)\n",
    "preds_valid.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(\n",
    "        f\"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}\"\n",
    "    )\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}\")\n",
    "preds_test = model.predict(dataset_test)\n",
    "preds_test.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(f\"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}\")\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Bank martketing No numeric\n",
    "ROC AUC valid: 0.7849241489978159\n",
    "ROC AUC test: 0.8066209120243591\n",
    "\n",
    "With numeric\n",
    "ROC AUC valid: 0.7415022647293126\n",
    "ROC AUC test: 0.7471501527935644"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.446564Z",
     "start_time": "2020-05-11T10:54:36.440214Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 13min, 0.9338321957865205 (batch 256)\n",
    "# cat-in-the-dat-ii 0.7643929251726093 1h10 (batch 256)\n",
    "# RL ROC AUC test: 0.9430279072306678 mobile net patience 20\n",
    "# BNP cardif (image size : 96) ROC AUC test: 0.7168011573597879 1h16\n",
    "# give-me-some-credit ROC AUC test: 0.8591709381962962 2h4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.455904Z",
     "start_time": "2020-05-11T10:54:36.449370Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 1h10\n",
    "# ROC AUC valid: 0.9432323040315754\n",
    "# ROC AUC test: 0.9345809216081824"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.463775Z",
     "start_time": "2020-05-11T10:54:36.458475Z"
    }
   },
   "outputs": [],
   "source": [
    "# EfficientNetB0\n",
    "# EfficientNetB2 2h => ROC AUC valid: 0.9357130861335854 ROC AUC test: 0.9330937133279599"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.474620Z",
     "start_time": "2020-05-11T10:54:36.466454Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.8164865598696714 => target size 96, whole re train\n",
    "# 0.8141824599511267 => target size 224, whole re train\n",
    "# ROC AUC valid: 0.8268400760249797 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8462134942186483 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "\n",
    "# ROC AUC valid: 0.8348917439829162\n",
    "# RL\n",
    "# ROC AUC valid: 0.892118469133795 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9233534348199217 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9465346534653467 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC valid: 0.9186458210299415 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8778868370932499 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8917381493730192 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9090374872044725 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.886417393797122 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9406275221953189 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC valid: 0.8811034128677376 => 96 => 96, whole, batch 128, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7970734141661526 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC valid: 0.7500980687987842 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC valid: 0.7206667869818926 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T10:54:36.484693Z",
     "start_time": "2020-05-11T10:54:36.477410Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.833417731838137 => target size 96, whole train\n",
    "# 0.8170226029745679 => target size 224, whole train\n",
    "# ROC AUC test: 0.8348549041045967 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8453497574694486 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "# ROC AUC test: 0.8447058873195916\n",
    "\n",
    "# RL\n",
    "# ROC AUC test: 0.9051288159651395 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9128674518211912 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC test: 0.9456874816987527 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC test: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC test: 0.916907759155486 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8931101859362467 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8938387451368033 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9067445823812874 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.8954549557710788 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.9062895529860363 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC test: 0.9403517762951931 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7959000291791145 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC test: 0.7487660412524685 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC test: 0.725934546476426 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}