dataset-alteration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "arbitrary-blink",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:27.914280Z",
     "iopub.status.busy": "2021-07-12T06:13:27.912502Z",
     "iopub.status.idle": "2021-07-12T06:13:29.160381Z",
     "shell.execute_reply": "2021-07-12T06:13:29.159548Z",
     "shell.execute_reply.started": "2021-06-26T07:37:12.152742Z"
    },
    "papermill": {
     "duration": 1.276774,
     "end_time": "2021-07-12T06:13:29.160596",
     "exception": false,
     "start_time": "2021-07-12T06:13:27.883822",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "integrated-facility",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:29.199942Z",
     "iopub.status.busy": "2021-07-12T06:13:29.199045Z",
     "iopub.status.idle": "2021-07-12T06:13:30.088570Z",
     "shell.execute_reply": "2021-07-12T06:13:30.087885Z",
     "shell.execute_reply.started": "2021-06-26T07:37:13.361199Z"
    },
    "papermill": {
     "duration": 0.911721,
     "end_time": "2021-07-12T06:13:30.088766",
     "exception": false,
     "start_time": "2021-07-12T06:13:29.177045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load in datasets\n",
    "# https://www.kaggle.com/yagnasrip2/dataset\n",
    "# https://www.kaggle.com/mahbubrob/usa-cities\n",
    "ds = pd.read_csv('/kaggle/input/dataset/Training Data.csv')\n",
    "usa_ds = pd.read_csv('/kaggle/input/usa-cities/usa_cities.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "armed-devil",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:30.141341Z",
     "iopub.status.busy": "2021-07-12T06:13:30.140360Z",
     "iopub.status.idle": "2021-07-12T06:13:30.143772Z",
     "shell.execute_reply": "2021-07-12T06:13:30.143108Z",
     "shell.execute_reply.started": "2021-06-26T07:37:14.27513Z"
    },
    "papermill": {
     "duration": 0.038732,
     "end_time": "2021-07-12T06:13:30.143925",
     "exception": false,
     "start_time": "2021-07-12T06:13:30.105193",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Extract the exact amount of required states and cities\n",
    "us_state_names = usa_ds.province.value_counts().keys().to_list()[:28]\n",
    "us_city_names = usa_ds.city_ascii.value_counts().keys().to_list()[:316]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "meaningful-frank",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:30.248187Z",
     "iopub.status.busy": "2021-07-12T06:13:30.245593Z",
     "iopub.status.idle": "2021-07-12T06:13:30.252764Z",
     "shell.execute_reply": "2021-07-12T06:13:30.251950Z",
     "shell.execute_reply.started": "2021-06-26T07:37:14.296439Z"
    },
    "papermill": {
     "duration": 0.092489,
     "end_time": "2021-07-12T06:13:30.252924",
     "exception": false,
     "start_time": "2021-07-12T06:13:30.160435",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create map of states\n",
    "states = list(ds.state.value_counts().index.to_list())\n",
    "\n",
    "states_set = set()\n",
    "for state in states:\n",
    "    states_set.add('_'.join(state.split()))\n",
    "\n",
    "states_set.remove('Uttar_Pradesh[5]')\n",
    "\n",
    "state_map = dict(zip(states_set, us_state_names))\n",
    "\n",
    "state_map['Andhra Pradesh'] = state_map['Andhra_Pradesh']\n",
    "state_map['Himachal Pradesh'] = state_map['Himachal_Pradesh']\n",
    "state_map['Jammu and Kashmir'] = state_map['Jammu_and_Kashmir']\n",
    "state_map['Madhya Pradesh'] = state_map['Madhya_Pradesh']\n",
    "state_map['Uttar Pradesh'] = state_map['Uttar_Pradesh']\n",
    "state_map['Uttar Pradesh[5]'] = state_map['Uttar_Pradesh']\n",
    "state_map['Uttar_Pradesh[5]'] = state_map['Uttar_Pradesh']\n",
    "state_map['West Bengal'] = state_map['West_Bengal']\n",
    "state_map['Tamil Nadu'] = state_map['Tamil_Nadu']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fifty-emergency",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:30.358258Z",
     "iopub.status.busy": "2021-07-12T06:13:30.333437Z",
     "iopub.status.idle": "2021-07-12T06:13:30.365113Z",
     "shell.execute_reply": "2021-07-12T06:13:30.364528Z",
     "shell.execute_reply.started": "2021-06-26T07:37:14.377323Z"
    },
    "papermill": {
     "duration": 0.094286,
     "end_time": "2021-07-12T06:13:30.365267",
     "exception": false,
     "start_time": "2021-07-12T06:13:30.270981",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create map of cities\n",
    "cities = list(ds.city.value_counts().index.to_list())\n",
    "\n",
    "cities_set = set()\n",
    "for city in cities:\n",
    "    cities_set.add(re.sub(r'\\[[0-9]+\\]', \"\", '_'.join(city.split())))\n",
    "    \n",
    "city_map = dict(zip(cities_set, us_city_names))\n",
    "\n",
    "for city in cities:\n",
    "    if city not in cities_set:\n",
    "        city_map[city] = city_map[re.sub(r'\\[[0-9]+\\]', \"\", '_'.join(city.split()))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "tough-biotechnology",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:30.403667Z",
     "iopub.status.busy": "2021-07-12T06:13:30.403016Z",
     "iopub.status.idle": "2021-07-12T06:13:30.437753Z",
     "shell.execute_reply": "2021-07-12T06:13:30.437082Z",
     "shell.execute_reply.started": "2021-06-26T07:37:14.462291Z"
    },
    "papermill": {
     "duration": 0.055387,
     "end_time": "2021-07-12T06:13:30.437899",
     "exception": false,
     "start_time": "2021-07-12T06:13:30.382512",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Copy datasets\n",
    "ds_copy = ds.copy(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "young-palestine",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:30.501093Z",
     "iopub.status.busy": "2021-07-12T06:13:30.486511Z",
     "iopub.status.idle": "2021-07-12T06:13:34.966433Z",
     "shell.execute_reply": "2021-07-12T06:13:34.965766Z",
     "shell.execute_reply.started": "2021-06-26T07:37:14.494934Z"
    },
    "papermill": {
     "duration": 4.512124,
     "end_time": "2021-07-12T06:13:34.966571",
     "exception": false,
     "start_time": "2021-07-12T06:13:30.454447",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Replace State and city names in the datasets\n",
    "for india in state_map:\n",
    "    ds_copy.state.replace(india, state_map[india], inplace=True)   \n",
    "\n",
    "for india in city_map:\n",
    "    ds_copy.city.replace(india, city_map[india], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "artificial-tamil",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:35.007281Z",
     "iopub.status.busy": "2021-07-12T06:13:35.006535Z",
     "iopub.status.idle": "2021-07-12T06:13:35.008846Z",
     "shell.execute_reply": "2021-07-12T06:13:35.009310Z",
     "shell.execute_reply.started": "2021-06-26T07:37:27.524816Z"
    },
    "papermill": {
     "duration": 0.027193,
     "end_time": "2021-07-12T06:13:35.009506",
     "exception": false,
     "start_time": "2021-07-12T06:13:34.982313",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Rename columns in the train and test dataset\n",
    "ds_copy.rename(columns = {\n",
    "    \"married\" : \"has_child\",\n",
    "    \"income\" : \"monthly_income\",\n",
    "    \"car_ownership\" : \"owns_bike\",\n",
    "    \"house_ownership\" : \"residential_status\",\n",
    "    \"current_job_years\" : \"years_employed\",\n",
    "    \"current_house_years\" : \"age_house\",\n",
    "    \"city\" : \"home_city\",\n",
    "    \"state\" : \"home_state\",\n",
    "    \"risk_flag\": \"ineligible_for_funds\"\n",
    "}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "geographic-associate",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:35.048404Z",
     "iopub.status.busy": "2021-07-12T06:13:35.047719Z",
     "iopub.status.idle": "2021-07-12T06:13:35.317853Z",
     "shell.execute_reply": "2021-07-12T06:13:35.318410Z",
     "shell.execute_reply.started": "2021-06-26T07:37:30.349764Z"
    },
    "papermill": {
     "duration": 0.292973,
     "end_time": "2021-07-12T06:13:35.318648",
     "exception": false,
     "start_time": "2021-07-12T06:13:35.025675",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Modify the data to make it untraceable\n",
    "ds_copy.has_child = ds_copy.has_child.apply(lambda x: \"yes\" if x == \"married\" else \"no\")\n",
    "\n",
    "ds_copy.monthly_income = ds_copy.monthly_income.apply(lambda x: x/12)\n",
    "\n",
    "ds_copy.residential_status = ds_copy.residential_status.apply(lambda x: \"on_rent\" if x == \"rented\" else \"own_house\" if x == \"owned\" else \"no_house\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "abandoned-wiring",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:35.355020Z",
     "iopub.status.busy": "2021-07-12T06:13:35.354270Z",
     "iopub.status.idle": "2021-07-12T06:13:48.864657Z",
     "shell.execute_reply": "2021-07-12T06:13:48.864087Z",
     "shell.execute_reply.started": "2021-06-26T07:37:33.251912Z"
    },
    "papermill": {
     "duration": 13.529343,
     "end_time": "2021-07-12T06:13:48.864839",
     "exception": false,
     "start_time": "2021-07-12T06:13:35.335496",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add underscores to some cities and states with space in names in order to make it a bit harder to work with\n",
    "ds_copy['home_state'] = ds_copy.apply(lambda x: '_'.join(x.home_state.split()) if (x.Id%5) == 0 else x.home_state, axis=1)\n",
    "\n",
    "ds_copy['home_city'] = ds_copy.apply(lambda x: '_'.join(x.home_city.split()) if (x.Id%37) == 0 else x.home_city, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "hidden-austria",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:48.918929Z",
     "iopub.status.busy": "2021-07-12T06:13:48.913660Z",
     "iopub.status.idle": "2021-07-12T06:13:51.474114Z",
     "shell.execute_reply": "2021-07-12T06:13:51.473497Z",
     "shell.execute_reply.started": "2021-06-26T07:37:47.277511Z"
    },
    "papermill": {
     "duration": 2.591774,
     "end_time": "2021-07-12T06:13:51.474263",
     "exception": false,
     "start_time": "2021-07-12T06:13:48.882489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add random numbers in square brackets (similar data was present in dataset), forcing participants to preprocess data efficiently\n",
    "for n in np.random.randint(low=0, high=ds_copy.shape[0], size = int(0.07 * ds_copy.shape[0])):\n",
    "    ds_copy.iat[n, 8] += f\"[{np.random.randint(low=0, high=100)}]\"\n",
    "    \n",
    "for n in np.random.randint(low=0, high=ds_copy.shape[0], size = int(0.07 * ds_copy.shape[0])):\n",
    "    ds_copy.iat[n, 9] += f\"[{np.random.randint(low=0, high=100)}]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "purple-colleague",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:51.517090Z",
     "iopub.status.busy": "2021-07-12T06:13:51.515851Z",
     "iopub.status.idle": "2021-07-12T06:13:51.566059Z",
     "shell.execute_reply": "2021-07-12T06:13:51.565384Z",
     "shell.execute_reply.started": "2021-06-26T07:37:49.914005Z"
    },
    "papermill": {
     "duration": 0.074938,
     "end_time": "2021-07-12T06:13:51.566240",
     "exception": false,
     "start_time": "2021-07-12T06:13:51.491302",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add a bit of noise to the income\n",
    "noise = np.random.uniform(-50, 200, (ds_copy.shape[0],))\n",
    "ds_copy.monthly_income += noise\n",
    "\n",
    "noise = np.random.normal(np.random.randn(), np.random.rand(), (ds_copy.shape[0],))\n",
    "ds_copy.experience = abs(ds_copy.experience + noise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "civilian-denmark",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:51.606387Z",
     "iopub.status.busy": "2021-07-12T06:13:51.605383Z",
     "iopub.status.idle": "2021-07-12T06:13:51.633010Z",
     "shell.execute_reply": "2021-07-12T06:13:51.632350Z",
     "shell.execute_reply.started": "2021-06-26T07:38:02.853559Z"
    },
    "papermill": {
     "duration": 0.050855,
     "end_time": "2021-07-12T06:13:51.633180",
     "exception": false,
     "start_time": "2021-07-12T06:13:51.582325",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Extract target label\n",
    "X = ds_copy.drop('ineligible_for_funds', axis=1)\n",
    "y = ds_copy['ineligible_for_funds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "informed-steel",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:51.673106Z",
     "iopub.status.busy": "2021-07-12T06:13:51.672407Z",
     "iopub.status.idle": "2021-07-12T06:13:51.761567Z",
     "shell.execute_reply": "2021-07-12T06:13:51.761012Z",
     "shell.execute_reply.started": "2021-06-26T07:38:16.829224Z"
    },
    "papermill": {
     "duration": 0.112716,
     "end_time": "2021-07-12T06:13:51.761732",
     "exception": false,
     "start_time": "2021-07-12T06:13:51.649016",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Use Stratified K-Fold to split the training set into train and test sets\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "train_idx, test_idx = next(StratifiedKFold(n_splits=4).split(X, y))\n",
    "\n",
    "train, train['ineligible_for_funds'] = X.loc[train_idx], y.loc[train_idx]\n",
    "test, test['ineligible_for_funds'] = X.loc[test_idx], y.loc[test_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "motivated-brighton",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:51.809380Z",
     "iopub.status.busy": "2021-07-12T06:13:51.803685Z",
     "iopub.status.idle": "2021-07-12T06:13:54.276594Z",
     "shell.execute_reply": "2021-07-12T06:13:54.275935Z",
     "shell.execute_reply.started": "2021-06-26T07:38:25.917579Z"
    },
    "papermill": {
     "duration": 2.498664,
     "end_time": "2021-07-12T06:13:54.276783",
     "exception": false,
     "start_time": "2021-07-12T06:13:51.778119",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create null values\n",
    "for n in np.random.randint(low=0, high=train.shape[0], size = int(0.09856 * train.shape[0])):\n",
    "    train.iat[n, 1] = None\n",
    "    \n",
    "for n in np.random.randint(low=0, high=train.shape[0], size = int(0.09601 * train.shape[0])):\n",
    "    train.iat[n, 5] = None\n",
    "    train.iat[n, 11] = None\n",
    "    \n",
    "for n in np.random.randint(low=0, high=train.shape[0], size = int(0.1001 * train.shape[0])):\n",
    "    train.iat[n, 6] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "reasonable-google",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:13:54.334580Z",
     "iopub.status.busy": "2021-07-12T06:13:54.329229Z",
     "iopub.status.idle": "2021-07-12T06:14:02.857210Z",
     "shell.execute_reply": "2021-07-12T06:14:02.856466Z",
     "shell.execute_reply.started": "2021-06-26T07:38:30.973424Z"
    },
    "papermill": {
     "duration": 8.560844,
     "end_time": "2021-07-12T06:14:02.857399",
     "exception": false,
     "start_time": "2021-07-12T06:13:54.296555",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Reindex all columns\n",
    "for i in range(1, train.shape[0] + 1):\n",
    "    train.iat[i-1, 0] = i\n",
    "    \n",
    "for i in range(train.shape[0] + 1, ds_copy.shape[0] + 1):\n",
    "    test.iat[i-1 - train.shape[0], 0] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "african-robertson",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:14:02.899614Z",
     "iopub.status.busy": "2021-07-12T06:14:02.898897Z",
     "iopub.status.idle": "2021-07-12T06:14:02.914310Z",
     "shell.execute_reply": "2021-07-12T06:14:02.913428Z",
     "shell.execute_reply.started": "2021-06-26T07:38:41.922897Z"
    },
    "papermill": {
     "duration": 0.039659,
     "end_time": "2021-07-12T06:14:02.914493",
     "exception": false,
     "start_time": "2021-07-12T06:14:02.874834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create test data and labels\n",
    "test_X = test.drop('ineligible_for_funds', axis=1)\n",
    "test_y = pd.concat([test.Id, test.ineligible_for_funds], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "practical-coaching",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T06:14:02.962116Z",
     "iopub.status.busy": "2021-07-12T06:14:02.961127Z",
     "iopub.status.idle": "2021-07-12T06:14:05.596323Z",
     "shell.execute_reply": "2021-07-12T06:14:05.595518Z",
     "shell.execute_reply.started": "2021-06-26T07:38:43.995777Z"
    },
    "papermill": {
     "duration": 2.662583,
     "end_time": "2021-07-12T06:14:05.596493",
     "exception": false,
     "start_time": "2021-07-12T06:14:02.933910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Export the data to .csv\n",
    "train.to_csv('train_dataset.csv', index=False)\n",
    "test_X.to_csv('test_dataset.csv', index=False)\n",
    "test_y.to_csv('test_labels_PRIVATE.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 47.916633,
   "end_time": "2021-07-12T06:14:06.971415",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-07-12T06:13:19.054782",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}