notebooks/result_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import ast\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "import src.evaluation.util as util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define paths\n",
    "regr_results_log_path = '../logs/regr_results/'\n",
    "aggr_class_report_log_path = '../data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load regression results table\n",
    "result_file = regr_results_log_path + \"regr_results.csv\"\n",
    "df_regr_results = pd.read_csv(result_file)\n",
    "\n",
    "df_regr_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_regr_results.query('model == \"basic\" & classification_task == \"4\" & delta == False')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize scatter plot of regression pearson r over negative test RMSE\n",
    "df_temp = df_regr_results.iloc[:]\n",
    "\n",
    "sns.set(font_scale = 1.1)\n",
    "sns.set_style(\"whitegrid\")\n",
    "f, (ax1) = plt.subplots(1, 1, figsize=(15, 10))\n",
    "sns.despine(left=True, bottom=True)\n",
    "\n",
    "sns.scatterplot(\n",
    "            data=df_temp,\n",
    "            x='pearsonr',\n",
    "            y='neg_rmse_test',\n",
    "            ax=ax1,\n",
    "            # hue=df_temp.index,\n",
    "            hue='delta',\n",
    "            palette='colorblind',\n",
    "            s=250,\n",
    "            marker='X'\n",
    "        ).set(\n",
    "            # xlabel='false labels ratio (%)',\n",
    "            # ylabel='model accuracy',\n",
    "            # ylim=(0.5,1),\n",
    "            title='pearsonr over neg_rmse_train'\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# view specific regression results - TODO: select by index\n",
    "df_temp = df_regr_results.iloc[[32, 34, 36, 38]]\n",
    "df_temp[['n', 'pearsonr', 'p_value', 'neg_rmse_train', 'neg_rmse_test', 'r2_train', 'r2_test']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --> TODO: select index of regression to use for y_true / y_pred visualization <--\n",
    "index = [32, 34, 36, 38]\n",
    "\n",
    "for ind in index:\n",
    "    print('##############################################')\n",
    "    print(f'\\nData for regression with index {ind}')\n",
    "    print(f'''\n",
    "        cnn : {df_regr_results.iloc[ind]['model']}\n",
    "        classification_task : {df_regr_results.iloc[ind]['classification_task']}\n",
    "        delta : {df_regr_results.iloc[ind]['delta']}\n",
    "        grouping : {df_regr_results.iloc[ind]['group_by']}\n",
    "        zscore_threshold : {df_regr_results.iloc[ind]['zscore_threshold']}\n",
    "    ''')\n",
    "    # set regression parameters based on row of regression result table using index\n",
    "    regressor = df_regr_results.iloc[ind]['regressor']\n",
    "\n",
    "    cnn = df_regr_results.iloc[ind]['model']\n",
    "    if cnn == 'all': cnn = False\n",
    "\n",
    "    classes = df_regr_results.iloc[ind]['classification_task']\n",
    "    if classes == 'all': classes = False\n",
    "\n",
    "    use_delta = df_regr_results.iloc[ind]['delta']\n",
    "\n",
    "    group_by = df_regr_results.iloc[ind]['group_by']\n",
    "    group_by = ast.literal_eval(group_by)\n",
    "    use_grouping = any(group_by.values())\n",
    "\n",
    "    random_state = df_regr_results.iloc[ind]['split_random_state']\n",
    "    zscore_threshold = df_regr_results.iloc[ind]['zscore_threshold']\n",
    "\n",
    "    best_params = df_regr_results.iloc[ind]['best_params']\n",
    "    best_params = ast.literal_eval(best_params)\n",
    "\n",
    "    estimator_name = df_regr_results.iloc[ind]['estimator'][:-2]\n",
    "    \n",
    "    # load classification results table\n",
    "    df_class_results_raw = util.load_aggr_class_reports(aggr_class_report_log_path)\n",
    "\n",
    "    # process data based on parameters\n",
    "    df_regression = df_class_results_raw.query(f'metric == \"{regressor}\"')\n",
    "    df_regression = df_regression.drop(['run'], axis=1)\n",
    "    df_regression = df_regression.rename(columns={'value': regressor})\n",
    "    df_regression = util.calc_delta(df_class_results_raw, df_regression, regressor)\n",
    "\n",
    "    # filter data for set properties\n",
    "    df_regression = util.filter_data(\n",
    "        use_grouping, group_by, df_regression, cnn, classes\n",
    "    )\n",
    "\n",
    "    # filter outliers if properties are set\n",
    "    df_regression = util.filter_outliers(\n",
    "        df_regression, regressor, use_grouping, zscore_threshold\n",
    "    )\n",
    "\n",
    "    df_regression.reset_index(inplace=True, drop=True)\n",
    "    print(f'\\tRegression df.shape: {df_regression.shape}')\n",
    "    \n",
    "    # get independent and dependent variables\n",
    "    if use_delta:\n",
    "        X_df = df_regression[[f'{regressor}_delta']]\n",
    "    else:\n",
    "        X_df = df_regression[[regressor]]\n",
    "        # X_df = df_regression.drop([f'{regressor}_delta', 'ratio'], axis=1)\n",
    "    y_df = df_regression['ratio']\n",
    "\n",
    "    # split data - random state to reproduce split in initial model selection\n",
    "    test_split = 0.2 if len(df_regression) > 100 else 0.1\n",
    "    if use_grouping:\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "                                                    X_df\n",
    "                                                    , y_df\n",
    "                                                    , test_size=test_split\n",
    "                                                    , random_state=random_state\n",
    "                                                )\n",
    "    else:\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "                                                    X_df\n",
    "                                                    , y_df\n",
    "                                                    , test_size=test_split\n",
    "                                                    , random_state=random_state\n",
    "                                                    , stratify=y_df\n",
    "                                                )\n",
    "\n",
    "    # scale X data\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    \n",
    "    # init estimator, set best params\n",
    "    if estimator_name == 'LinearRegression':\n",
    "        estimator = LinearRegression()\n",
    "    else:\n",
    "        estimator, temp_params = util.init_estimator(estimator_name)\n",
    "    estimator.set_params(**best_params)\n",
    "\n",
    "    # train estimator\n",
    "    estimator.fit(X_train, y_train)\n",
    "    y_train_pred = estimator.predict(X_train)\n",
    "\n",
    "    print(f'\\n{estimator_name}')\n",
    "    if estimator.coef_:\n",
    "        print(f'\\t{estimator_name} coef: {estimator.coef_}')\n",
    "    if estimator.intercept_:\n",
    "        print(f'\\t{estimator_name} intercept: {estimator.intercept_}')\n",
    "\n",
    "    y_pred = estimator.predict(X_test)\n",
    "\n",
    "    print(f'''\n",
    "        {df_regr_results.iloc[ind]['n']} observations\n",
    "        ---------\n",
    "        Regressor: {regressor}\n",
    "            {round(df_regression[regressor].mean(), 5)} mean\n",
    "            {round(df_regression[regressor].std(), 5)} std\n",
    "        Regressand: False Labels Ratio\n",
    "            {round(df_regression['ratio'].mean(), 5)} mean\n",
    "            {round(df_regression['ratio'].std(), 5)} std\n",
    "\n",
    "        Estimator: {estimator_name}\n",
    "            {round(df_regr_results.iloc[ind]['pearsonr'], 5)} : pearsonr\n",
    "            {round(df_regr_results.iloc[ind]['p_value'], 5)} : p_value\n",
    "            {round(df_regr_results.iloc[ind]['neg_rmse_train'], 5)} : neg_rmse_train\n",
    "            {round(df_regr_results.iloc[ind]['neg_rmse_test'], 5)} : neg_rmse_test\n",
    "            {round(df_regr_results.iloc[ind]['r2_train'], 5)} : r2_train\n",
    "            {round(df_regr_results.iloc[ind]['r2_test'], 5)} : r2_test\n",
    "    ''')\n",
    "\n",
    "    # visualize estimator performance\n",
    "    sns.set(rc={'figure.figsize':(11,8)})\n",
    "    ax = sns.scatterplot(x=y_test, y=y_pred)\n",
    "    ax.set(xlabel='True false labels ratio', ylabel='Predicted false labels ratio')\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.12 ('false-labels')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c041ee5eca0bebec98353fb08f3f656fe2677ac141976eeebd4959b2f854b27b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}