collate_csvs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e65b970-fb60-438f-9114-7f88d87dfc35",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "634e8e57-e410-47e2-8699-242a365d2274",
   "metadata": {},
   "source": [
    "# Collate ACIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "217c6b17-9020-43d3-9829-b53d9e4e5ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = input(\"Experiment_name. This is the file_name with which u issued the command to run.\")\n",
    "acic_dir = Path(f\"results/experiments_benchmarking/acic2016\")\n",
    "acic_format = \"v{}_\" + experiment_name + \".csv\"\n",
    "collated_acic = {\n",
    "    \"seed\": [],\n",
    "}\n",
    "acic_dfs = []\n",
    "for seed in [2, 7, 26]:\n",
    "    acic_file = acic_format.format(str(seed), str(seed))\n",
    "    df = pd.read_csv(acic_dir / acic_file)\n",
    "    acic_dfs.append(df)\n",
    "    collated_acic[\"seed\"].extend([seed] * len(df))\n",
    "acic_df = pd.concat(acic_dfs)\n",
    "\n",
    "for key, value in collated_acic.items():\n",
    "    acic_df[key] = value\n",
    "print(acic_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e1fa6d3-3dc7-4020-9a3c-3cde0b1876e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "acic_df.to_csv(acic_dir / f\"collated_results-{experiment_name}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57eabe04-c39c-48b3-9b6a-474d7e918196",
   "metadata": {},
   "source": [
    "# Collate Twins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fab0a96-f253-4aeb-ac4e-b507917cc10b",
   "metadata": {},
   "outputs": [],
   "source": [
    "twins_dir = Path(\"results/experiments_inductive_bias/twins\")\n",
    "experiment_name = input(\"Experiment name. This is the file_name with which u issued the command to run.\")\n",
    "twins_format = experiment_name + \"_{}_{}.csv\"\n",
    "collated_twins = {\n",
    "    \"prop_t\": [],\n",
    "    \"num_trn\": []\n",
    "}\n",
    "twins_df = []\n",
    "for prop_treated in [0.1, 0.25, 0.5, 0.75, 0.9]:\n",
    "    for num_trn in [500, 1000, 4000, \"None\"]:\n",
    "        twins_file = twins_dir / twins_format.format(str(prop_treated), str(num_trn))\n",
    "        df = pd.read_csv(twins_file)\n",
    "        \n",
    "        collated_twins[\"prop_t\"].extend([prop_treated] * len(df))\n",
    "        collated_twins[\"num_trn\"].extend([num_trn] * len(df))\n",
    "        twins_df.append(df)\n",
    "twins_df = pd.concat(twins_df)\n",
    "for key, value in collated_twins.items():\n",
    "    twins_df[key] = value\n",
    "print(twins_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13c06412-f9a6-4f1f-8dd4-0d51680eac42",
   "metadata": {},
   "outputs": [],
   "source": [
    "twins_df.to_csv(twins_dir / f\"collated_results-{experiment_name}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6a33175-679a-480f-b064-e6e967878509",
   "metadata": {},
   "source": [
    "# Final Table"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "953c842e",
   "metadata": {},
   "source": [
    "## Discrete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6543a278",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import numpy as np\n",
    "from scipy.stats import ttest_ind\n",
    "from pathlib import Path\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dad6729a",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = input(\"Experiment name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42627580",
   "metadata": {},
   "outputs": [],
   "source": [
    "def t_test(x, y, alternative=\"lesser\"):\n",
    "    # Code taken from: https://stackoverflow.com/questions/15984221/how-to-perform-two-sample-one-tailed-t-test-with-numpy-scipy\n",
    "    if type(x) == torch.Tensor:\n",
    "        x = x.numpy()\n",
    "    if type(y) == torch.Tensor:\n",
    "        y = y.numpy()\n",
    "    _, double_p = ttest_ind(x, y, equal_var=False)\n",
    "    if alternative == \"both-sided\":\n",
    "        pval = double_p\n",
    "    elif alternative == \"greater\":\n",
    "        if np.mean(x) > np.mean(y):\n",
    "            pval = double_p / 2.0\n",
    "        else:\n",
    "            pval = 1.0 - double_p / 2.0\n",
    "    elif alternative == \"lesser\":\n",
    "        if np.mean(x) < np.mean(y):\n",
    "            pval = double_p / 2.0\n",
    "        else:\n",
    "            pval = 1.0 - double_p / 2.0\n",
    "    return pval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1086b1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "algos = ['TNet', 'RNet', 'DRNet', 'XNet', 'TARNet', 'CFRNet', 'DragonNet', 'FlexTENet', 'PairNet']\n",
    "print(algos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6f3f0d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "ihdp_csv = pd.read_csv(f\"results/experiments_benchmarking/ihdp/{experiment_name}.csv\")\n",
    "acic_csv = pd.read_csv(f\"results/experiments_benchmarking/acic2016/collated_results-{experiment_name}.csv\")\n",
    "twins_csv = pd.read_csv(f\"results/experiments_inductive_bias/twins/collated_results-{experiment_name}.csv\")\n",
    "\n",
    "csvs = {\n",
    "    \"IHDP\": ihdp_csv,\n",
    "    \"ACIC\": acic_csv,\n",
    "    \"Twins\": twins_csv\n",
    "}\n",
    "\n",
    "ihdp_csv_main = pd.read_csv(f\"results/experiments_benchmarking/ihdp/collated_results_main-ihdp.csv\")\n",
    "acic_csv_main = pd.read_csv(f\"results/experiments_benchmarking/acic2016/collated_results_main-acic.csv\")\n",
    "twins_csv_main = pd.read_csv(f\"results/experiments_inductive_bias/twins/collated_results_cate_in_out.csv\")\n",
    "\n",
    "main_csvs = {\n",
    "    \"IHDP\": ihdp_csv_main,\n",
    "    \"ACIC\": acic_csv_main,\n",
    "    \"Twins\": twins_csv_main\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6820455",
   "metadata": {},
   "outputs": [],
   "source": [
    "ihdp_csv_main.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6992274b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dict = {\n",
    "    \"ITE in\": [],\n",
    "    \"ITE out\": [],\n",
    "}\n",
    "\n",
    "results_df = []\n",
    "\n",
    "for did, dataset in enumerate(csvs.keys()):\n",
    "    print(dataset)\n",
    "    \n",
    "    ds_df = deepcopy(df_dict)\n",
    "    \n",
    "    df = csvs[dataset]\n",
    "    df_main = main_csvs[dataset]\n",
    "        \n",
    "    for algo in algos:\n",
    "        ref_algo = algo if algo != \"PairNet\" else \"FCTNBRNet\"\n",
    "        \n",
    "        if dataset == 'Twins':\n",
    "            in_name = '_cate_in'\n",
    "            out_name = '_cate_out'\n",
    "        else:\n",
    "            in_name = '_in'\n",
    "            out_name = '_out'\n",
    "        \n",
    "        t_ref_in = df_main[ref_algo + in_name].values\n",
    "        t_ref_out = df_main[ref_algo + out_name].values\n",
    "        \n",
    "        in_name = algo + in_name\n",
    "        out_name = algo + out_name\n",
    "        \n",
    "        t_in = df[in_name].values\n",
    "        t_out = df[out_name].values\n",
    "        \n",
    "        pval_in = t_test(t_ref_in, t_in)\n",
    "        pval_out = t_test(t_ref_out, t_out)\n",
    "        \n",
    "        mean_in = np.mean(t_ref_in) - np.mean(t_in)\n",
    "        mean_out = np.mean(t_ref_out) - np.mean(t_out)\n",
    "        \n",
    "        ds_df[\"ITE in\"].append(f\"{round(mean_in, 2):.2f} ({round(pval_in, 2):.2f})\")\n",
    "        ds_df[\"ITE out\"].append(f\"{round(mean_out, 2):.2f} ({round(pval_out, 2):.2f})\")\n",
    "    \n",
    "    results_df.append(pd.DataFrame(ds_df))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0c55e42",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_all = pd.concat(results_df, axis=1)\n",
    "results_all.index = algos\n",
    "print(results_all.to_latex())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "catenets",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "vscode": {
   "interpreter": {
    "hash": "77d077712a57853e150bafe84dc038be690e04e1522d83a0d5708d9437d435a7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}