inference_colwise_gittables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import json\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "from time import time\n",
    "import mlflow\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "from collections import defaultdict\n",
    "\n",
    "# import pytrec_eval\n",
    "import torch\n",
    "from torch.nn import CrossEntropyLoss, BCEWithLogitsLoss\n",
    "from torch.utils.data import DataLoader, RandomSampler\n",
    "import torch.nn.functional as F\n",
    "from transformers import BertTokenizer, BertForSequenceClassification, BertConfig, AutoTokenizer\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "from accelerate import Accelerator\n",
    "from copy import deepcopy\n",
    "\n",
    "torch.backends.cuda.matmul.allow_tf32 = True\n",
    "\n",
    "from watchog.dataset import (\n",
    "    # collate_fn,\n",
    "    TURLColTypeTablewiseDataset,\n",
    "    TURLRelExtTablewiseDataset,\n",
    "    SatoCVTablewiseDataset,\n",
    "    ColPoplTablewiseDataset\n",
    ")\n",
    "\n",
    "from watchog.dataset import TableDataset, SupCLTableDataset, SemtableCVTablewiseDataset, GittablesColwiseDataset, GittablesTablewiseDataset\n",
    "from watchog.model import BertMultiPairPooler, BertForMultiOutputClassification, BertForMultiOutputClassificationColPopl\n",
    "from watchog.model import SupCLforTable, UnsupCLforTable, lm_mp\n",
    "from watchog.utils import load_checkpoint, f1_score_multilabel, collate_fn, get_col_pred, ColPoplEvaluator\n",
    "from watchog.utils import task_num_class_dict\n",
    "from accelerate import DistributedDataParallelKwargs\n",
    "import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "import torch\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import json\n",
    "import re\n",
    "import transformers\n",
    "from torch.utils import data\n",
    "from torch.nn.utils import rnn\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "from typing import List\n",
    "from functools import reduce\n",
    "import operator\n",
    "\n",
    "from itertools import chain\n",
    "import copy\n",
    "class SingleColDataset(data.Dataset):\n",
    "    def __init__(\n",
    "            self,\n",
    "            group_df,\n",
    "            target_col_idx: int,\n",
    "            tokenizer: AutoTokenizer,\n",
    "            max_length: int = 64,\n",
    "            device: torch.device = None,\n",
    "            max_num_col=8,\n",
    "            random_sample=False, # TODO\n",
    "            train_only=True,\n",
    "            context_encoding_type=\"v1\",\n",
    "            adaptive_max_length=False,\n",
    "            ): # TODO: not used\n",
    "        if device is None:\n",
    "            device = torch.device('cpu')\n",
    "        assert max_num_col >= 1, \"max_num_col must be greater than 1\"\n",
    "        assert 512//max_num_col >= max_length  , \"max_length cannot be smaller than 512//max_num_col\"\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "        data_list = []\n",
    "        \n",
    "\n",
    "\n",
    " \n",
    "        \n",
    "        # df.drop(df[(df['data'] == '') & (df['class_id'] == -1)].index, inplace=True)\n",
    "        group_df.sort_values(by=['col_idx'], inplace=True)\n",
    "        labeled_columns = group_df[group_df['class_id'] > -1]\n",
    "        # if len(labeled_columns) > 1:\n",
    "        #     print(\"Here\")\n",
    "        unlabeled_columns = group_df[group_df['class_id'] == -1]\n",
    "        for idx in [target_col_idx]:\n",
    "            token_ids_list = []\n",
    "            target_column = labeled_columns.loc[idx]\n",
    "            rest_columns = labeled_columns[labeled_columns.index != idx] # the rest of labeled columns\n",
    "            # tokenize target column\n",
    "            token_ids_list.append(tokenizer.encode(\n",
    "            tokenizer.cls_token + \" \" + target_column[\"data\"], add_special_tokens=False, max_length=max_length, truncation=True))\n",
    "            # tokenize context\n",
    "            unlabeled_columns = unlabeled_columns\n",
    "\n",
    "            group_df = pd.concat([rest_columns, unlabeled_columns])\n",
    "            group_df = group_df[: max_num_col-1] if not random_sample  else group_df.sample(min(max_num_col-1, len(group_df))) # TODO\n",
    "            group_df.sort_values(by=['col_idx'], inplace=True)\n",
    "            \n",
    "            if adaptive_max_length and (len(group_df)+1) < max_num_col:\n",
    "                cur_maxlen = 512 // (len(group_df)+1)\n",
    "            else:\n",
    "                cur_maxlen = max_length\n",
    "                \n",
    "            if context_encoding_type == \"v0\":\n",
    "                token_ids_list += group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "                tokenizer.cls_token + \" \" + x, add_special_tokens=False, max_length=cur_maxlen, truncation=True)).tolist(\n",
    "                )\n",
    "            elif context_encoding_type == \"v0.1\": # only one cls token to separate context and target\n",
    "                token_ids_list.append([tokenizer.cls_token_id])\n",
    "                token_ids_list += group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "                \" \" + x, add_special_tokens=False, max_length=cur_maxlen-1, truncation=True)).tolist(\n",
    "                )                    \n",
    "            elif context_encoding_type == \"v1\":\n",
    "                token_ids_list += group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "                tokenizer.sep_token + \" \" + x, add_special_tokens=False, max_length=cur_maxlen, truncation=True)).tolist(\n",
    "                )                    \n",
    "            elif context_encoding_type == \"v1.1\": # only one cls token to separate context and target\n",
    "                token_ids_list.append([tokenizer.sep_token_id])\n",
    "                token_ids_list += group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "                \" \" + x, add_special_tokens=False, max_length=cur_maxlen-1, truncation=True)).tolist(\n",
    "                )   \n",
    "            else:\n",
    "                raise ValueError(\"context_encoding_type {} is not supported\".format(context_encoding_type))\n",
    "            token_ids = torch.LongTensor(reduce(operator.add,\n",
    "                                                token_ids_list)).to(device)\n",
    "            cls_index_list = [0] \n",
    "            for cls_index in cls_index_list:\n",
    "                assert token_ids[\n",
    "                    cls_index] == tokenizer.cls_token_id, \"cls_indexes validation\"\n",
    "            cls_indexes = torch.LongTensor(cls_index_list).to(device)\n",
    "            class_ids = torch.LongTensor([target_column[\"class_id\"]]\n",
    "                ).to(device)\n",
    "            data_list.append(\n",
    "                [0,\n",
    "                len(group_df), token_ids, class_ids, cls_indexes])\n",
    "        self.table_df = pd.DataFrame(data_list,\n",
    "                                     columns=[\n",
    "                                         \"table_id\", \"num_col\", \"data_tensor\",\n",
    "                                         \"label_tensor\", \"cls_indexes\"\n",
    "                                     ])\n",
    "        \"\"\"\n",
    "        # NOTE: msato contains a small portion of single-col tables. keep it to be consistent.  \n",
    "        if multicol_only:\n",
    "            # Check\n",
    "            num_all_tables = len(self.table_df)\n",
    "            self.table_df = self.table_df[self.table_df[\"num_col\"] > 1]\n",
    "            assert len(self.table_df) == num_all_tables\n",
    "        \"\"\"\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.table_df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            \"data\": self.table_df.iloc[idx][\"data_tensor\"],\n",
    "            \"label\": self.table_df.iloc[idx][\"label_tensor\"]\n",
    "        }\n",
    "        #\"idx\": torch.LongTensor([idx])}\n",
    "        #\"cls_indexes\": self.table_df.iloc[idx][\"cls_indexes\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "args={\"wandb\": false, \"model\": \"Watchog\", \"unlabeled_train_only\": false, \"context_encoding_type\": \"v0\", \"pool_version\": \"v0.2\", \"random_sample\": false, \"comment\": \"debug\", \"shortcut_name\": \"bert-base-uncased\", \"max_length\": 64, \"adaptive_max_length\": false, \"max_num_col\": 8, \"batch_size\": 16, \"epoch\": 1, \"random_seed\": 4649, \"train_n_seed_cols\": -1, \"num_classes\": 101, \"multi_gpu\": false, \"fp16\": false, \"warmup\": 0.0, \"lr\": 5e-05, \"task\": \"gt-semtab22-dbpedia-all0\", \"colpair\": false, \"metadata\": false, \"from_scratch\": false, \"cl_tag\": \"wikitables/simclr/bert_100000_10_32_256_5e-05_sample_row4,sample_row4_tfidf_entity_column_0.05_0_last.pt\", \"dropout_prob\": 0.5, \"eval_test\": true, \"small_tag\": \"semi1\", \"data_path\": \"/data/zhihao/TU/\", \"pretrained_ckpt_path\": \"/data/zhihao/TU/Watchog/model/\"}\n",
      "gt-semtab22-dbpedia-all0/wikitables-simclr-bert_100000_10_32_256_5e-05_sample_row4,sample_row4_tfidf_entity_column_0.05_0_last.pt_bert-base-uncased-poolsemi1-max_colsv0.2-rand8-bsFalse-ml16-ne64-do10.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/3314320431.py:214: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ckpt = torch.load(ckpt_path, map_location=device)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(augment_op='sample_row4,sample_row4', batch_size=32, data_path='/data/zhihao/TU/TURL/', fp16=True, gpus='0', lm='bert', logdir='/data/zhihao/TU/Watchog/model/', lr=5e-05, max_len=256, mode='simclr', model='Watchog', n_epochs=10, pretrain_data='wikitables', pretrained_model_path='', projector=768, run_id=0, sample_meth='tfidf_entity', save_model=10, single_column=False, size=100000, table_order='column', temperature=0.05)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    }
   ],
   "source": [
    "    device = torch.device(2)\n",
    "\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument(\"--wandb\", type=bool, default=False)\n",
    "    parser.add_argument(\"--model\", type=str, default=\"Watchog\")\n",
    "    parser.add_argument(\"--unlabeled_train_only\", type=bool, default=False)\n",
    "    parser.add_argument(\"--context_encoding_type\", type=str, default=\"v0\")\n",
    "    parser.add_argument(\"--pool_version\", type=str, default=\"v0.2\")\n",
    "    parser.add_argument(\"--random_sample\", type=bool, default=False)\n",
    "    parser.add_argument(\"--comment\", type=str, default=\"debug\", help=\"to distinguish the runs\")\n",
    "    parser.add_argument(\n",
    "        \"--shortcut_name\",\n",
    "        default=\"bert-base-uncased\",\n",
    "        type=str,\n",
    "        help=\"Huggingface model shortcut name \",\n",
    "    )\n",
    "    parser.add_argument(\n",
    "        \"--max_length\",\n",
    "        default=64,\n",
    "        type=int,\n",
    "        help=\n",
    "        \"The maximum total input sequence length after tokenization. Sequences longer \"\n",
    "        \"than this will be truncated, sequences shorter will be padded.\",\n",
    "    )\n",
    "    parser.add_argument(\n",
    "        \"--adaptive_max_length\",\n",
    "        default=False,\n",
    "        type=bool,\n",
    "    )    \n",
    "    parser.add_argument(\n",
    "        \"--max_num_col\",\n",
    "        default=8,\n",
    "        type=int,\n",
    "    )   \n",
    "\n",
    "    parser.add_argument(\n",
    "        \"--batch_size\",\n",
    "        default=16,\n",
    "        type=int,\n",
    "        help=\"Batch size\",\n",
    "    )\n",
    "    parser.add_argument(\n",
    "        \"--epoch\",\n",
    "        default=1,\n",
    "        type=int,\n",
    "        help=\"Number of epochs for training\",\n",
    "    )\n",
    "    parser.add_argument(\n",
    "        \"--random_seed\",\n",
    "        default=4649,\n",
    "        type=int,\n",
    "        help=\"Random seed\",\n",
    "    )\n",
    "    \n",
    "    parser.add_argument(\n",
    "        \"--train_n_seed_cols\",\n",
    "        default=-1,\n",
    "        type=int,\n",
    "        help=\"number of seeding columns in training\",\n",
    "    )\n",
    "\n",
    "    parser.add_argument(\n",
    "        \"--num_classes\",\n",
    "        default=78,\n",
    "        type=int,\n",
    "        help=\"Number of classes\",\n",
    "    )\n",
    "    parser.add_argument(\"--multi_gpu\",\n",
    "                        action=\"store_true\",\n",
    "                        default=False,\n",
    "                        help=\"Use multiple GPU\")\n",
    "    parser.add_argument(\"--fp16\",\n",
    "                        action=\"store_true\",\n",
    "                        default=False,\n",
    "                        help=\"Use FP16\")\n",
    "    parser.add_argument(\"--warmup\",\n",
    "                        type=float,\n",
    "                        default=0.,\n",
    "                        help=\"Warmup ratio\")\n",
    "    parser.add_argument(\"--lr\", type=float, default=5e-5, help=\"Learning rate\")\n",
    "    parser.add_argument(\"--task\",\n",
    "                        type=str,\n",
    "                        default='gt-semtab22-dbpedia-all0',\n",
    "                        choices=[\n",
    "                            \"sato0\", \"sato1\", \"sato2\", \"sato3\", \"sato4\",\n",
    "                            \"msato0\", \"msato1\", \"msato2\", \"msato3\", \"msato4\",\n",
    "                            \"gt-dbpedia0\", \"gt-dbpedia1\", \"gt-dbpedia2\", \"gt-dbpedia3\", \"gt-dbpedia4\",\n",
    "                            \"gt-dbpedia-all0\", \"gt-dbpedia-all1\", \"gt-dbpedia-all2\", \"gt-dbpedia-all3\", \"gt-dbpedia-all4\",\n",
    "                            \"gt-schema-all0\", \"gt-schema-all1\", \"gt-schema-all2\", \"gt-schema-all3\", \"gt-schema-all4\",\n",
    "                            \"gt-semtab22-dbpedia\", \"gt-semtab22-dbpedia0\", \"gt-semtab22-dbpedia1\", \"gt-semtab22-dbpedia2\", \"gt-semtab22-dbpedia3\", \"gt-semtab22-dbpedia4\",\n",
    "                            \"gt-semtab22-dbpedia-all\", \"gt-semtab22-dbpedia-all0\", \"gt-semtab22-dbpedia-all1\", \"gt-semtab22-dbpedia-all2\", \"gt-semtab22-dbpedia-all3\", \"gt-semtab22-dbpedia-all4\",\n",
    "                            \"gt-semtab22-schema-class-all\", \"gt-semtab22-schema-property-all\",\n",
    "                            \"turl\", \"turl-re\", \"col-popl-1\", \"col-popl-2\", \"col-popl-3\", \"row-popl\",\n",
    "                            \"col-popl-turl-0\", \"col-popl-turl-1\", \"col-popl-turl-2\",\n",
    "                            \"col-popl-turl-mdonly-0\", \"col-popl-turl-mdonly-1\", \"col-popl-turl-mdonly-2\"\n",
    "                        ],\n",
    "                        help=\"Task names}\")\n",
    "    parser.add_argument(\"--colpair\",\n",
    "                        action=\"store_true\",\n",
    "                        help=\"Use column pair embedding\")\n",
    "    parser.add_argument(\"--metadata\",\n",
    "                        action=\"store_true\",\n",
    "                        help=\"Use column header metadata\")\n",
    "    parser.add_argument(\"--from_scratch\",\n",
    "                        action=\"store_true\",\n",
    "                        help=\"Training from scratch\")\n",
    "    parser.add_argument(\"--cl_tag\",\n",
    "                        type=str,\n",
    "                        default=\"wikitables/simclr/bert_100000_10_32_256_5e-05_sample_row4,sample_row4_tfidf_entity_column_0.05_0_last.pt\",\n",
    "                        help=\"path to the pre-trained file\")\n",
    "    parser.add_argument(\"--dropout_prob\",\n",
    "                        type=float,\n",
    "                        default=0.5)\n",
    "    parser.add_argument(\"--eval_test\",\n",
    "                        action=\"store_true\",\n",
    "                        help=\"evaluate on testset and do not save the model file\")\n",
    "    parser.add_argument(\"--small_tag\",\n",
    "                        type=str,\n",
    "                        default=\"semi1\",\n",
    "                        help=\"e.g., by_table_t5_v1\")\n",
    "    parser.add_argument(\"--data_path\",\n",
    "                        type=str,\n",
    "                        default=\"/data/zhihao/TU/\")\n",
    "    parser.add_argument(\"--pretrained_ckpt_path\",\n",
    "                        type=str,\n",
    "                        default=\"/data/zhihao/TU/Watchog/model/\")    \n",
    "\n",
    "    args = parser.parse_args([])\n",
    "    task = args.task\n",
    "    if args.small_tag != \"\":\n",
    "        args.eval_test = True\n",
    "    \n",
    "    args.num_classes = task_num_class_dict[task]\n",
    "    if args.colpair:\n",
    "        assert \"turl-re\" == task, \"colpair can be only used for Relation Extraction\"\n",
    "    if args.metadata:\n",
    "        assert \"turl-re\" == task or \"turl\" == task, \"metadata can be only used for TURL datasets\"\n",
    "    if \"col-popl\":\n",
    "        # metrics = {\n",
    "        #     \"accuracy\": CategoricalAccuracy(tie_break=True),\n",
    "        # }\n",
    "        if args.train_n_seed_cols != -1:\n",
    "            if \"col-popl\" in task:\n",
    "                assert args.train_n_seed_cols == int(task[-1]),  \"# of seed columns must match\"\n",
    "\n",
    "    print(\"args={}\".format(json.dumps(vars(args))))\n",
    "\n",
    "    max_length = args.max_length\n",
    "    batch_size = args.batch_size\n",
    "    num_train_epochs = args.epoch\n",
    "\n",
    "    shortcut_name = args.shortcut_name\n",
    "\n",
    "    if args.colpair and args.metadata:\n",
    "        taskname = \"{}-colpair-metadata\".format(task)\n",
    "    elif args.colpair:\n",
    "        taskname = \"{}-colpair\".format(task)\n",
    "    elif args.metadata:\n",
    "        taskname = \"{}-metadata\".format(task)\n",
    "    elif args.train_n_seed_cols == -1 and 'popl' in task:\n",
    "        taskname = \"{}-mix\".format(task)\n",
    "    else:\n",
    "        taskname = \"\".join(task)\n",
    "    cv = int(task[-1])\n",
    "\n",
    "    if args.from_scratch:\n",
    "        if \"gt\" in task:\n",
    "            tag_name = \"{}/{}-{}-{}-pool{}-max_cols{}-rand{}-bs{}-ml{}-ne{}-do{}{}\".format(\n",
    "                taskname,  \"{}-fromscratch\".format(shortcut_name), args.small_tag, args.comment, args.pool_version, args.max_num_col, args.random_sample,\n",
    "                batch_size, max_length, num_train_epochs, args.dropout_prob, \n",
    "                '-rs{}'.format(args.random_seed) if args.random_seed != 4649 else '')\n",
    "        else:\n",
    "            tag_name = \"{}/{}-{}-{}-bs{}-ml{}-ne{}-do{}{}\".format(\n",
    "                taskname,  \"{}-fromscratch\".format(shortcut_name), args.small_tag, args.comment, \n",
    "                batch_size, max_length, num_train_epochs, args.dropout_prob, \n",
    "                '-rs{}'.format(args.random_seed) if args.random_seed != 4649 else '')\n",
    "        \n",
    "    else:\n",
    "        if \"gt\" in task:\n",
    "            tag_name = \"{}/{}_{}-pool{}-max_cols{}-rand{}-bs{}-ml{}-ne{}-do{}{}\".format(\n",
    "                taskname, args.cl_tag.replace('/', '-'),  shortcut_name, args.small_tag, args.pool_version, args.max_num_col, args.random_sample,\n",
    "                batch_size, max_length, num_train_epochs, args.dropout_prob,\n",
    "                '-rs{}'.format(args.random_seed) if args.random_seed != 4649 else '')\n",
    "        else:\n",
    "            tag_name = \"{}/{}_{}-{}-bs{}-ml{}-ne{}-do{}{}\".format(\n",
    "                taskname, args.cl_tag.replace('/', '-'),  shortcut_name, args.small_tag,\n",
    "                batch_size, max_length, num_train_epochs, args.dropout_prob,\n",
    "                '-rs{}'.format(args.random_seed) if args.random_seed != 4649 else '')\n",
    "\n",
    "    # if args.eval_test:\n",
    "    #     if args.small_tag != '':\n",
    "    #         tag_name = tag_name.replace('outputs', 'small_outputs')\n",
    "    #         tag_name += '-' + args.small_tag\n",
    "    print(tag_name)\n",
    "    file_path = os.path.join(args.data_path, \"Watchog\", \"outputs\", tag_name)\n",
    "\n",
    "    dirpath = os.path.dirname(file_path)\n",
    "    if not os.path.exists(dirpath):\n",
    "        print(\"{} not exists. Created\".format(dirpath))\n",
    "        os.makedirs(dirpath)\n",
    "    \n",
    "    if args.fp16:\n",
    "        torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True\n",
    "        \n",
    "      \n",
    "        \n",
    "    # accelerator = Accelerator(mixed_precision=\"no\" if not args.fp16 else \"fp16\")   \n",
    "    ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)\n",
    "    accelerator = Accelerator(mixed_precision=\"no\" if not args.fp16 else \"fp16\", kwargs_handlers=[ddp_kwargs])\n",
    "\n",
    "    device = torch.device(2)\n",
    "    ckpt_path = os.path.join(args.pretrained_ckpt_path, args.cl_tag)\n",
    "    # ckpt_path = '/efs/checkpoints/{}.pt'.format(args.cl_tag)\n",
    "    ckpt = torch.load(ckpt_path, map_location=device)\n",
    "    ckpt_hp = ckpt['hp']\n",
    "    print(ckpt_hp)\n",
    " \n",
    "    setattr(ckpt_hp, 'batch_size', args.batch_size)\n",
    "    setattr(ckpt_hp, 'hidden_dropout_prob', args.dropout_prob)\n",
    "    setattr(ckpt_hp, 'shortcut_name', args.shortcut_name)\n",
    "    setattr(ckpt_hp, 'num_labels', args.num_classes)\n",
    "    \n",
    "    \n",
    "    \n",
    "    tokenizer = BertTokenizer.from_pretrained(shortcut_name)\n",
    "    padder = collate_fn(tokenizer.pad_token_id)\n",
    "    if task == \"turl-re\" and args.colpair:\n",
    "        model = BertForMultiOutputClassification(ckpt_hp, device=device, lm=ckpt['hp'].lm, col_pair='Pair')\n",
    "    elif \"col-popl\" in task:\n",
    "        model = BertForMultiOutputClassificationColPopl(ckpt_hp, device=device, lm=ckpt['hp'].lm, n_seed_cols=int(task[i][-1]), cls_for_md=\"md\" in task)\n",
    "    else:\n",
    "        model = BertForMultiOutputClassification(ckpt_hp, device=device, lm=ckpt['hp'].lm, version=\"v0\", use_attention_mask=True)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bert', 'is', 'a', 'powerful', 'language', 'model', '.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"BERT is a powerful language model.\"\n",
    "tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "test 750\n"
     ]
    }
   ],
   "source": [
    "dataset_cls = GittablesTablewiseDataset\n",
    "cv = 0\n",
    "src = None\n",
    "# train_dataset = dataset_cls(cv=cv,\n",
    "#                             split=\"train\",\n",
    "#                             src=src,\n",
    "#                             tokenizer=tokenizer,\n",
    "#                             max_length=max_length,\n",
    "#                             gt_only='all' not in task,\n",
    "#                             device=device,\n",
    "#                             base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "#                             small_tag=\"semi1\")\n",
    "# valid_dataset = dataset_cls(cv=cv,\n",
    "#                             split=\"valid\", src=src,\n",
    "#                             tokenizer=tokenizer,\n",
    "#                             max_length=max_length,\n",
    "#                             gt_only='all' not in task,\n",
    "#                             device=device,\n",
    "#                             base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "#                             small_tag=\"semi1\")\n",
    "test_dataset = dataset_cls(cv=cv,\n",
    "                            split=\"test\", src=src,\n",
    "                            tokenizer=tokenizer,\n",
    "                            max_length=max_length,\n",
    "                            gt_only='all' not in task,\n",
    "                            device=device,\n",
    "                            base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "                            small_tag=\"semi1\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_dataloader = DataLoader(train_dataset,\n",
    "#                                 batch_size=batch_size,\n",
    "#                             #   collate_fn=collate_fn)\n",
    "#                             collate_fn=padder)\n",
    "# valid_dataloader = DataLoader(valid_dataset,\n",
    "#                                 batch_size=batch_size,\n",
    "#                             #   collate_fn=collate_fn)\n",
    "#                             collate_fn=padder)\n",
    "test_dataloader = DataLoader(test_dataset,\n",
    "                                batch_size=1,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_cls_tokens(hidden_states, cls_indexes, head=False):\n",
    "    cls_embeddings = []\n",
    "    for i, j in cls_indexes:\n",
    "        sub_sentence_cls_embeddings = hidden_states[i, 0, :] if head else hidden_states[i, j, :]\n",
    "        cls_embeddings.append(sub_sentence_cls_embeddings)\n",
    "    cls_embeddings = torch.stack(cls_embeddings)\n",
    "    return cls_embeddings\n",
    "#pooled_outputs = extract_cls_tokens(hidden_states, cls_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"label\"].cpu().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 768])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embs.detach().cpu().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_dataloader' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [274]\u001b[0m, in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m plm_embs_train \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m      4\u001b[0m labels_train \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 5\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch_idx, batch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[43mtrain_dataloader\u001b[49m):\n\u001b[1;32m      6\u001b[0m     cls_indexes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mnonzero(\n\u001b[1;32m      7\u001b[0m                     batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mT \u001b[38;5;241m==\u001b[39m tokenizer\u001b[38;5;241m.\u001b[39mcls_token_id)\n\u001b[1;32m      8\u001b[0m     embs \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mbert(batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mT)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_dataloader' is not defined"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "model = model.to(device)\n",
    "plm_embs_train = []\n",
    "labels_train = []\n",
    "for batch_idx, batch in enumerate(train_dataloader):\n",
    "    cls_indexes = torch.nonzero(\n",
    "                    batch[\"data\"].T == tokenizer.cls_token_id)\n",
    "    embs = model.bert(batch[\"data\"].T)\n",
    "    embs = extract_cls_tokens(embs[0], cls_indexes)\n",
    "    label = batch[\"label\"].cpu()\n",
    "    plm_embs_train.append(embs.detach().cpu()[label>-1])\n",
    "    labels_train.append(label[label>-1])\n",
    "plm_embs_train = torch.cat(plm_embs_train, dim=0)\n",
    "labels_train = torch.cat(labels_train, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3463, 768])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plm_embs_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import silhouette_samples\n",
    "from sklearn.metrics import silhouette_score\n",
    "def silhouette_score_per_cluster(embeddings, labels, metric=\"cosine\"):\n",
    "    sample_silhouette_values = silhouette_samples(embeddings, labels, metric=metric)\n",
    "    unique_labels = np.unique(labels)\n",
    "    silhouette_per_cluster = []\n",
    "    for label in unique_labels:\n",
    "        cluster_silhouettes = sample_silhouette_values[labels == label]\n",
    "        silhouette_per_cluster.append(np.mean(cluster_silhouettes))\n",
    "    return np.array(silhouette_per_cluster).reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "plm_embs_train_silhouette_values = silhouette_samples(plm_embs_train, labels_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.21431549"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(plm_embs_train_silhouette_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVHklEQVR4nO3df5BdZ13H8fe3aIsjxbQmxvQXQYmORTEkSys/VGpRC1WLDhYyCsEJ2ShthVodC/yBo8MMKgWkODWb0qE4QK0IEqACbQggIwU2sSZpKG2E1iYbmoQhgCLUlK9/3LOnt9m7u/cm99xzzu77NXPm3Pucc+9+mu7e7z3POed5IjORJAnglLoDSJKaw6IgSSpZFCRJJYuCJKlkUZAklb6v7gAnY+nSpbly5cq6Y0iazY4dnfXatfXm0GPs2LHjSGYu67Wt1UVh5cqVTE5O1h1D0mwiOmv/ThslIh6YbZvdR5KkkkVBklSyKEiSShYFSVKp1SeaJTWcY6u1jkcKkqSSRUGSVLIoSKrO2rXeuNYynlOQVJ2dO+tOoAFZFLSobbzqGqaOHJ3RftbSJWy5/rrRB5JqZlHQojZ15CinX/SKme3bb6whjVQ/zylIkkoeKUg97Nmzm0vXbXhMm11KWgwsClIPD+cpM7qV7FLSYmBRkFSdjRvrTqABWRQkVWdiou4EGpAnmiVJJYuCpOrs2PHolJxqBbuPJFVnbKyzdrTU1vBIQZJUsihIkkoWBUlSyaIgSSpZFCRJJYuCJKlUWVGIiHMjYntE7I2IuyPiVUX7n0XEgYi4q1he0PWa10TEvoj4UkT8alXZJI3I5GRnUWtUeZ/CMeCazNwZEacDOyLi9mLbWzLzTd07R8T5wEuApwJnAXdExE9k5iMVZpRUJafibJ3KjhQy82Bm7iwefwv4InD2HC+5DLglM7+bmV8B9gEXVJVPkjTTSM4pRMRK4OnA54qmKyNiV0TcFBFnFG1nAw92vWw/PYpIRIxHxGRETB4+fLjK2JJO1vh4Z1FrVF4UIuIJwD8Br87MbwI3AD8OrAYOAgPNWpKZE5k5lpljy5YtG3ZcScO0ZUtnUWtUWhQi4vvpFIR3Z+b7ATLzocx8JDO/B2zh0S6iA8C5XS8/p2iTJI1IlVcfBfAO4IuZ+eau9hVdu/0msKd4vBV4SUScFhFPBlYBn68qnyRppiqvPno28FJgd0TcVbS9FlgXEauBBO4HNgFk5t0RcSuwl86VS1d45ZEkjVZlRSEzPwNEj023zfGaNwBvqCqTJGlu3tEsSSo5yY6k6qxZU3cCDciiIKk6TsXZOnYfSZJKFgVJUsmiIKk6EZ1FrWFRkCSVLAqSpJJXH2lR2HjVNUwdOTqjfe8993LhRaPPIzWVRUGLwtSRo5x+0StmtH9n99U1pJGay+4jSVLJoiBJKtl9JKk6mzfXnUADsihIqo5TcbaO3UeSpJJFQVJ1JiY6i1rD7iNJ1dm0qbO2G6k1PFKQJJUsCpKkkkVBklTynIIWnF7jHDnGkdQfi4IWnF7jHDnGkdQfu48kSSWPFCRVJ7PuBBqQRUHq0549u7l03YYZ7WctXcKW66+rIZE0fBYFqU8P5yk952SY2n5jDWmkalgU1FrOptYCa9d21jt21JtDfbMoqLWaMpua3Upz2Lmz7gQakEVBOkl2K2khqeyS1Ig4NyK2R8TeiLg7Il5VtJ8ZEbdHxH3F+oyiPSLibRGxLyJ2RcSaqrJJknqr8j6FY8A1mXk+8HPAFRFxPnAtsC0zVwHbiucAzwdWFcs4cEOF2SRJPVRWFDLzYGbuLB5/C/gicDZwGXBzsdvNwAuLx5cB78qOO4ElEbGiqnySpJlGckdzRKwEng58DliemQeLTV8FlhePzwYe7HrZ/qLt+Pcaj4jJiJg8fPhwdaElaRGq/ERzRDwB+Cfg1Zn5zYgot2VmRsRAtzxm5gQwATA2NubtklKTbdxYdwINqNKiEBHfT6cgvDsz3180PxQRKzLzYNE9dKhoPwCc2/Xyc4o2SW3lVJytU+XVRwG8A/hiZr65a9NWYH3xeD3wwa72lxVXIf0c8I2ubiZJ0ghUeaTwbOClwO6IuKtoey3wRuDWiNgAPABcXmy7DXgBsA/4NvB7FWaTNArTdzJP39msxqusKGTmZ4CYZfPFPfZP4Iqq8kiqwdhYZ+1oqa3hHc1qhTbOptZr+AuHvlDTWRTUCm2cTa3X8BcOfaGmc+Y1SVLJoiBJKlkUJEkli4IkqeSJZknVmZysO4EGZFGQVB1vWmsdu48kSSWLgqTqjI93FrWGRUFSdbZs6SxqDYuCJKlkUZAklSwKkqSSRUGSVLIoSJJK3rwmqTpr1tSdQAOyKEiqzvR0nGoNi4IapdcMa9D8WdakhcKioEbpNcMaNH+WNWmh8ESzpOpEdBa1hkVBklSyKEiSSn0VhYh4dj9tkqR26/dI4fo+2yRJLTbn1UcR8UzgWcCyiPijrk1PBB5XZTBpIdqzZzeXrtswo/2spUvYcv11NSSSHmu+S1JPBZ5Q7Hd6V/s3gRdVFUpaqB7OU3pecju1/cYa0kgzzVkUMvNTwKci4p2Z+cCIMklaKDZvrjuBBtTvzWunRcQEsLL7NZn5S7O9ICJuAn4NOJSZP120/RmwEThc7PbazLyt2PYaYAPwCPCHmfmxgf5LJDWPU3G2Tr9F4R+BvwNupPOh3Y93Am8H3nVc+1sy803dDRFxPvAS4KnAWcAdEfETmdnvz5IkDUG/ReFYZt4wyBtn5qcjYmWfu18G3JKZ3wW+EhH7gAuAzw7yMyU1zMREZ+0RQ2v0e0nqhyLilRGxIiLOnF5O8GdeGRG7IuKmiDijaDsbeLBrn/1F2wwRMR4RkxExefjw4V67SGqKTZs6i1qj36KwHvgT4N+AHcUyeQI/7wbgx4HVwEFg4GvwMnMiM8cyc2zZsmUnEEGSNJu+uo8y88nD+GGZ+dD044jYAny4eHoAOLdr13OKNknSCPVVFCLiZb3aM/P4k8jzvc+KzDxYPP1NYE/xeCvwnoh4M50TzauAzw/y3pKkk9fvieZndD1+PHAxsJOZVxaVIuK9wHOBpRGxH3g98NyIWA0kcD+wCSAz746IW4G9wDHgCq88kqTR67f76Kru5xGxBLhlntes69H8jjn2fwPwhn7ySJKqcaJDZ/8PMJTzDJKk5uj3nMKH6HT5QGcgvJ8Cbq0qlKQFInP+fdQo/Z5T6L4D+RjwQGburyCPFomNV13D1JGjM9r33nMvF140+jySOvo9p/CpiFjOoyec76sukhaDqSNHe44W+p3dV9eQRtK0fmdeu5zOJaK/DVwOfC4iHDpb0tzWru0sao1+u49eBzwjMw8BRMQy4A7gfVUFk7QA7NxZdwINqN+rj06ZLgiFrw3wWklSS/R7pPDRiPgY8N7i+YuB26qJJEmqy3xzND8FWJ6ZfxIRvwU8p9j0WeDdVYeTJI3WfEcKbwVeA5CZ7wfeDxARP1Ns+/UKs0mSRmy+orA8M3cf35iZuweYQEfSPPbs2c2l6zbMaD9r6RK2XD/wCPPSCZuvKCyZY9sPDDGHtKg9nKf0vG9javuNNaQZoo0b606gAc1XFCYjYmNmbulujIhX0JloR5JmNz0dp1pjvqLwauADEfE7PFoExoBT6cyHIElaQOYsCsVMac+KiIuAny6aP5KZn6g8maT221F8l/Su5tbod+yj7cD2irNIWmjGxjprR0ttDe9KliSVLAqSpJJFQZJUsihIkkoWBUlSyaIgSSr1O3S2JA1ucrLuBBqQRUFSdbxprXXsPpIklSwKkqozPt5Z1BoWBUnV2bKls6g1LAqSpJJFQZJUqqwoRMRNEXEoIvZ0tZ0ZEbdHxH3F+oyiPSLibRGxLyJ2RcSaqnJJkmZX5ZHCO4FLjmu7FtiWmauAbcVzgOcDq4plHLihwlySpFlUdp9CZn46IlYe13wZ8Nzi8c3AJ4E/LdrflZkJ3BkRSyJiRWYerCqfRmfjVdcwdeToY9r23nMvF15UTx5Jsxv1zWvLuz7ovwosLx6fDTzYtd/+om1GUYiIcTpHE5x33nnVJdXQTB05OmNS+u/svrqmNBqpNfYEt01tdzRnZkbEwNMxZeYEMAEwNjbmdE5Sk01Px6nWGPXVRw9FxAqAYn2oaD8AnNu13zlFmyRphEZ9pLAVWA+8sVh/sKv9yoi4BbgQ+IbnEyTYs2c3l67b8Ji2s5YuYcv119WUSAtdZUUhIt5L56Ty0ojYD7yeTjG4NSI2AA8Alxe73wa8ANgHfBv4vapySW3ycJ4y43zM1PYba0pzAiI667Snty2qvPpo3SybLu6xbwJXVJVFktQfh86WWqZXlxLYraThsChILdOrSwla1q2kxnLsI0lSyaIgSSpZFCRJJc8pSKrO5s11J9CALAqSquNUnK1j95EkqWRRkFSdiYnOotaw+0hSdTZt6qztRmoNjxQkSSWLgiSpZFGQJJUsCpKkkieaNTQbr7qGqSNHZ7TvvedeLrxo9HkkDc6ioKGZOnK05+id39l9dQ1pJJ0Ii4K0QDRy6k5nXGsdi4K0QLR+6k41gieaJUkljxQ0ME8oq29r13bWO3bUm0N9syhoYJ5QVt927qw7gQZk95EkqWRRkCSVLAqSpJJFQZJUsihIkkpefSSpOhs31p1AA7IoSKqOU3G2Ti1FISLuB74FPAIcy8yxiDgT+AdgJXA/cHlmfr2OfJK0WNV5TuGizFydmWPF82uBbZm5CthWPJfUZjt2eDdzyzSp++gy4LnF45uBTwJ/WlcYaSHoNXIqjHD01LHiO5+jpbZGXUUhgY9HRAKbM3MCWJ6ZB4vtXwWW93phRIwD4wDnnXfeKLJKrdVr5FRw9FTNrq6i8JzMPBARPwLcHhH3dG/MzCwKxgxFAZkAGBsb8+uHJA1RLecUMvNAsT4EfAC4AHgoIlYAFOtDdWSTpMVs5EUhIn4wIk6ffgz8CrAH2AqsL3ZbD3xw1NkkabGro/toOfCBiJj++e/JzI9GxBeAWyNiA/AAcHkN2SRpURt5UcjMLwM/26P9a8DFo84jSXpUky5JlbTQTE7WnUADsihIqs70dJxqDYuC5tRrPmbnYpYWLouCgN4f/lAUgD/4q8e0ORez+jY+3lk7MF5rWBQEwNSRoz3vfLUALEwjG/5iy5bO2qLQGhYFaRFy+AvNxpnXJEkli4IkqWRRkCSVLAqSpJInmiVVZ82auhNoQBYFSdVxKs7WsShIKvW6f2FkU3eqESwKi8ycdy47dMWi1+v+hY9f/6p653nWSFkUFhnvXNagTupGt868KZDOnNsWFoUFzMHsJA3KorCA9Toq8IhA0ly8T0GSVPJIYQHw5LGkYbEoLACePJY0LHYfSZJKHilIqs7mzXUn0IAsCpJOSF+zt01Px6nWsChIOiHO3rYweU5BUnUmJpyfuWU8Umio2S4zdbwZtcqmTZ213UitYVFoqNkuM/XQXE3Xfa7hI0Xbxquu6fllxi8/zWNRGKFefwCD/vL3OrnnTWpqkseca7jlJoCeH/zT7X75aRaLwgj1+gMY9Je/18k9b1LTQuO8DvVpXFGIiEuAvwEeB9yYmW+sOVKlZrusz2//Wsyc16E+jSoKEfE44G+BXwb2A1+IiK2ZubfeZL3N1h/6lX1f4slP+ckZ7b0+6Ge7rM9v/1pIhvHlZ7a/lV7FYrZCMcjf7KDvUVVxGvXPa1RRAC4A9mXmlwEi4hbgMmDoRWHQD/Re7XvvuZcL/+CvZuz79d1X8zQ/6KVSlV9+er33bN2ys53D6PU3O+h7VHUeZNQ/L7JBMyJFxIuASzLzFcXzlwIXZuaVXfuMA9PXt/0k8KWRB32spcCRmjPMp+kZzXdyzHfymp5x2PmelJnLem1o2pHCvDJzAmjM3TARMZmZY3XnmEvTM5rv5Jjv5DU94yjzNe2O5gPAuV3PzynaJEkj0LSi8AVgVUQ8OSJOBV4CbK05kyQtGo3qPsrMYxFxJfAxOpek3pSZd9ccaz6N6cqaQ9Mzmu/kmO/kNT3jyPI16kSzJKleTes+kiTVyKIgSSpZFAYUEWdGxO0RcV+xPmOW/R6JiLuKZaQny/vNWOz7xIjYHxFvb1K+iHhSROws/v3ujojfb1i+1RHx2SLbroh4cZPyFft9NCKORsSHR5Trkoj4UkTsi4hre2w/LSL+odj+uYhYOYpcA+T7heJ37lhxz9TI9ZHxjyJib/E7ty0injTsDBaFwV0LbMvMVcC24nkv/5uZq4vlN0YXD+g/I8BfAJ8eSapH9ZPvIPDMzFwNXAhcGxFnNSjft4GXZeZTgUuAt0bEkgblA/hr4KWjCNQ1RM3zgfOBdRFx/nG7bQC+nplPAd4C/OUosg2Q77+AlwPvGVWubn1m/HdgLDOfBrwPmDmkwkmyKAzuMuDm4vHNwAvrizKrvjJGxFpgOfDx0cQqzZsvMx/OzO8WT09jtL+r/eS7NzPvKx5PAYeAnneI1pEPIDO3Ad8aUaZyiJrMfBiYHqKmW3fu9wEXR0Q0JV9m3p+Zu4DvjSjT8frJuD0zv108vZPOvVxDZVEY3PLMPFg8/iqdD9VeHh8RkxFxZ0S8cDTRSvNmjIhTgOuAPx5lsEJf/4YRcW5E7AIeBP6y+PBtTL5pEXEBcCrwn1UHKwyUb0TOpvP/adr+oq3nPpl5DPgG8MMjSddfvroNmnED8C/DDtGo+xSaIiLuAH60x6bXdT/JzIyI2a7pfVJmHoiIHwM+ERG7M3NoHxpDyPhK4LbM3F/Fl7Vh/Btm5oPA04puo3+OiPdl5kNNyVe8zwrg74H1mTm0b5jDyqeFKSJ+FxgDfnHY721R6CEznzfbtoh4KCJWZObB4gPh0CzvcaBYfzkiPgk8nSF+kxxCxmcCPx8RrwSeAJwaEf+dmXOdfxhlvu73moqIPcDP0+l2aES+iHginRknX5eZdw4j1zDzjVg/Q9RM77M/Ir4P+CHga6OJ14ohdPrKGBHPo/Pl4Be7uliHxu6jwW0F1heP1wMfPH6HiDgjIk4rHi8Fnk0Fw3/PYd6Mmfk7mXleZq6k04X0rmEVhGHki4hzIuIHisdnAM9hdCPi9pPvVOADdP7dhlKoBjBvvhr0M0RNd+4XAZ/I0d0924YhdObNGBFPBzYDv5GZ1XwZyEyXARY6faDbgPuAO4Azi/YxOjPFATwL2A38R7He0LSMx+3/cuDtTcpHZ6KlXcW/4S5gvGH5fhf4P+CurmV1U/IVz/8VOAz8L53+6V+tONcLgHvpHBG/rmj7czofYACPB/4R2Ad8HvixUf0/7TPfM4p/p/+hcwRz9yjz9ZnxDuChrt+5rcPO4DAXkqSS3UeSpJJFQZJUsihIkkoWBUlSyaIgSSpZFCRJJYuCJKn0/4CBxCKsZZPLAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Assuming plm_embs_train_silhouette_values is your data\n",
    "sns.histplot(plm_embs_train_silhouette_values)\n",
    "\n",
    "# Add a red vertical line at x = 0\n",
    "plt.axvline(x=0, color='red', linestyle='--', linewidth=2)\n",
    "\n",
    "# Display the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "semi1_cv_{}.csv 1 64 None v0 False\n",
      "train 3463\n"
     ]
    }
   ],
   "source": [
    "dataset_col = GittablesColwiseDataset\n",
    "train_dataset_col = dataset_col(cv=cv,\n",
    "                            split=\"train\",\n",
    "                            tokenizer=tokenizer,\n",
    "                            max_length=max_length,\n",
    "                            train_ratio=1.0,\n",
    "                            device=device,\n",
    "                            small_tag=\"semi1\",\n",
    "                            base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "                            max_num_col=1,\n",
    "                            sampling_method=None,\n",
    "                            random_seed=0,\n",
    "                            context_encoding_type=args.context_encoding_type,\n",
    "                            adaptive_max_length=False                                       \n",
    "                            )\n",
    "# dataset_cls(cv=cv,\n",
    "#                                             split=\"train\",\n",
    "#                                             tokenizer=tokenizer,\n",
    "#                                             max_length=max_length,\n",
    "#                                             gt_only='all' not in task,\n",
    "#                                             device=device,\n",
    "#                                             base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "#                                             small_tag=args.small_tag,\n",
    "#                                             max_num_col=args.max_num_col,\n",
    "#                                             sampling_method=args.sampling_method,\n",
    "#                                             random_seed=args.random_seed,\n",
    "#                                             context_encoding_type=args.context_encoding_type,\n",
    "#                                             adaptive_max_length=args.adaptive_max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader_col = DataLoader(train_dataset_col,\n",
    "                                batch_size=batch_size,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "model = model.to(device)\n",
    "plm_embs_train_col = []\n",
    "labels_train_col = []\n",
    "for batch_idx, batch in enumerate(train_dataloader_col):\n",
    "    cls_indexes = torch.nonzero(\n",
    "                    batch[\"data\"].T == tokenizer.cls_token_id)\n",
    "    embs = model.bert(batch[\"data\"].T)\n",
    "    embs = extract_cls_tokens(embs[0], cls_indexes)\n",
    "    plm_embs_train_col.append(embs.detach().cpu())\n",
    "    labels_train_col.append(batch[\"label\"].cpu())\n",
    "plm_embs_train_col = torch.cat(plm_embs_train_col, dim=0)\n",
    "labels_train_col = torch.cat(labels_train_col, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3463, 768])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plm_embs_train_col.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3463, 768])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plm_embs_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "plm_embs_train_silhouette_values_col = silhouette_samples(plm_embs_train_col, labels_train_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.22704358"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(plm_embs_train_silhouette_values_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "st_model = SentenceTransformer(\"all-mpnet-base-v2\", cache_folder=\"/data/zhihao/TU/Watchog/sentence_transformers_cache\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "base_dirpath = \"/data/zhihao/TU/GitTables/semtab_gittables/2022/\"\n",
    "seperator = \"semi1\"\n",
    "basename = seperator+\"_cv_{}.csv\"\n",
    "df_list = []\n",
    "filepath = os.path.join(base_dirpath, basename.format(cv))\n",
    "df_test = pd.read_csv(filepath)\n",
    "\n",
    "df_group_test = df_test.groupby(\"table_id\")\n",
    "\n",
    "\n",
    "\n",
    "df_list = []\n",
    "for i in range(5):\n",
    "    if i == cv:\n",
    "        continue\n",
    "    filepath = os.path.join(base_dirpath, basename.format(i))\n",
    "    df_list.append(pd.read_csv(filepath))\n",
    "df_train = pd.concat(df_list, axis=0)\n",
    "df_group_train = df_train.groupby(\"table_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_cluster = []\n",
    "embeddings_cluster = []\n",
    "embeddings_cluster_context = []\n",
    "for i, (index, group_df) in enumerate(df_group_train):\n",
    "    labeled_columns = group_df[group_df['class_id'] > -1]\n",
    "    labels_cluster += labeled_columns[\"class_id\"].values.tolist()\n",
    "    # if len(labeled_columns) > 1:\n",
    "    #     embeddings_cluster.append(ft_model.get_sentence_vector(\" \".join(labeled_columns[\"data\"].values)))\n",
    "    #     embeddings_cluster_context.append(ft_model.get_sentence_vector(\" \".join(group_df[\"data\"].values)))\n",
    "    #     break\n",
    "    for text in labeled_columns[\"data\"].values:\n",
    "        embeddings_cluster.append(st_model.encode(text))\n",
    "        embeddings_cluster_context.append(st_model.encode(st_model.tokenizer.sep_token.join(group_df[\"data\"].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_cluster = np.stack(embeddings_cluster, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_cluster_context = np.stack(embeddings_cluster_context, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_cluster = np.array(labels_cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_cluster_silhouette_values = silhouette_samples(embeddings_cluster, labels_cluster)\n",
    "embeddings_cluster_context_silhouette_values = silhouette_samples(embeddings_cluster_context, labels_cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.08592572"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(embeddings_cluster_silhouette_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.15294842"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(embeddings_cluster_context_silhouette_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoMAAAGMCAYAAACh0KjGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAChs0lEQVR4nOzdd3gU1dfA8e9sSe8JEDoIBqJAgNCVGkQ6CAiigApSVER9BUEFERVRVKogUvyhiBQxgiKgIkqRHpTee01vpG6Z94+YlSVtk2wq5/M8PpLZmTt3J7OTs7ecq6iqqiKEEEIIIe5JmpKugBBCCCGEKDkSDAohhBBC3MMkGBRCCCGEuIdJMCiEEEIIcQ+TYFAIIYQQ4h4mwaAQhZDdZHyZoF94cg2FKDz5HJUtRfH7srXMMh0Mbt26lREjRtC6dWuaNGlC3759+eabbzAYDCVdtUKrV68ey5YtK+lq2GTDhg20a9eORo0asWTJkmz3SU9PZ/HixfTs2ZNGjRrRrFkzhg4dyi+//GK139ChQxk9ejQA+/bto169ehw9ehSASZMm0bNnz6J9M/mwdu1a5syZY/n51q1bjBgxgtjY2EKXffnyZV5//XUefvhhGjRoQLt27Zg0aRJXr14tdNlFZf78+TRp0qTQ5Zw9e5ann37aDjUqm+78DNwr7HXvFLTsu58tZen5m+nu93D386m0PT+vXbtGvXr12LJlS0lXpVT47LPP+Pbbb+1a5sGDBxk3bpxN++rseuZiNG3aNFavXk3fvn0ZPHgwLi4u7N+/n5kzZ7Jv3z7mzJmDVqst6WoW2Jo1a6hSpUpJV8MmH3zwAfXr12fs2LFUr149230mTpzInj17GDNmDPXq1SMlJYXffvuNcePGMXXqVJ588kkApk6dikZTNr6jLFq0iA4dOlh+3r17N7t27Sp0ubdu3WLQoEHUrVuXN998Ez8/P27cuMGyZct4/PHHCQ0NLTP3RkFs2bLF8gVAiJJQlp6/mV544QWSk5MtP9/9fBKl2/z583n99dftWua6deu4ePGiTfuWyWBw/fr1fPvtt7z77rsMGjTIsr1NmzYEBATw6quv8tNPP9G3b9+Sq2QhNW7cuKSrYLO4uDjatm1L8+bNs3392rVrbNq0idmzZ9O9e3fL9k6dOpGUlMT8+fMtwWDdunWLpc6l2bp16zCbzSxbtgxHR0fL9rZt29K5c2e++eYbuz80hBD/KUvP30w1atQo6SqIMqxsNMHcZdmyZdSrV88qEMzUvXt3hg8fjre3t2XbtWvXePnlly3dyc8//zyXLl2yvD5//nz69evH+vXreeSRR2jUqBHPPPMMERERrF69mg4dOhAcHMz48eNJSUkB/uvC3LVrF4899hiNGjWiX79+7Nmzx6o+R44cYeTIkTRr1owGDRrw6KOPsnr1asvroaGhtGzZkqVLl9KyZUvat29PcnKyVTeFyWRi5syZdOjQgQYNGtC9e3dWrVpldZ5Tp07x3HPP0aJFC1q0aMGECROIioqyvD5p0iTGjRvHV199RceOHWnUqBFDhw7l/PnzuV7rmJgYJk+eTLt27QgKCmLYsGGWVpvMawDw8ccfW/6dXRkAZrM5y2vPPfcczz33nKVr35Yusq+//jrX9/Dbb7/Rv39/GjduTPv27ZkzZw5Go9HyeqdOnXj33Xetjpk+fTqdOnXKcp4uXbrQoEEDevTowaZNm6zKuH79OitXrqRevXqEhobyxhtvANC6dWvmz58PgNFoZO7cuXTo0IGGDRtme4/cLTo6Otvr5evry1tvvUWLFi0s21JTU/noo49o164dTZo04YknnuDgwYOW15OSkvjoo4/o1KkTjRo1YsCAAVatl5m/w9WrV/PQQw/RsmVLS1f0xo0b6dWrFw0aNKBz586sWLEi13pnWr9+PR07diQoKIjRo0dz+fJlq9ePHTvG008/TVBQEK1ateK9996zfK7mz5/PZ599ZvkM/O9//yMwMJDQ0FDL8Vu3bqVevXqsW7fOsu2XX37hwQcfJDExEYC//vqLxx9/nEaNGtGuXTvmzp2LyWSyqkde7y/z9/rqq6/SpEkTWrZsyfTp063upezs27ePp556iiZNmtCuXTtmzJhBWlqa5fUDBw7w1FNP0bRpU9q0acO7775LUlJStmXl1JXWp08fJk2aZDlfvXr12Lt3r+U99+zZk4MHD3Lw4EH69u1LUFAQTz75pNXvoqDvL7ffH2R8hqdPn87HH39Mq1ataNq0KVOnTiU5OZn333+fZs2a8fDDD/PFF19kKbsw906mZcuW0bFjRxo3bsyECRNITU21et1oNPLJJ5/w0EMP0bRpU2bMmJHl3rjz+Zv592Hjxo08+uijNGzYkP79+3Po0CGrYzZv3mwZBjNgwADLfbpv3z4AkpOTeeutt3j44Ydp1KgRjz32GL/++mu219hsNtOyZUvLcwTg5MmT1KtXj7lz51pdj3r16nH27FmrbuC7n093yuv5ebcLFy4wbtw4WrVqRYMGDejUqRMLFiywjEXLvP8OHjzIE088QcOGDQkJCeG7776zKufw4cMMHjyYoKAgevXqxYkTJ3I97533OGQ0OtSvX9/qi3BMTAz169fnzz//zHK8qqqsXbuWXr160ahRI7p06cLy5ctzPWd2TCYTixYtonPnzgQFBdGnTx+2bt1qed1gMLB48WLLvdGrVy9++ukny+uZn+Ft27YxYsQIgoKCaNu2LZ9//rlln8zf0cyZM63+DuX2HDt8+DCBgYF88sknlv0PHTpEYGAgS5cuZdKkSfzwww+cPXvW6j7MSZkLBiMiIjhz5gzt27fPcZ+JEydaXr916xaPP/44ly9f5p133mHGjBlcu3aNJ598kvDwcMsxFy9eZMmSJbz++uu8//77HD58mKFDh/L999/zzjvv8NJLL7Fx40a+/vprq3O99tprhISEMH/+fHx8fBg5ciRnzpwB4MaNGwwbNgwXFxfmzp3LggULqF27NlOnTuX06dOWMhITE/npp5/45JNPeOONN3BxcbE6xxdffMH333/PK6+8wrJly2jbti3vvPMOO3fuBDIeEoMGDcJgMPDhhx/y5ptvcvDgQYYMGWLVbbB7927Wr1/PW2+9xccff8zly5etPmx3S0pKYvDgwezevZvXXnuN2bNno6oqQ4YM4fTp0zz44IOsWbMGyPgDkPnvu9WvX59KlSoxdepUPvroI/bv32/549ioUSNGjBiBXq/PsR53On/+vOU9fPjhh1y8eJEJEyZYXl+zZg1jx46lUaNGfPbZZwwZMoQvv/wy1/eZnc8++4yPPvqI7t27s2jRItq0acP//d//sXnzZsvrFSpU4NFHH2XNmjV06NCB559/HoClS5fy+OOPAzBlyhT+97//MWzYMBYsWMB9993HyJEjs/whuVO7du2Ij49n0KBBrFy5kgsXLlheGzBggFXXzyuvvMLatWt57rnnWLBgAb6+vowcOZLLly9jNpt57rnnCA0NZdSoUcyfP58qVaowatQoy72TacmSJbz33nu88cYbVK9enR9++IHXXnuN5s2bs2jRIvr27cuMGTNYunRprtctJSWFTz75hHHjxjFz5kwuXbrEM888Y7kPz507x5AhQ1AUhTlz5jB+/Hg2bdrEK6+8AsDjjz/OgAEDcHJyYs2aNfTp04eGDRuyd+9eyzkyH2phYWGWbbt27aJJkya4u7uzZ88eRo4cSbVq1fjss88YMWIE//vf/3j//fct+9v6/j744AN8fHxYuHAhTz31FF9//TVr167N8f0fOXKE4cOH4+7uzuzZs3nppZdYt24d06dPB2D79u0MGzaMChUqWF7fuHEjo0ePzvbLUn5MmDCBAQMG8Nlnn2E2m3nllVd48803eeaZZ/j00085f/58li9B+X1/ef3+Mn3//fecP3+eWbNmMXz4cFavXs1jjz1GYmIi8+bNo23btsyaNYu///7bckxh7x3ICAQ//fRTHnvsMebNm4fBYOCrr77K8p5XrFjByJEjmTVrFqdOnbJ8rnNy6dIl5s2bx9ixY5k/fz5paWm8/PLLlsB5x44dvPrqqzRs2JAFCxbQpk0bXnvtNasypk+fzt69e3nrrbdYvHgxderU4eWXX842GNNoNLRp08am+75q1arcf//9Vsff/XzKlNfz825JSUkMGzaMuLg4PvroI7744gtatmzJvHnz+OOPP6z2ffXVV3n00UdZvHgxDzzwAJMnT+bcuXNARkD0zDPP4OjoyLx58+jfv3+ez+S2bdtaBTAHDx5EVVWr9//XX3/h4OBAq1atshw/a9Ys3nnnHTp16sTChQvp2rUrH330EbNnz871vHebMWMGn332Gf369WPRokUEBQUxbtw4y5fuiRMnsnDhQgYOHMjnn39OkyZNGD9+fJZg+I033iAoKIhFixbRsWNH5syZw/bt2wGs/o5+9tlnAHk+x4KCghg6dCjLly/nwoULpKWl8dZbb9G4cWOGDx/OCy+8QPv27alevTpr1qzhwQcfzP2NqmXM4cOH1YCAAPXbb7+1af8ZM2aojRs3VqOjoy3boqOj1SZNmqgzZsxQVVVV582bpwYEBKj//POPZZ9XX31VDQgIUK9du2bZNnjwYPX5559XVVVV9+7dqwYEBKgff/yx5fW0tDS1bdu26qRJk1RVVdU///xTffrpp9X09HTLPnFxcWpAQIC6YsUKVVVV9fvvv1cDAgLUX3/91areAQEB6tKlS1VVVdXhw4erw4cPt3p91qxZ6oEDB1RVVdWxY8eqHTp0UNPS0iyvnz17Vq1fv7769ddfq6qqqhMnTlTr16+vhoeHW/b56quv1ICAADUmJibba/f111+r9evXV8+ePWv1Hjt06KCOHTs227rm5OTJk2qPHj3UgIAANSAgQG3QoIH6zDPPqD///LPVfkOGDFFHjRqlqup/1/jIkSNW7+HWrVuW/b/88ks1ICBATUxMVI1Go9qqVSv11VdftSpz1apVakBAgHry5ElVVVW1Y8eO6rRp06z2ef/999WOHTuqqqqq8fHxasOGDdVZs2ZZ7fPGG2+oISEhlp/vLifzd5l5r507d04NCAhQ165da1XOsGHD1KFDh+Z6vZYvX642btzYcr0eeugh9c0331TPnTtndU0DAgLUH374wbItLS1N7d69uxoaGqr+/vvvakBAgLpjxw6rsgcOHKg+9thjqqr+d43/97//WV43mUzqww8/rL722mtWx3322WdqkyZN1KSkpGzrnPk52r17t2XbmTNn1Hr16lmuwauvvqqGhIRY3asHDhxQAwIC1P3791vKady4seX1+fPnq+3atbP83Lt3b7Vv377qI488YtnWsWNH9YsvvrC8vyeeeMKqbj/88INav3599erVqza/v4CAAPW5556z2qdv377q6NGjs33/qqqqL7zwgtqlSxfVaDRatq1YsULt16+fajQa1ccee0wdOHCg1TE7duxQAwIC1N9//11VVevPwNWrV9WAgAB18+bNVsf07t1bnThxoqqq//0OM9+/qv53z995b8yePVsNDg62/FyQ92fL72/IkCFq8+bN1ZSUFMs+Dz30kNq5c2fVZDKpqppxnwYGBlruO3vcOyaTSW3ZsqX69ttvW143m81qr169LPdTbGysGhgYaHWtUlNT1datW6s9evSwujaZz7TMuh0+fNjy+tatW9WAgAD16NGjqqqq6qBBg9QhQ4ZYXat3331XDQgIUPfu3auqqqo++uij6pQpUyyvp6WlqTNmzFBPnTqV7bUODQ1VH3zwQct1fP7559W+ffuqQUFBlr8pQ4YMsbzfiRMnWr2Hu59PeT0/s3P06FF18ODBVn8/TSaT2qxZM/XDDz9UVfW/+2/x4sWWfeLj49V69eqpy5YtU1VVVT/44AO1RYsWanJycpZz331vZ9q3b58aEBCgXr58WVXVjGd037591YCAAMt7mDhxojpixIgsx8bExKgPPvig+sknn1ht/+STT9QHH3zQ6v3kJvN++eyzz6y2DxkyRP3888/VU6dOqQEBAeqqVausXn/11VfVVq1aqUaj0fIZnjp1quV1k8mktmjRQn333Xct2+7+O5rXc0xVVTU5OVkNCQlRhw8frs6aNUtt3LixeunSJcv+d98TuSlzLYOZk0Js/RZ94MABWrZsiY+Pj2Wbj48PrVu3Zv/+/ZZtiqLQoEEDy8++vr74+PhQtWpVyzYvLy9LN1SmHj16WP7t4OBA27ZtLd8Y2rdvz/LlyzGbzZw6dYotW7ZYukbS09Otyqldu3aO76FZs2bs2rWLoUOH8tVXX3H16lVeffVVmjVrZnmPISEhODg4WI6pW7cu9erV48CBA5ZtVapUoWLFipaf/f39AbJ0s2Q6cOAAdevWtRrH5+DgwCOPPGJ17WxRv359fvrpJ1atWsWYMWN44IEH2LdvH6+++mqWb9C5qVKlCpUqVbL8nPn7SUhI4MKFC8TExNC1a1erYzJ/R3d2n+bmn3/+IS0tjQ4dOmA0Gi3/tWvXjqtXr9o8ozfzGrVr186qnPbt23Po0KEs98Cdnn76aXbu3Mns2bPp168fDg4OrFu3jj59+li+kWe2Lt7ZreDg4MDPP//MY489xoEDB3B1daVt27ZWZXfv3p0TJ05w+/Zty7Y777+LFy8SERGR7ftPSkriyJEjOdbb3d2d1q1bW36+//77qV69uuXb/L59+2jTpg0ajcZSbuPGjXFzc8ux+7xt27bcunWLS5cuERcXx+nTpxkxYgSXL18mKiqKixcvcv36ddq3b09KSgpHjhyhY8eOWepuNpvZt29fvt5fUFCQVV0qVapk1dp+t7///pt27dpZTV4bMmQI33//PampqZw4cSLL/dm2bVs8PT2tPqsF0ahRI8u//fz8AKyeadk9v/L7/mz9/QUEBODk5GT52dfXlwceeMAyOczBwQEXFxer+hT23rl48SKxsbG0a9fOUoaiKHTp0sXy8+HDhzGZTFb7ODo65trTBKDT6ayu5Z3PzrS0NA4fPkxISIjVMXf/nps1a8batWsZM2YMa9asITY2lkmTJuU4vObhhx/GaDQSFhaG2Wzm4MGDjBgxgpSUFE6cOEFycjJ///13nnW/U27Pz+w0aNCAb7/9Fnd3d86dO8fWrVv57LPPMBqNWZ5fd46z9PDwwMXFxXIvHTp0iObNm+Ps7GzZ587fS3aaNGmCm5ubpXV0//79PPHEE7i6ulruib/++ivb93/48GEMBkO2fwsMBgOHDx/O9dx3lmMymbIMIVqxYgVjxoyx/E25+zzdu3cnJibGqtX3zuuj0WioWLFijp81W55jAM7Ozrz33nvs2rWLxYsXM378eGrWrGnTe7tbmZtAUrlyZQBu3ryZ4z4RERH4+fmh0WhISEggMDAwyz6+vr6WJmzIuKh3zz6+88bNSYUKFax+9vHxIT4+HsgYa/Dhhx+yZs0aDAYDNWrUsARw6l25f+4MVu82atQonJ2dWbduHR988AEffPABwcHBfPjhh9SoUYOEhAR8fX2zfY93/sG/+/1kPphzCqwTEhIsf1Tu5Ofnl+MYp9woikLTpk1p2rQpkDE27r333mPjxo08/vjj2Tb13y2395B53e++Fu7u7jg4OFhdi9zExcUB8MQTT2T7emRkZI6zprMr584/PHeKjY21ejDfzc3Nje7du1sm3Rw8eJDXXnuNd999l44dOxIfH49er8fDwyPb43P7/amqavU7vPP+y6z3a6+9lm2gHhkZmWOds7sPfXx8LH/04+LiWLNmTbZDCnIqt2HDhnh5ebFv3z58fX2pUKECXbt2ZcqUKRw8eJCoqCj8/f2pV68e4eHhmM1mPv30Uz799NNsz5Gf95fd/Xb3Z/dO8fHx2V4DyBgOoqpqjtfI1vszJ66urlm23RmQZSe/78/W3192dcnreVrYeyfz83/neHHA6jOQGfTktk92HBwcrLIc3P3cMZvNWZ7hd7+fyZMnU7FiRTZs2MAff/yBRqOhffv2lq76u1WoUIHAwED27duHj48PycnJdOrUiVq1anHw4EFiYmJQFMWm52am/P4NAPj8889ZtmwZiYmJVK1alSZNmqDT6bLcJ3ffa3feSwkJCdSvXz/L+8uNXq+ndevW7Nu3jy5dunD69GlatGhBkyZNOHjwIHXr1iUiIiLbYDDzXrj795r5O7H1s5ZZTk5/n+Pj49HpdHh5eVltzzzv7du3LcO+crs+d0tISMjzOZapefPmVKlShVu3bmX54p8fZS4Y9PHx4YEHHmDXrl2MHz8+232effZZ/Pz8+Oqrr/D09LSaSJEpKioqyy+wIOLi4qxuuOjoaMuN8/nnn7N27Vo++ugj2rdvj4uLCykpKVYD322h1Wp55plneOaZZ7hx4wZbt25l/vz5vPvuuyxduhRPT0/LpIM7RUVFUadOnQK/N09PT6vxapkiIyPzde0+/PBD9u/fbzUJADI+mO+99x6bN2/m/Pnz+XqoZSezTndfi4SEBNLT063qfPfD785vaO7u7gAsWLAg22Att1bcO7m7u6MoCqtXr842zdHdf5Ag4wtE586dGTJkCCNGjLB6rVmzZgwfPpwPPviAlJQU3N3dMRgMJCYmWuoMGa1THh4eOd77mQ+SnH6HmWW9/fbbVq1NmapVq5bje86uhSEqKoqAgAAgI8ANCQlh8ODBWfbL7npAxkPz4YcfZt++fVSoUIFmzZqh0+ksfxSuX79uCbgzg5Dnn38+S0sNQMWKFS0P+IK8v7y4ublZJkxliouL4/jx4wQFBaEoSo6f1ex+H4qiALnfr8WpIL8/WxX23slsqcru+mfKvMYxMTFWn+0798kvX19f9Hp9lvPe/bOTkxPjxo1j3LhxXLhwgV9++YWFCxcyd+5cpk2blm3ZmePm/Pz8eOCBB3BxcaFFixYcPHiQmzdv0rx58yxjzO1p/fr1zJ07l6lTp9KzZ0/Ls+HOFlxbeHl5ZbnvbcnH2rZtW+bPn09YWBh+fn7Url2b5s2bs2nTJmrUqEGtWrWynUWd+XuOioqy+j1nPg9t/fuV+X7v/uJ+8uRJVFXF09MTo9FIXFycVZn5Pc/dbHmOZVqyZAkxMTHUqFGDKVOmZBkja6sy100MGV1oJ0+ezDJAEzISIJ87d45evXoBEBwczL59+6w+mDExMezZs8fSQlUYdw6iTU9PZ8eOHbRs2RLI6G5s0KAB3bp1s3xgMwfu5/bt+27Dhw9nxowZQEYz/7Bhw+jcubOldTQ4OJjff//dqtn+/PnznDlzplDvMTg4mHPnzlk1daenp7N169Z8lVuzZk2OHz+ebQ6+zFndmQ/8wqhduzbe3t5ZZl5mzgLOrLObmxsRERGW181ms9VA9qCgIPR6PdHR0TRs2NDy39mzZ1mwYIFlv7vzId79c3BwMKqqcvv2baty9uzZw/Lly9Hpsn4X02q1VKhQgbVr12bb+nr58mWqVauGs7OzJZHu3ffgK6+8woYNGwgODiYpKSnLZJHNmzfz4IMPWqWtudN9992Hl5cX4eHhVvWOi4tj7ty5uX6rjomJ4fjx45afjx8/zrVr1ywzoIODg7lw4QINGjSwlFu5cmU+/fRTzp49m+11hIw/Cvv37+fgwYOW1vVmzZqxd+9e9u/fb2kdcHNzo379+ly9etWq7nq9nlmzZnHr1q1Cvb+8NGnShB07dlgFb5s2bbLMkA8MDMxyf+7cuZPExMRsP1Nubm4AVvdreHg4165dK3AdC8OW319BFfbeqV27NhUrVswyQzdzkD5k/H4cHBys9jEajfz1118FrrdWq6Vx48Zs27bNavvvv/9u+bfJZKJnz56W2az33Xcfzz//PI0bN861l6tdu3YcO3aMHTt2WN33YWFh/PXXXzn2OkD2n6P8+vvvv/H392fw4MGWwOj48ePExMTk629Yy5Yt2bdvn1XAv2PHjjyPa9euHZGRkXz33XcEBwcDGe//zJkzbNq0Kcf3n/mZz+5vgU6ny/ZLYHYaNWqETqfLMlnm7bffZtmyZZY6ZXceX19fatWqZdN5wPr3ZctzDDL+zi9cuJAxY8bw/vvvs3fvXqu4KD/3QJlrGYSMKed//vknb7/9NkeOHCEkJARFUdi1axerVq2iW7du9O/fH4BnnnmGH374geHDh1tme37++ec4ODjYZZWDhQsXotfrqV27Nl9//TXJyck899xzQMYNuWTJEr755hsCAgI4evQoCxYsQFGULOkOchMcHMznn39OhQoVaNiwIefPn2fLli2W+o8ZM4YnnniCkSNH8swzz5CYmMicOXOoWrVqoXIt9uvXj6+++oqRI0fyyiuv4O7uzvLly4mKimLMmDH5KueHH37gxRdf5Mknn6R169Y4OTlx/Phxli5dSkhISI45CvNDq9UyduxY3nvvPTw9PQkJCeH06dPMnz+frl27WgLOdu3a8b///Y8VK1ZQt25dVq9eTXR0tOXbmI+PD0OHDuXDDz8kPj6eRo0acerUKWbPnk1ISIjlD7SHhwfHjx9n//79NG/e3NJd+9tvv/HQQw8RGBjIo48+yoQJExg7dix16tRh//79fP755zz33HM5flDfeOMNnnnmGQYMGMDQoUOpW7cuycnJbN26le+//96SWuLBBx+kY8eOvPfee9y+fZuaNWuyevVqUlJSGDRoEP7+/gQFBTFhwgReffVVKleuTGhoKIcPH7ZKa3A3nU7HSy+9xIcffghktAJcu3aNTz/9lFq1auXacubg4MD//d//MX78eAwGA5988gn169fn0UcfBTIS4z7xxBO8/PLL9O/fn/T0dBYuXMjNmzd54IEHLNc1JSWFrVu30qhRIypWrEjbtm2ZNGkSkZGRli9GzZs3Z+7cuZbupEzjxo3jxRdfxM3NjUceeYTY2FjmzJmDRqMhICCgUO8vL2PGjOGpp55i3LhxDBw4kFu3bjFnzhyGDBmCm5sbL730Ei+88AKvvPIK/fr14+bNm8yaNcuShuZunp6eBAUF8eWXX1K5cmW0Wi2fffZZjkMDipotv7+CKuy9oygK48aNY8qUKfj6+vLQQw+xefNmjh8/bmmZd3NzY8SIESxZsgQnJycCAwNZtWoVUVFRhcrT9+KLL/Lss88yefJkunbtyj///MM333wDZPxB1mq1NGrUiAULFuDo6Mh9993H4cOHCQsLy7FVEDLGmTk7O7Nz506eeuopIOO+j4+PJz4+Ptfxgnc/nwqiYcOGrF69ms8++4wWLVpw/vz5Av0Ne/rpp1mzZg0jR45kzJgx3Lp1yzJrNjeVK1embt26/PHHH0yZMgXICNAcHBw4fPiw1eoaJ06cwMHBgbp161qe4cuWLUOr1dK8eXMOHDjAsmXLeOaZZ/D09AQyso3cunWLBx54wGrMfSZfX1+eeOIJPv/8c8u40c2bN3Py5Enefvtty/354YcfkpSURL169fj999/5+eefefvtt/MVjHl4eBAWFkazZs0sM5Zze46ZzWYmT55M1apVGTFiBA4ODvTp04eZM2fSvn17KlasiIeHB7du3eKvv/6iQYMGlvednTIZDCqKwqxZs1i7di2hoaH8+uuvGAwGatWqxeTJkxkwYICle6Vy5cqsXLmSjz/+mEmTJqHVamnZsiWzZ8+2DAIujNdff51vvvmGa9eu0ahRI1auXGkZTzZq1CgiIyP57LPPSEtLo1atWrz99tv89NNPVi1ReRkzZgxms5lVq1YxZ84cKlSowNNPP83YsWOBjEG+X331FbNmzeLll1/G2dmZ9u3bM2HCBEvgUhBubm6sXLmSjz76iHfffReTyUTjxo1ZuXJlvh78jo6OfPXVVyxbtozffvuN1atXYzKZqFmzpqX7216GDBmCk5MTX375Jd999x0VK1bk2Wef5YUXXrDsM2bMGCIjI5k9ezY6nY7evXszevRoy8MbMtJ0+Pj4sHbtWubNm0fFihWtrjnA6NGjmTp1KiNHjuSXX36hdevWPPzww7z33nsMHDiQt99+m08++YS5c+eyePFioqOjqVq1Kq+99lqWLuA7NWnShHXr1vHFF1/wxRdfEB0djbOzM40aNWL58uWWlmeA2bNn8+mnn7JgwQKSk5Np0KABy5cvtwwMX7p0KZ988gmzZ88mJSWFwMBAFi9enGuLwp3Xcfny5Xz55Zd4eXnRtWtXXn31VctnKztVq1blmWeeYdq0aSQlJdG+fXumTJliedBm3qtz5sxh3LhxODo60rRpU2bOnGnphunRowfr16/nlVde4eWXX2bkyJGWCQjXr1+3pNEICgqyHH/nGLWQkBAWLlzIggULCA0Nxc3NjTZt2jB+/HjLmKmCvr+8NG7cmGXLljF79mxefPFF/Pz8GDp0qOXLU2aOtgULFvDCCy/g5eVFz549efXVV3NcMWnGjBm88847jB8/ngoVKjBq1Ch2795d4DoWhi2/v4Kyx72TmdJp8eLFrFy5kjZt2jBmzBirZTJffvllnJycWLlyJQkJCXTp0oWBAwdapXHJr9atWzNz5kwWLFjA+vXreeCBB3jttdeYMWOGpVdo8uTJuLi4sGjRIsuzYOLEiZY6Z0en09GmTRt+++03SytUlSpVqFq1KlqtNtchK3c/nwqiX79+XLp0idWrV7N06VJL4HH+/HmrFC958fX15ZtvvmH69Om88sor+Pv7M23aNF588cU8j23Xrh3nzp2ztIw6ODgQFBTE0aNHrXKujh07lqpVq1ryhU6YMAFvb2/WrFljqfvEiRMZNmyY5ZjvvvuOzz77jN9//z3HL4Fvvvkm3t7erFy5ktjYWO6//36WLFlCw4YNASzP+OXLlxMXF8d9993Hxx9/TO/evW2+Ppn1nzNnDgcPHmT37t15PsdWrFjBoUOHWL58ueUzMnHiRP744w/eeecdFi5cyKBBg/jjjz8YPXo0M2fOtFr04W6Kmp+2XmGxb98+hg0bxrp16yw3hRBCiHvP1q1bqVGjhtVwlzVr1vDOO++wb9++EmvJFcJWZbJlUAghhCgt/vjjD3bt2sVrr71G5cqVOX/+PLNnz6Z3794SCIoyQYJBIYQQohDefPNNSxqQ6OhoKlasyBNPPGFTN6gQpYF0EwshhBBC3MPKZGoZIYQQQghhH/dcMKiqKmlpafnKkSSEEEIIUV7dc8Fgeno6x44dy3Zd2DsTnorCkWtpH3a9joqS8V8xKMZT2WTx4sW88847JV2NckE+2/Yj19I+5DoW3j0XDOYmP0k0Re7kWtqHXEf7GD16dK7JfYXt5J60H7mW9iHXsfAkGBRCCCGEuIdJMCiEEEIIcQ+TYFAIIYQQ4h5WrMGgqqpMmjSJZcuWAWAymXj//ffp2rUrjzzyCKtWrbLse+nSJZ588km6d+/OgAEDOH/+vOW1devW0a1bN7p06cLUqVMxGAzF+TaEEEIIIcqNYluB5Pz580ybNo3Dhw9bFppfvXo1ly9fZuPGjSQlJTFo0CAefPBBGjVqxPjx43n66afp1asX27dvZ9y4cWzcuJGzZ88yf/58fvjhB7y8vBg/fjzLly9n5MiRxfVWhBBCCLsxm81ERUURFxeHyWQq6eqUOTqdjpMnT5Z0NQpNq9Xi5eWFn58fGk3xdtwWWzC4cuVK+vXrR5UqVSzbtm7dysCBA9HpdHh6etKjRw9+/PFHKlWqxIULF+jRowcA7du3Z9q0aZw4cYIdO3bQqVMnfHx8ABg0aBDvv/++BINCCCHKpGvXrqEoCrVq1UKv16OUprxMZUBSUhKurq4lXY1CUVUVg8FAeHg4165do0aNGsV6/mILPd9++2369u1rte3mzZtUrlzZ8rO/vz+3bt3i5s2bVKxY0SoyrlSpkuW1u48JDw8v8voLIcouVVU5ePBgSVdDiGwlJSVRtWpVHBwcJBC8RymKgoODA1WrViUpKanYz19sLYPZyW4VEI1Gg9lsznZ/rVab4zH5dezYsWy3h4WF5bsskT25lvZhr+sYbOfybDlbabsHSlt9yiq5jvYTFhaGTqcjJSWlpKtSppVEAFVU0tPTs3zGgoODc9jbPko0GKxcuTKRkZGWn8PDw/H396dKlSpERUWhqqrlW1Lma5UrVyYiIiLLMfnVoEEDHB0drbaFhYUV+QW/V8i1tI+iuI7F+XspTfeA3JP2IdfRfjKv5cmTJ0tVN+e1a9cICQlh0KBBvPvuu5btJ0+epG/fvsyYMYN+/foVS12OHz/Opk2bmDBhQpbXfvrpJz7//HPS0tIYPnw4Tz31VJZ9Tp48yeTJk7l9+zbNmjVj2rRp6HQ6jhw5wrRp00hPT6dKlSq8//77VKhQgeXLl1OzZk06duxYHG8vWw4ODgQFBRXrOUs0tUxISAjff/89RqORhIQEfv75Zzp37oy/vz81atRg06ZNAOzcuRONRkNAQACdOnVi27ZtREdHo6oqa9asoXPnziX5NoQQpVxwcDBDhgwp6WoIUWZ4eXmxc+dOqwktmzZtsozXLy4zZszIdk5AeHg4s2fP5ttvv2XNmjWsWbOGc+fOZdlvwoQJTJkyhV9++QVVVVm7di2qqjJu3DgmTJjATz/9RJ8+fZgyZQoATz75JJ9//nm2S9aWZyUaDA4ePJjq1avTp08fBgwYwIABA2jRogUAs2bNYvXq1fTs2ZPZs2czd+5cNBoN9evX58UXX+Tpp5+ma9euaLVamTwihMjVoUOHOHXqVElXQ4gyw9XVlcDAQA4cOGDZ9tdff9GmTRvLzzt27GDAgAH07duXsWPHEhsbC8DmzZsZOHAgvXv35tFHH7WUMXToUGbOnMmgQYN45JFH2L59e6512LNnDxUqVMDLyyvLa7t376ZVq1Z4eXnh7OzMo48+ypYtW6z2uX79OqmpqTRu3BiAfv36sWXLFmJjY0lNTaVVq1YAdOzYkV27dpGeno6DgwPBwcH89NNP+b5mZVmxdxN/+OGH/51cp+Ott97Kdr9atWqxYsWKbF/r378//fv3L5L6CSGEECWpRw/4t2PM7rp3h59/tm3fbt268csvv9CqVSuOHDlCvXr1LOP2Y2Ji+PTTT/n666/x9PRk9erVfPLJJ7z33nusXr2aRYsW4ePjw7p161i2bBnNmzcHwGAwsGbNGrZt28bcuXNp3759jufftm0bzZo1y/a1iIgIKlSoYPm5YsWKHDlyJNd9KlSoQHh4ON7e3ri4uLBr1y4efvhhfv75ZwwGA7GxsVSqVIlmzZoRGhp6T8UZJTpmUAghhBClU8eOHZkzZw5ms5nNmzfTrVs3y/Ctw4cPc/PmTYYNGwZk5Er09PREo9GwYMECtm3bxsWLF9m/f7/VJM+2bdsCcP/99xMXF5fr+S9fvmxpvbtbdpNJ756JndM+iqIwb948PvroIz755BP69OmDl5cXer0egKpVq3L58uVc61beSDAohBBClCK2ttwVNTc3N+rXr09YWBh79+7ltddeswSDJpOJpk2bsmjRIgDS0tJISkoiKSmJ/v3706dPH5o3b069evVYuXKlpczMiZu2pNDRaDTodBlhyu+//868efMA6NSpEzVq1LBKFxUREUHFihWtjq9UqRJRUVGWnyMjIy376HQ6S+9jXFwcCxcutHRH63S6ey7Fj6xNLIQodUzmrN/ohRDFr1u3bnz66ac0aNDAEpgBBAUF8c8//3Dx4kUAFi5cyMyZM7l06RIajYYxY8bQqlUrduzYUeBVVapXr87169eBjAmnGzZsYMOGDbz88su0adOGPXv2EBMTQ0pKCr/++ivt2rWzOr5q1ao4Ojpa0rSsX7/ess+bb75p6Vb+8ssv6dq1q6UF89q1a9SsWbNAdS6rpGVQCFFkvjsfX6DjHq/jaeeaCCEKomPHjrz11lu8/PLLVtsrVKjABx98wCuvvILZbKZSpUp8/PHHeHh4EBgYSLdu3XBycqJ58+bcuHGjQOfu1KkTq1ev5sknn8zyWqVKlXj11VcZNmwYaWlpDBw4kEaNGgEwcuRIxo0bR8OGDfnkk0+YPHkySUlJPPDAA5Zu7XfeeYepU6eSkpJCvXr1mD59uqXsffv2ERISUqA6l1WKml2nejmWlpbGsWPHJM9gEZNraR92vY6Z3R7F8JHPPNXac6UjGBw1ahRRUVGEhobatdx7kXy27efOPIOBgYElXZ1SR1VVBg8ezMKFC3NNaWPP5ejS09N54oknWL16NQ4ODnYpM79K4n6QbmIhRLm3ePHiHDMXCCFKJ0VRePPNN1myZEmxnfObb77hhRdeKLFAsKRIN7EQQgghSqVGjRpZun+Lw/Dhw4vtXKWJtAwKIcq9sLAwTp48WdLVEEKIUklaBoUQ5V5m4lpZkk4IIbKSlkEhhBBCiHuYBINCCCGEEPcwCQaFEEIIIe5hEgwKIYQQwsqWLVvo168fvXv3plevXixdujTPY9asWcPGjRuzbI+IiGDEiBH06dOHxx57jD179mTZJzQ0lHr16mU5fvny5dSrV49r164V/M3Y4Ouvv+b333/Psl1VVT766CO6du1K9+7dLauZZOf27dv07NnTqq5vvPEGXbp0oU+fPvTp04fffvuNpKQkxo4dW+CVWYqCTCARQgghhEV4eDgfffQRoaGheHt7k5SUxNChQ6ldu3auK3P8/ffftGjRIsv2mTNn0rFjR4YMGcKFCxcYOnQoO3bsQKvVWu3n7+/PL7/8Qs+ePS3bfvvtNzw8POz35rIRFRXFtm3bWL58eZbXfvnlF86fP8+mTZu4fPkyo0aNYvPmzVZL8wEcPnyYyZMnc+nSJavtx44d45tvvsmybnLr1q1ZvXo1Tz31lL3fToFIy6AQQgghLGJjYzEYDKSmpgLg6urKhx9+SN26dQE4cuQIgwcP5rHHHmP48OFcvXqV3bt3s23bNubNm8fOnTutyuvSpQu9evUCoGbNmqSlpZGcnJzlvM2bN+fYsWOW165fv46rqyvu7u6WfRYvXsxjjz1G7969mTlzJpmLqM2ePZuBAwfy6KOP8sQTTxAZGQnAww8/zHvvvUffvn3p378/V69ezXLelStX8uijj2Z7LbZv30737t3RaDTUrl2bKlWq8Pfff2fZb+3atUydOtUq6EtOTubGjRtMmTKFXr16MW/ePMxmMwA9evTg66+/prQsAifBoBCi3Dt48CArVqwo6WoIYZsePTLWdCyK/3r0yPP09evXJyQkhM6dOzNgwAA+/vhjzGYzNWvWJD09ncmTJ/Ppp5/yww8/8OyzzzJlyhTatGlDp06dGDduHG3btrUqr0uXLnh6ZiwxuWzZMgIDA60CvEw6nY6HH36Y7du3A7B582a6detmeX3Hjh0cO3aMdevWsX79esLDw/nxxx+5cuUKFy5cYPXq1fzyyy/UqFGDn376CYDIyEhat27N+vXrad68OStXrsxy3m3bttG8efNsr0VERIRVgFehQgVu3bqVZb/p06dbUlhlio6OplWrVnzwwQesXbuWgwcPsm7dOgC8vLxwcXHh9OnT2Z63uEkwKIQo94KDg2XtVyHyYdq0aWzbto3Bgwdz48YNBg4cyK+//sqlS5e4evUqzz//PH369OGTTz7JtrUtO8uXL2fNmjXMnDkzx326devGL7/8AsDWrVvp3Lmz5bU9e/Zw5MgR+vXrx2OPPcaxY8c4d+4cNWrUYOLEiXz33Xd8+OGH/PPPP1Ytj5nB6f333098fNb10i9fvoy/v3+29cmu5U6jsS10ql69OgsWLMDX1xdnZ2eGDh1qCXQBqlSpkqVbuaTImEEhhBCiNPn55xI9/Z9//klycjLdu3enf//+9O/fn7Vr17Ju3Tr+7//+j2rVqrFhwwYATCYTUVFReZY5c+ZMtm/fzsqVK3MMvABatmzJ5MmTOXPmDN7e3lYtiCaTiaeffppnn30WgISEBLRaLSdOnGDy5Mk888wzPProo2g0GqsgztHREchY6zi74E5RFMv4xblz57Jt2zYAxo0bR6VKlSxdzpDR0nj3+L+cnD59mkuXLlm6oFVVtRprqNPpbA4si1rpqIUQQhShUaNGMX369JKuhhBlgpOTE59++qllVqyqqpw7d47AwEDuu+8+4uPjOXjwIADff/8948ePB0Cr1WY7Q3b58uXs27ePVatW5RoIZpbx8MMP8/bbb9O9e3er11q1asWGDRtISkrCaDTy4osv8ssvv3Do0CFatGjB4MGDqVu3Ln/99Ve+ZurWqFGDGzduAPDyyy+zYcMGNmzYQEhICO3ateOnn37CZDJx+fJlLl26RMOGDW0qV1VVPvjgA+Lj4zEYDKxZs4ZHHnnE8vq1a9eoUaOGzfUsStIyKIQo95YsWVLSVRCizGjVqhVjx45lzJgxGAwGIKOr9cUXX8TBwYG5c+cyffp00tLScHNz46OPPgKgTZs2zJo1C3d3d7p27QpkBEQLFizAzc2NoUOHWs6xePFiKlWqlO35u3XrxoYNG+jUqZPV9k6dOnHq1CkGDhyIyWSibdu2PPbYY1y6dInXX3+dXr16odfr852KpmPHjuzdu5c6depkea1r164cOXKE3r17AxljA52cnAgPD2fUqFGWFtLs1K9fn1GjRjF48GCMRiNdunSxzJROSEjg9u3b1K9f3+Z6FiVFLS1TWYpJWloax44do0GDBpam40xhYWEEBweXUM3KF7mW9mHX66goGf8vho985qnWnss6PscWj9fxtGNtMrqBIPvxPyJ/5LNtP5nX8uTJkzKmtRCSkpJwdXUt8PGRkZG88sor2U4uKSpfffUVOp0u29QyJXE/SDexEEIIIe5ZFSpU4JFHHmHr1q3Fcr6kpCT27NnDoEGDiuV8tpBuYiGEEELc05555pliO5erqyuLFi0qtvPZQloGhRBCCCHuYRIMCiGEEELcw6SbWAhR7jVt2jTb5a+EEEJIMCiEuAeEhYURFhZW0tUQQohSSbqJhRBCiFLGZC6aNEhFVW5xW7NmDRs3bizQsW+88QbXr1+3c43KNmkZFEIIIUoZrUbhu/MFy9OZG3vn8Cwpf//9Ny1atCjQsfv27ePFF1+0c43KNmkZFEIUKVVVSUg3E59uItVkxlwCiZ8VRaFZs2bFfl4hyqJevXpx/vx5AF577TWmTp0KwD///MPIkSOt9r148SJDhw6lV69eDBo0iCNHjuRadlxcHC+++CLdunWjT58+7NmzB4A//viDPn360KtXL1544QXLesedOnVizpw5DBgwgB49enDs2DF2797Ntm3bmDdvHjt37iQ6OpoXXniBfv360b9/f3bv3g3ASy+9xJw5cwBYtGgRL7/8MosXLyYiIoJRo0YRGxtrt2tW1knLoBCiyJhVlfAUE8nG/wJArQJVXXXoNUoJ1kwIkZP27duzZ88e6tSpw5kzZyzbd+zYQYcOHaz2nTBhAqNGjaJLly78888/vPzyy/zyyy84ODhkW/bcuXOpUaMGCxYs4PTp07z99tsEBATw9ttvs2rVKqpVq8bSpUt59913mTdvHgBeXl6sW7eOFStW8MUXXzB//nw6depEixYtaNu2LS+99BL9+/cnJCSEiIgInnzySdavX88777zDY489RmBgIN999x3ff/89Xl5erF69msWLF+Pt7V1k17CskZZBIUSRuZ5kJNmo4uekoYabjkrOWlQVwlNMsjScEKVUhw4d2LNnD+fOnaNu3bpoNBqio6PZsWMHHTt2tOyXlJTElStX6NKlCwCNGzfG09OTCxcu5Fj2gQMH6NOnDwD16tVjzZo1HDlyhEaNGlGtWjUABg0axN69ey3HtG3bFoD777+fuLi4LGXu37+fefPm0adPH0aOHInRaOTq1av4+voyadIkxo0bx5QpU/Dy8irspSm3pGVQCFFkDGao7KLFRZfxvTOzNTA8xURMmhlfJ21JVk8IkY0mTZrw+uuvs3v3blq0aIGvry9btmzBYDBQpUoVy36qqmb5UqeqKiaTKceydTrrsOP8+fOYzeYsZRiNRsvPjo6OwH9rjN/NZDLx1VdfWYK98PBw/Pz8ALhw4QK+vr4cO3YsS6um+I+0DAohioyPo8YSCGZy02vw0GuISzeTbDTncKQQoqRotVqCgoJYsWIFLVq0oFWrVixatIj27dtb7efm5kb16tX59ddfgYwxhVFRUdx///05lt2sWTM2bdoEZASCI0eOJCgoiMOHD3Pt2jUgY6Zwy5Yt86xjZtDZvHlzvv32WwDOnTtH7969SUlJ4eTJk/zwww+EhoYSGhrKqVOnshwrMkgwKISwK9MdLQWeDtk/YnydNOgUiE2TYFCI0qh9+/akpKRQp04dWrRoQXR0tKVlbe7cuaxatQqAjz/+mBUrVtCrVy/effdd5s+fj4ODA7///jtvvfVWlnLHjRvHpUuX6N27NxMmTGDmzJn4+fnx7rvvMnbsWHr06MH+/fuZNm1arvVr06YNixYtYsuWLUycOJHDhw/Tq1cvXn31VWbOnImjoyOTJk3ijTfewN/fn9dff52JEydiMBjo0KEDo0aN4urVq3a/bmWVot5jA3fS0tI4duwYDRo0sDQ9ZwoLCyM4OLiEala+yLW0D7tex8wuliL+yB+ISKFFJWcA1p7LOTVGbFpGV3ENt6yTSeyd/iKze+kee9wVCfls20/mtTx58iSBgYFWr5nMKtoimGRVVOWWpKSkJFxdXUu6GnaT3f1Q1GTMoBDCbtJNKn/dSgac89zXXa8hJs1MQnrRjx384osvuHz5cpGeQwh7KqqArbwFgsI+JBgUQtjNqbg0Uk22tb7pNAouOoVEgxkfR02Og8PtYdSoUbIcnRBC5EDGDAoh7OafqNR8tfJ56DWYVKzyEAohhCheEgwKIewiIsXIjWQjjX2dbD7GRaegVSDBULQTSRYvXkxoaGiRnkMIIcoqCQaFEHbxT1QqWgUa+DjmvfO/FEXBXa8h2ahazUK2t9GjR/PBBx8UWflCCFGWSTAohCg0g1nleGwa9b0ccdbl77HiossYK5giXcVCCFEiJBgUQhTaqdg00kwqQX62dxFnctIqKEgwKIQQJUWCQSFEoZ2KS8PTQUN11/wnKFAUBWedQopJElALkUk1GfPeqYTKPXr0qCWh9Jo1a9i4cSMAkyZNKraxuXeeN7/eeOMNrl+/bucalW2SWkYIUShpJjOXEg009XMqcHoYZ61CslHFaFbRSR40IVC0Ooyrltm9XN3gEYUuo2HDhjRs2BCAv//+mxYtWhS6zPwqzHn37dvHiy++aOcalW3SMiiEKJQLCQZMKgR42T5x5G6Z4wylq1iIkterVy/Onz8PwGuvvcbUqVOBjLWHR44cyb59+xg6dCi7d+9m27ZtzJs3j507dwLw559/MmDAADp27MiaNWuylB0XF8eLL75It27d6NOnD3v27AHgjz/+oE+fPvTq1YsXXniBqKgoADp16sScOXMYMGAAPXr04NixY1nOGx0dzQsvvEC/fv3o378/u3fvBuCll15izpw5ACxatIiXX36ZxYsXExERwahRo4iNjS3S61iWSMugEKJQTsel4apTqFqALuJMDhrQKJBsMuMu31GFKFHt27dnz5491KlThzNnzli279ixw7I+MWSsD9ypUydatGhB27Zt+fnnn0lPT+e7777j7NmzDBs2jEGDBlmVPXfuXGrUqMGCBQs4ffo0b7/9NgEBAbz99tusWrWKatWqsXTpUt59913mzZsHgJeXF+vWrWPFihV88cUXzJ8/3+q8L730Ev379yckJISIiAiefPJJ1q9fzzvvvMNjjz1GYGAg3333Hd9//z1eXl6sXr2axYsX4+3tXSzXsyyQp64QosCMZpULCQbu93REU4gVRBRFwVmrkGJUi2T9YFVVOXjwoN3LFaI86tChA3v27OHcuXPUrVsXjUZDdHQ0O3bsoGPHjrkeGxISgqIo3H///dm2vB04cIA+ffoAUK9ePdasWcORI0do1KgR1apVA2DQoEHs3bvXckzbtm0BuP/++4mLi8tS5v79+5k3bx59+vRh5MiRGI1Grl69iq+vL5MmTWLcuHFMmTIFLy+vAl6R8k9aBoUQBXYp0UC6WSXAy6HQZTnrFJKMKkWcf1oIkYcmTZrw+uuvs3v3blq0aIGvry9btmzBYDBQpUoVrl69muOxWm3GCkQ5jR/W6azDjvPnz2M2W3/oVVXFaPxvooujo2OuZZpMJr766itLsBceHo6fnx8AFy5cwNfXl2PHjlm1agpr0jIohCiwM3FpOGoUarrpC12WS+a4QZlVLESJ0mq1BAUFsWLFClq0aEGrVq1YtGgR7du3z3Zfk8lkc9nNmjVj06ZNQEYgOHLkSIKCgjh8+DDXrl0DMmYKt2zZMs86Zp63efPmfPvttwCcO3eO3r17k5KSwsmTJ/nhhx8IDQ0lNDSUU6dOFajO9wJpGRRCFIhZVTmbkE4dTwe0dpgBrFNAq0BqEUwiCQ4OJjk5mZMnT9q9bCHKo/bt23PgwAHq1KlDhQoViI6OzrZlrU2bNsyaNQt3d3ebyh03bhyTJ0+md+/e6HQ6Zs6ciZ+fH++++y5jx461tD5Onz4913LuPO/EiROZMWMGvXr1AmDmzJk4OjoyadIk3njjDfz9/Xn99deZOHEi69ato0OHDowaNYqlS5dSvXr1fF+b8khRi2KATimWlpbGsWPHaNCggaXpOVNYWBjBwcElVLPyRa6lfdj1OmZ2sdjpI38jycDXZ+LpVdONB32sk01nnmrtufh8lXkz2YjBrPJakJ9d6vhffTIqdI897oqEfLbtJ/Nanjx5ksDAQKvXVJMRRWv/9pqiKrckJSUl4erqWtLVsJvs7oeiJt3EQogCuZhoAKC2e+HHC2Zy1CoYzBm5C4W4lxVVwFbeAkFhHxIMCiEK5GJCOv7OOlz09nuMOP7b3RyeIuN5hBCiuJSKYPC3336jV69e9OnTh6FDh3LlyhVMJhPvv/8+Xbt25ZFHHmHVqlWW/S9dusSTTz5J9+7dGTBggCU5phCieKSZzNxIMlLbo/ATR+7kqM0IBm8lF81SXEIIIbIq8fbi1NRUJkyYwIYNG6hZsybLly/n/fffp3379ly+fJmNGzeSlJTEoEGDePDBB2nUqBHjx4/n6aefplevXmzfvp1x48axcePGAi+FJYTInyu3DZiBWu72DQZ1GgWtAuESDIp7jNlsRqMpFe0zogTdnWanuJT4nWcymVBVlcTERCBjIKijoyNbt26lX79+6HQ6PD096dGjBz/++CPh4eFcuHCBHj16ABkznlJSUjhx4kRJvg0hyg2TOe9JFhcTDOg1UNXVvsEgZLQO3kqRYFDcO1xdXbl+/Trp6ekyyekepaoq6enpXL9+vUQmw5R4y6CrqyvTpk3jiSeewMvLC7PZzKpVqxg9ejSVK1e27Ofv78/p06e5efMmFStWtPoGValSJW7dusWDDz5YEm9BiHJFq1H47nzus4Cv3Dag1yj8cDEhhz08C3x+R61CdKqJdJOKg9Y+rf0jR460rHUqRGlTrVo1oqKiuHz5slWyZWGb9PR0HBzsN5GtpGQ2fmUmzC7Wcxf7Ge9y+vRpFixYwKZNm6hRowZff/01L730UrZNpRqNJscm1Mys57Y6duxYttvDwsLyVY7ImVxL+7DXdcxMBpJXecHBwcTH5xwMmlAwKM44mNKIj0/KYa+MYDC3cnIuXwOKEzv+OY43afk+PjujR48G5J60F7mO9iPX0j7KSxCdkJCQ7QovRZ3OqcSDwV27dtG0aVNq1KgBwFNPPcWMGTNo2bIlkZGRlv3Cw8Px9/enSpUqREVFoaqqZYxg5mv5IXkGi5ZcS/soiutoS3menjm37MWnmyDVjI+bS54td7mVkxOjWSXhthHP6nUIruic7+NzIvekfch1tB+5lvYh17HwSnzM4AMPPMCBAwcsXThbt26lWrVqhISE8P3332M0GklISODnn3+mc+fO+Pv7U6NGDctyNjt37kSj0RAQEFCSb0OIe0aKUUWrgB0zyljRaRTcdBq7zigOCwuT1UeEECIHJd4y2Lp1a0aMGMHQoUPR6/V4enqycOFCateuzZUrV+jTpw8Gg4FBgwbRokULAGbNmsWUKVP4/PPPcXBwYO7cuTILS4hioKoqqSYVZ61SpLP3K7loCbfjJJJmzZoBMGTIELuVKYQQ5UWJB4OQ0TX81FNPZdn+1ltvZbt/rVq1WLFiRVFXSwhxF4MZTCo464r2y5e/i44LCSl2nUQihBAie9KcJoSwWaopI+2FUxEHaBWddahATJqsRCKEEEVNgkEhhM1SjOYiHS+Yyc8pIztAVGr5mCEohBClmQSDQgibqKpKSjGMFwTwctSiUSAqVVoGhRCiqEkwKISwiVHNGC/opCv6MXxaRcHHUSvBoBBCFAMJBoUQNkkxZowXdNYWz2PD10lLtHQTCyFEkSsVs4mFEKVfismMphjGC2byc9JyJi4dg1lFrylca+TBgwclz6AQQuRAgkEhhE1SjcUzXjCTn9O/M4pTTVRyKdyjSlYnEEKInEk3sRAiT0azilEt+pQyd/L9d0ZxtIwbFEKIIiUtg0KIPGXmFyzqZNN38nHUopCZXsYxr91zNWrUKKKioggNDbVL3YQQojyRYFAIkadUk4oCOBRjX4JOo+BtpxnFS5YssUONhBCifJJuYiFEnlKNKo7FOF4wk6+TlmhZhUQIIYqUBINCiFyZVZU0s1qs4wUz+TlpiU01YTKrxX5uIYS4V0gwKITIVVoxrUecHT8nLWYgVloHhRCiyEgwKITIVWoJBoO+ThnDmmUlEiGEKDoSDAohcpVqUtFrQFvIxM8FkZleRoJBIYQoOjKbWAiRI1VVSTWpuBbDesTZ0WsUPB00xBSym7hp06YkJyfbqVZCCFG+SDAohMiRUQWzCo4l0EWcydtRW+gxg2FhYYSFhdmpRkIIUb5IN7EQIkepxszxgiX3qPB21BKTZkJVZUaxEEIUBQkGhRA5Kolk03fzdtSSZlJJMUkwKIQQRUG6iYUQOUo1mXEqgWTTd/J2zIhEY9NMuBRwObzM+kvrohBCZCUtg0KIbJlVlXRzyaSUuZOPY8aMYsk1KIQQRUOCQSFEtiz5BUtoJnEmTwctChIMCiFEUZFgUAiRrcxgsCRnEgPoNAoeDhpi08wlWg8hhCivJBgUQmQrLTPZdAmOF8yUOaNYCCGE/UkwKITIQlVVUo1qiY8XzJSZa1AmgAghhP1JMCiEyMJgBjMlm1/wTpJeRgghio6klhFCZGGZPFJKWgbvnFFckPQyX3zxBZcvX7Z3tYQQolyQYFAIkUWqyYxGAX3paBi0yjVY1VWf7+NHjRoly9EJIUQOSsmjXghRmqSa1BJPNn2nzPQyMolECCHsT4JBIYSVVKMZQylINn2nzPQycQVML7N48WJCQ0PtXCshhCgfJBgUQli5mWwESj6/4N0Kk15m9OjRfPDBB3aukRBClA8SDAohrNwopcGgj6SXEUKIIiHBoBDCys0kY6lJNn0nr8z0MkYJBoUQwp4kGBRCWKiqys1kQ6lrFYT/ZhTHpcskEiGEsCcJBoUQFgkGM0mlYOURkzlr65+nQ0auwfh02yaRZFeGEEKIrCTPoBDC4mZS6RgvqNUofHc+3mqb+d+xgjtvJnEsJjXPMh6v41kkdRNCiPJGWgaFEBY3ko1oFXDUlL5uYo2ioFHAIC1+QghhV9IyKISwuJFkoJKzrtQkm76bXqNgLECqQVVVZQUSIYTIgbQMCiGAjG7Y8BQjlV1L73dEvbQMCiGE3ZXep74QolhFpZowmKGKi464AiR37lnNBWdH63WD8xq3l5JmYOO1ZJvPodMoGI0qqqqW2tZLIYQoayQYFEIAcOPfySNVXPWciE3L9/HOjnquLl30709jAO74OXvVnxuTr3Po/x3LaFQzWgltFRwcTHJyMidPnszX+YQQ4l4gwaAQAoCbyQactApeDqV39Iju36oZzKolMLTFoUOHiqhGQghR9pXep74QoljdSDJS2aX0Th6BO1oGCzCJRAghRPYkGBRCkG5SiUo1UaUUTx4B0P0bp8okEiGEsB8JBoUQ3EoxogKVXfR57luSFEVBp4BRgkEhhLAbCQaFENxMMgAZM4lLO71GwSCxoBBC2I0Eg0IIbiQb8XTQ4KIv/Y8EnUZaBoUQwp5KfzOAEKLI3UwyUrWUjxfMpNcomFQVs6qisXGyy8iRI4mKiirimgkhRNlUNp7+Qogic9tgJsFgpplr6R4vmEn374xigxkctbYds3jxYlmOTgghclD6+4SEEEXqZnLZGS8IkNmTLV3FQghhHxIMCnGPu5lkRAEq2SkY1BgNdiknJ3ols2XQ9mAwLCxMVh8RQogclI2mACFEkbmRbKSCszZfK3pkR72diEdsOM7JCZZtLomxJLt7F7aKVjQKKGQsSWerZs2aATBkyBC71kUIIcoDaRkU4h6mqio3k41UKWR+QX1aCsYv5+OcnECyq6dlu0dCFG4J0aDar0tXURT0Gkk8LYQQ9iItg0Lcw2LSTKSZVCoXYiaxYjbTfNf3kBBHjF81DI7OlteSXTxwS4zBpNWRckeQWFg6jSJjBoUQwk6kZVCIe9iNJCNQuMkj9Y5up+LNC2h79LcKBAESvCqS5uCMe0IUitlUqLreSa9RMJozWjaFEEIUTqkIBk+fPs3QoUPp27cv/fr149ixYwAsWrSIrl278sgjjzB//nzLgz8mJobnnnuO7t2707NnTw4dOlSS1ReizLqZbMRBo+DrZGOOlrs4J8Vz//HdXK3VAE2Tlll3UBQSvSqgmM24JcQUsrb/0SlgJuM/IYQQhWNzMPj+++9z5MgRu1cgJSWFESNG8Nxzz7F+/XpeeOEFxo8fz/bt29myZQuhoaFs3LiRffv2sXnzZgCmTZtGs2bN2LRpEx9//DEvv/wyKSkpdq+bEOXdzWQj/i46m5M3363+ke0AnGjcKcd9jHpHUlw8cEmKQ2tML9B57paZa9Ao0aAQQhSazcFgTEwMTz/9NJ07d2b27NmcOXPGLhX466+/qF69Ou3btwcgJCSEOXPm8Ntvv9GzZ09cXFxwdHSkX79+/PjjjxiNRv78808GDhwIQGBgILVq1WLnzp12qY8Q9wqjWSU8xUjlAnYRu8dFUOPCYS4GNCPFzSvXfW97+ALgcju+QOe6m16T//QyQgghsmfzX4FZs2aRmprKH3/8webNmxk4cCDVq1enZ8+edO/enerVqxeoAhcvXqRChQq8+eabnDp1Cg8PDyZMmMDNmzdp3bq1ZT9/f3/Cw8OJjY3FbDbj4+Njea1SpUrcunWrQOcX4l4VkWLErEKVAk4eCTj2F0adnjMN2ua5r1mrI83JDeeUBBI9fUEp3AgVXT4TTx88eFDyDAohRA7y9VfAycmJbt260a1bN5KSkliyZAkLFy5kzpw5NGrUiIEDB9KvXz+UfHQ5GY1Gtm/fztdff01QUBBbt25l1KhR3HfffVn21Wg0mM3Z9wtptfkb85Q5LvFusmSV/ci1tA97Xcfgu8q7gjtofIg6f5Lb/De5Izg4mPj43FvwnFKTqHL5OKdrNSIyzQBp8YAnBqPRar87f050cqVC6m10SQmkOLlZtmd/Ls9c66ACCs4kpaahpOaU5NrT6toFBgbKPWknch3tR66lfZT36xgcHJz3ToWQr2BQVVUOHDjA5s2b+e2330hPT6dnz5706NGDiIgIFi5cyK5du5g9e7bNZVasWJH77ruPoKAgADp37szkyZPRaDRERkZa9gsPD8ff3x9f34zupvj4eDw9PS2vVapUKT9vhQYNGuDo6Gi1LSwsrMgv+L1CrqV9FMV1zCzvxqVE3G4beKhB4yz7ZH62clLv8mG0qpnrDR/C0+O/ffU660fKnT+btO6YEqNxT03CeEe3ck7nyqsOibcNaDQOeLq45LjPnddO7kn7kOtoP3It7UOuY+HZ3Fczbdo02rZty6hRo4iNjeWdd95h165dTJ8+nTZt2tC3b19eeeUV/vzzz3xVoF27dly/ft3SUnfgwAEUReHpp5/mxx9/JDk5mfT0dEJDQ+ncuTM6nY4OHTqwZs0aAE6dOsX58+dp2TKbmYxCiBzdSDYUaLygYjZR62wY4ZXrkPTvWEDbDlRIdvHAMS0ZrR2WrMtPrsFRo0Yxffr0Qp9TCCHKI5v/Ely7do3x48fzyCOP4Orqmu0+DRs25PPPP89XBSpUqMCCBQuYNm0aKSkpODg4MH/+fJo1a8aZM2d4/PHHMRgMhISE0LdvXwCmTp3K5MmT6dmzJ4qiMHPmTNzd3fN1XiHuZSlGM7FpZhr55D8YrHz1NM4pifzTske+j011ccc9MQbH1CSS85h0khe9RiHFqKKqap5DU5YsWVKocwkhRHlm81+CJUuWcOjQIU6ePGlZ53P+/Pm0a9fO0sVbvXr1Ak0kad68Od99912W7WPGjGHMmDFZtvv5+bFo0aJ8n0cIkeFmcsZYvoKsPFL94hFSXNwJr1I338eadA4YdXq7BIM6JWPsoFkFbeGWVRZCiHuazX8Jvv/+e6ZNm8bEiRMtweCNGzcYNmwYM2bMoHv37kVWSSGEfWUGg/45dBP3rOaCs2PW9YrVlGSMN86jafEwj9/vXaBzpzm54nI7DiWHyWC2ykwvY1ShYCmzhRBCQD6CwUWLFjFjxgx69Piva2jGjBm0adOGefPmSTAoRBlyI8mAr5MWJ232w4adHfVcXZq19d05KR5Ps4mI8xcw3vV69eeytuJnJ83JFdfbcTikJee/4nfQ3ZFr0FGaBoUQosBsnkASGRnJgw8+mGV7w4YNuXnzpl0rJYQoOqqqciPJWKD1iJ1SbmPU6jHqHfPeOQfpDs6YFQ2OqUkFLgPyn2tQCCFE9mwOBhs2bMhXX32VZWH4lStXUr9+fbtXTAhRNOLSzaSY1Hwnm9aYjDikJZPq4gYFXL4OAEUhzckFx9QkVLXgXcVaRUEDGGRJOiGEKBSb/xq88cYbPPvss2zfvp3AwEAgI61LcnIyX3zxRZFVUAhhXzeSMtK6VHHJOiYwN04pt1GAVOfCz9xPc3LFOeU2hN8Css9OYAudBoxq3i2DTZs2JTm5cN3SQghRXtkcDD7wwANs2bKFTZs2cf78efR6PQ899BC9e/fGzc0t7wKEEKXCjWQjeg1UcM7ftAunlEQMOodCdRFnSndwBsB85QL4NixwObbmGgwLCyv3KxQIIURB5aufyNvbm6eeeqqo6iKEKAY3koz4u+jQ5KOrV2M04JCeSmJ+kkznwqzVYdLqUAoZDOYn16AQQojs2RwMXr16ldmzZ3P06FGMRmOWsYP5XXlECFEywlOMNK/gnK9jnFJuA5DqbKdeAEUh3cEJ7eWL0Fgt8BhEyTUohBCFZ3Mw+PrrrxMfH8+wYcOkW1iIMsysku/JI84piaTrHTHpHOxWj3QHZ5zjI3G5HUeye8FyFmbmGjSoKlpyjgYzWw3v/hIrhBAiH8Hg8ePHCQ0NpW7d/K86IIQoXfITDGoN6egNaSR4+Nm1DgbHjNZJ34grBQ4GM3MNGs1I5mkhhCggm1PL1KpVi+jo6KKsixCiGLjrNbjrbY+cnFISUSEjpYwdGXUO4OSMb8SVApchuQaFEKLwbG4eGD58OFOmTOHpp5+mevXq6PXWaSlat25t98oJIewvX13Eqopzym0MDs6YtflLRZMnRUGpURvfmwUPBiXXoBBCFJ7NfxUmTZoEwHvvvZflNUVROHnypP1qJYQoMvlZeURnTEdnTCfes0KR1EWpXgv3MyfQpyVjcHQpUBm25hoUQgiRPZv/Kpw6daoo6yGEKCZVXG1v4XNK/reL2F6ziO+iVKkOgFfMTSIr1ylQGXqNQrp0EwshRIHZPGYQIC0tjR9//JH58+cTFxfH3r17iYyMLKq6CSGKgL+tLYOqilNKIumOLqja/K9jbAulcjUAvKILvr55RuJpmSkshBAFZfMT/vLlyzz99NPodDpu3bpF3759Wb16NXv27GHZsmU0aNCgKOsphLCTzHQsee5nSENnMpLkbp9E09lRnF1IcvPCK6YQwaANuQa/+OILLl++XOBzCCFEeWZzy+D7779P586d+e233yyTR2bNmsWjjz7KBx98UGQVFEIUXkFazTK6iBVSnQu+drAt4nwqFyoYvDPXYE5GjRpFv379CnwOIYQoz2wOBv/++2+eeuopqyWfNBoNzz33nEweEaKUi0415Wt/1WzGKSWRNCcXVE3RJvCL86mM6+049GkpBTreKtegEEKIfLM5GHRxccl2fOCZM2fw8PCwa6WEEAVjymEixfVkY77KUa9cQGs2kersbo9q5SrOtzIAngVsHczMNWjIZRLJ4sWLCQ0NLVD5QghR3tk8ZvCJJ57g7bffZvz48QCcP3+ePXv2MGfOHAYPHlxkFRRC2E6rUfjufHyW7ZEpJoL+/Xd2r9/p8TqeqEf/xqwopDkVbRcxZLQMQsaM4qjK9+X7+Mxcg7m1DI4ePRqA6dOnF6SKQghRrtkcDL7wwgu4u7vz/vvvk5KSwpgxY/D19eXZZ59lxIgRRVlHIUQhpZps70NVTUbMJ4+Q5uSKqslXwoECMTi6kOzqWbhJJJJrUAghCixf+SKGDh3K0KFDSU5OxmQy4e5e9F1IQojCMasq6fkYT6eeOwUpyaT+22JXHOJ8KhcqvYzkGhRCiIKzORhct25drq8PGDCg0JURQthfmil/QZL5cBi4uhVLF3GmeG9/qlw9hdaYjknnkO/jdRqFZKOKqqpWk9yEEELkzeZgcOHChVY/m0wmoqOj0el0NG3aVIJBIUqp1HwEg/q0FNQzx9E0ewiuFbylLr8SvCoC4B4XSZxf1Xwfr9dk5Bo0qRl5B4UQQtjO5mBw27ZtWbYlJyczdepU6tQp2DJSQoiil2pS0ds49K/q5eNgMqEJCoZrG4u2YndI8MpY+9gjvmDBoO7f1kCjqqJDokEhhMiPQo0Od3FxYezYsXz11Vf2qo8Qwo5UVSXVpOKU09Icd6l+4QhU8Af//AdkhZHk5o1Jq8MjLqJAx0uuQSGEKLhCLzh64sQJzGZ5AgtRGhnMGcu0OWnz/t7nmhCNb9Q1NJ17FNu4O9Vk5PE6ngAYKvpzf3oMgf/+nLn9TilpBjZeS86yXZ9HrkFVVQkLC7NTrYUQonyxORh88skns/yBSEpK4uzZszz77LN2r5gQovAyxwva0jJY/eIRVEDTMLiIa/UfRavj6tJFAHjGxeMQfpObSxdR2VXPzSRD1jo+NybbcjSKgkaRlkEhhCgIm4PBNm3aZNnm4OBAw4YNad26tV0rJYSwj1STGQ3kPWZQVal+4SiRle+jqkfWFrniYNA74pySiGI2Afp8H69XFMk1KIQQBWBzMDh27NiirIcQogikmlScdEqe3b6+kVdwTYrjZFAHine04H+M+oyUMjpDOuCU7+N1GnLMNRgcHExycrKsoy6EENmwORh8/fXXbS505syZBaqMEMJ+TKqKwQzuehu6iC8cwajTc7NG/WKoWfb+CwbTCnR8brkGDx06VOj6CSFEeWXzbGIfHx82b97MhQsX8PDwwM/Pj8jISH788UdSUlLQarWW/4QQJS/VaNt4QV16GlUvHed6jQcKlPDZXswaHWZFg96YXqDj78w1KIQQwnY2twxevXqV4cOH8+qrr1pt/9///se+ffuYMWOG3SsnhCi4zMkjjnkEg9UvHkFvTOdiQLPiqFbOFAWj3qHgLYOZuQbNqiXVjBBCiLzZ3DK4e/du+vbtm2V7hw4d2LNnjz3rJISwg1STiqNGQZPbeEFVpfaZg8T6VCbOt0rxVS4HRp0DWmPWWcS2sOQalJZBIYTIF5uDwVq1avHdd99ZbTObzXz11VfUr19y44yEEFmpqkrav5NHcuMbeQWP+MiMVsFSsKavUeeA1mxCNZnyfWxeuQaFEEJkz+Zu4rfeeovRo0fz66+/Uq9ePVRV5eTJk6iqyuLFi4uyjkKIfEozqajkPV6w9umDpDs4cb1Wg+KpWB5Mun9TyqSnkd/0MpJrUAghCsbmYLBZs2b8+uuv/Pzzz1y8eBEnJydCQkLo2bMnjo6ORVlHIUQ+2ZJs2jHlNlWunuRCQPP/grASljmjmPR0KECd9IqSbcvgyJEjiYqKKmz1hBCiXMrXcnS+vr48+uijXLx4kaCgIJKSkiQQFKIUSjWp6BRynUhR89zfaMxmLt1ffCuO5MWk1aMCSnoa6FzzfXxOuQYXL14sy9EJIUQObB4zmJSUxLhx42jfvj3Dhw8nKiqKKVOmMGjQIKKjo4uyjkKIfFBV1ZJsOkdmM7XOhhHhX5vbnn7FV7m8KApGnUNGy2AB6DQKRnPGNRBCCGEbm4PBjz76iNjYWH7//XdLa+CkSZNQFIX333+/yCoohMgfo5qRay+3LmL/62dxSU4o+XQy2TDp9P+OGcy/nHINhoWFyeojQgiRA5u7ibdt28bixYupWvW/xapq1qzJO++8w7Bhw4qkckKI/Psv2fR/3/V6VnOx/PvxOp4Ydx9CdffkofbNUTSlK1G8UecAqUmgqvme4XxnrsE7NWuWEfQOGTLEPpUUQohyxOZgMDU1Fb0+64Du9PR06ZIRohRJNaloAIc72v2dHf/77N78fC5+EVdI9PAl6cslWY6v7KpHN3hEMdQ0e5mTSLRGAyZ9/lZE0f87RtIgjyQhhLCZzd3EISEhfPrppyQkJFi2Xbp0iffee48OHToURd2EEAWQajLjqFWyrM+byfV2LGZFIdnVs5hrZpvMJfF0BViWTvfvE+3ulkEhhBA5szkYnDJlCnq9npYtW5KSkkLfvn3p1q0bXl5evPXWW0VZRyGEjVKNZtLN4JzL5BGn5ERSXDxQS1n3cCbjvyllChIMSq5BIYTIP5u7iW/evMm8efO4du0a58+fx2g0Urt2berUqVOU9RNC5MP1JCOQd7LpZDevYqhNwagaLWi1BV6WLqdcg0IIIbJnczA4bNgwlixZQoMGDahevXpR1kkIUUDXkzICKMdcgsE0J1dLV2yp5eBYoJZByDnXoBBCiOzZ3E3s7+9PeHh4UdZFCFFIV24bcNQoaHKZhZvk5l2MNSogvUPBWwYl16AQQuSLzS2D9erVY9y4cQQGBlK1atUsK4/MnDnT7pUTQtjOYFa5mWzEXZ/1O56qmskMDw0OTsVbsYLQ69GaTShmM6rG5u+sQEbLoArcNppx12eMizx48KDkGRRCiBzYHAwqikLv3r2Lsi5CiEK4kWTIMdm0evqEJRjMb+6+EuHwb3oZkwGjJn9LXmaml4lP+y8YDA4uPUvuCSFEaZNrMNirVy+++eYbPD09mTFjBgAxMTF4eXmhyee3dSFE0bpy24ACWZehU1XMu363fUxIaXBHrkGjPn/BYGbi6dg0E9XcsuZGFUIIYS3Xvw9nz57FaDRabevcuTPXr18v0koJIfLvym0DlZx1aO9q+fMLv4R6/UoJ1aqA9AXPNZjZSx6XbrJsGzVqFNOnT7dL1YQQoryxuZs4kwzKFqL0MZpVbiQZaernREyayeq1gGO7wM29hGpWMIpWi0lTsPQyiqKgUyAu7b9kg0uWZF1pRQghRIYy1XMkhMjejSQjJhVquFt3i3pFXafirYtoWrUvoZoVnFGrR2sq+IziO1sGhRBC5EyCQSHKgSu3M4Km6q7WwWDA8b9Id3BC06x1SVSrUEw6PboCppfRaTLGDAohhMhbnt3EGzduxNXV1fKz2Wxm8+bN+Pj4WO03YMCAQldm69atvP766xw6dAiARYsWsX79ekwmE71792bs2LEoikJMTAyvv/46N27cQKPR8O6779K0adNCn1+IsurKbQMVnbU46f77fuceF0mVq6c41bAtDR3LQDqZuxh1epxSEkE1g5K/7616jUJimpl0k4pDHquxCCHEvS7XYLBKlSp89dVXVtt8fX1ZvXq11TZFUQodDF66dImPPvrIMiZx+/btbNmyhdDQULRaLSNGjKBOnTp0796dadOm0axZM8aMGcPJkycZNWoUv/76K87OzoWqgxBlUcZ4QQON/awDvvuP/4VRq+dCvZY0LKG6FYZJp0cBtEYjJn3+VkzJTC8Tl26ionO+h0YLIcQ9Jden5LZt24qlEikpKUyYMIFJkyYxfvx4AH777Td69uyJi4sLAP369ePHH3+kS5cu/Pnnn0ydOhWAwMBAatWqxc6dO+nSpUux1FeI0uRGkhGjCjXuSKPikhhLtUtHuVCvBelOLiVYu4Iz6TLej86YXuBgMDZNgkEhhMhLqRgz+PbbbzNo0CDq1atn2Xbz5k0qV65s+TlzObzY2FjMZrNVN3WlSpW4detWsdZZiNLiUmI6CtaTR+of3Y6q0XLugTYlV7FCMmr/SzydX5m95XH/jhts2rQp9evXt1vdhBCiPCnxr8wrV65Ep9MxYMAArl27ZtmeXQobjUaD2WzOsh1Aq9Xm67zHjh3LdntYWFi+yhE5k2tpH3ldx+NKJTxQOP7P3xkrbVy7SLULRzlRpwnhBjPExwOelv0Nd+UOzUpv435Z3X2MLWVkv4+edLMZs6JBSU+z2ic+Pt6mujjpXDl7PRzdtRgWL14MyD1pL3Id7UeupX2U9+tY1KsolXgw+MMPP5CamkqfPn0wGAyWfz/wwANERkZa9gsPD8ff3x9fX18g4w+Cp6en5bVKlSrl67wNGjTIsr5yWFiYLFtlJ3It7SOv65hqMrP1SAytKzkTXKUqAM0vHMSk13OlSSc8s+ki1uts+9jbul9ux9hSRk776PV6TDo9DmaT1T6Zn/u86IwqehcfguvWBuSetBe5jvYj19I+5DoWXol3E69bt46NGzeyYcMGFi9ejJOTExs2bOCRRx7hxx9/JDk5mfT0dEJDQ+ncuTM6nY4OHTqwZs0aAE6dOsX58+dp2bJlCb8TIYrf1dsGVKDmv13E5muXqXrlJOfrtyqzYwXvZNTpC5R4GsDbQSO5BoUQwgYl3jKYk06dOnHmzBkef/xxDAYDISEh9O3bF4CpU6cyefJkevbsiaIozJw5E3f3srXCghD2cCnRgE6Bqq56VNWMefMPpDi7c7YMjxW8k0mrx8l0G1QVlPyliPFy1HI6Lh2zqqL9dy11WUFJCCGyKlXBYLVq1fj7778tP48ZM4YxY8Zk2c/Pz49FixYVZ9WEKJUuJxqo7qZHp1Ew/3MQ9cZVjrfpm+/Zt6WVJb2MyYBJl7/35OWoxQwkpGc/zlgIIUSGEu8mFkIUTKLBRFSqiVruetSUZExbf0apVpNrtctiVsHsGf8NAAvSVezlkPF4k65iIYTInQSDQpRRlxMzAqSa7g6YNv8AKUloe/TPd3dqaZaZa7BAwaBjRoaBuDRpGRRCiNxIMChEGXUp0YCzVqHC5ROoRw+hafcIin/Vkq6WXZk1WlRFKdAaxe56DVpF1igWQoi8SDAoRBmkqioXEtJ5QEnC/NN34F8VzcMhJV0t+1MUjFp9gRJPaxQFTwetBINCCJGHUjWBRAhhm1spRgypabTetxpUFd2AoSj5TLxeVpgKkV7Gx1GCQSGEyIsEg0KUQRdjU+gRth6HmAi0T41E8a1Q0lUqMiadHsfU5Iz0Mvnk46TlUmI6ixYt4sqVK0VQOyGEKPskGBSijFEN6VTb+A2Vw8+j6TEATZ16eR9Uhhl1DiioaEz5Xx7Px1GLUYUnnn2Oc0f/sX/lhBCiHJAxg0KUIWpcDIblC/G/dZ7z7fqibda6pKtU5EzajBnFugKMG/R2zHjExaZKV7EQQuREWgaFKANUVUU9egjT5h8wm81sbN6P1i1alHS1ioWxEOllfJwyxlEuXbIYNeamrF8qhBDZkGBQiBKmmowo2uw/isHBwZivXsT820bUq5dQqtZgV4s+XMedyi7ZH/N4Hc+irG6xM2t1qCgFCgbddBr0Gpg+/iUApk+fbu/qCSFEmSfBoBAlTNHqMK5aZrVNVVVITsIYcQudIR20WqhUGdXFlROKO7Wir2BevZ7s0infTMoaNFV/LuuyjmWGomDS6QqUa1BRFHwcy+csayGEsBcJBoUoRTKDQKIjITUFjUYDFSqBpzeKRsNNJ2+SjSq1kyJKuqrFylTAXIOABINCCJEHCQaFKCXUzCAwJRl0eqhYmdsqeHp7W/Y5614ZDXDf7fCSq2gJMOr0OCenZgTL+eTtJMGgEELkRoJBIUqYGh2Jeu1yRougVgcV/TNaAhUF4uKs9j3n5k8Ndz1O5oK1kpVVJp0ejWqGlKR8Hystg0IIkTsJBoUoIarJhHnnVsy7fs9IqOxXCbwyuoOzE+3gRoyjO808HYq5piUvM72MGhMNeOXrWAkGhRAidxIMClEC1JgoTKErUa9fQWnQBDU5CUWX+8fxnFtlAO6/B4PBzPQyxEaDi1e+jpVgUAghcifBoBDFSFVV1MMHMW3+ATQatAOGonmwcZbZxNk5614Z/5RY3B38yP9aHGWbSXdHy6BLnXwd66TTMPdIFN6GhKKomhBClHkSDApRTFSDAdPG71CPhKHUvA/tY0+ieHrnfSCQqHPilrM3bSNOAPcXbUVLI0WDSaNDiY2Cavk/3MdRS5JBb/96CSFEOSDBoBDFQE2Ix7Tmf6g3rqLp8Ciatp1zHBuYncwu4rq3bxZVFUs9k06PLja6QMd6O2qJuC2POyGEyI48HYUoYubrVzCt/h+kp6Ed9Cya+g3yXcYZ98r4pCXim367CGpYNhh1ehxiChYMvtarHYkGMy8e+xtHrSzJLoQQd5JgUIgiZD5+GNMP34K7B7oho1AqVc53GYk6J666+NEm6lQR1LDsMOn0kBCN1miwjCG01emj/wAQk2aisosEg0IIcSd5KgphR6rpv6kd5kP7MH2/AqVqdXQjXy5QIAhw2r0qKAqBCdftVc0yKTO9jMvt2AKXEZVisld1hBCi3JCWQSHsKHOdYTU2GiLDwcUV1cER04Y1OR6jGzwi1zJPelSjUkos3ob8J1wuTzLTy7gmxpLoVbFAZUSlSjAohBB3k5ZBIexIVVXUqIiMQNDNA6rWyNdEkbvFOXkQ7ux1z7cKwn/pZVwTYwpcRlTqvZaURwgh8iYtg0LYkfnPXyAmCjy8oFLljCXlCuG8b21QVeolSjCoarTg5IxrYbqJpWVQCCGykJZBIezEtG8n5h2/2S0QVIHzfrWpnhyFuzHVLnUs6xRvX1wTCx4MxqebSTepdqyREEKUfdIyKIQdmE8cwbxlPUr9hqgmY6EDQYBbTl7EO3vS8uY/ha9geeHji+uVq/k+bOTIkVyNyggio1ONVHaVBNRCCJFJWgaFKCT15jVM61ehVKuJtv9TdgkEAY561URnMkgX8R0Ubz9cbsehmM35Om7x4sW88dZkQLqKhRDibhIMClEIavJtjKv/B84uaAc9g5LP/Hc5SVe0nHSvxn3Rl3A0y6SHTIq3LxrVjHNyfL6PdcaIVpFgUAgh7ibBoBAFpKoqpvWrISkR3aBnUNw87FImwCmPqhi0OoJT7t3l57Ll4wuQ73GDYWFhnD55Eh9HrcwoFkKIu8iYQSFyoZqMKNrsPybmvTtQz55E07UvSpXqdjmfoijcTDIQVr0mninxGG/d4KbO+vzVgZtJhmyPL+9j4RTv/4LByHzk8G7WrBkAGy4mcC2HayeEEPcqCQaFyEVmEum7qWmpcOUiuLpjPn8a9cIZIO8E0raIdfYk2tWHplcPY5/Rh+WIhycmjRbX2wXLNejrpOVEbBrpJhUHrVxdIYQA6SYWIt9UVYXwG6DR2CWFzN3O+dVGYzZRO/qyXcstDxRFQ7Kbd4HTy/g5aYGMGcVCCCEySDAoRH7FxkBqKlTwR9HZt3E9VaPjgm8tasRew8mUbteyy4skd+8CJ56u4JTx+4qUSSRCCGEhwaAQ+aAaDBAdAa5u4F74CSN3O+pVE6NWR2D4WbuXXV4kuXnjkhgLav6TR3s5atAqEC3BoBBCWEgwKER+RIVn/L+iv927h80oHPK+j4qJkfikxNm17PIkyd0bvTEdh7TkfB+rURR8nbREpEg3sRBCZJJgUAgbqclJkJgAPn4oege7l3/GvTKJehfqh5+xe9nlSZKbNwCuiQWbRFLJWUd4itGSxkcIIe51MptYCBuoqgqRt0Cnh3/Tm9hbmE8dvNJvUzVecgvmJsndBwC3xBhiK9iW0ufgwYOcPHkSgEouOo7GpHHbaMZdry2yegohRFkhwaAQtkiMh7Q08K+KorF/g/o1Zx9uOvvQ6dYRaa7PQ5KbN6qi4JZge8tgcHCw5d+VnDMee+HJJtw9JRgUQgj5uyNEHlSzGaIiwdGpSCaNAOz2q4+LMZWG8VeKpPzyRNVqSXL1KnA3cUXnjAAwXMYNCiEEIC2DQuQtPhaMhiLJKQhw7baBK64V6BB+DL0qs1xtkeThg1tCtM37jxo1iqioKEJDQ3HUavBx1BKeLMGgEEKABINC5Eo1GiEmGpxdUFzdiuQcu24l42JMJSjuUpGUXx7ddvfFJ+JqRnoZGwL0JUuWWP1cyVnLdQkGhRACkG5iIXKlHj4AJiP4+BVJ+TecvLmUaKB5zDlpFcyH2x4+6I3pOKbcLtDxlVx0JKSbSTGa7VwzIYQoeyQYFCIHqtmE6a8/MsYKurjav3xgV4VAnHUKQbGX7F5+eXbnjOKCsEwikXGDQgghwaAQOVFPHIHY6Iy8gkUwVvCSa0WuuFbgoUouOEirYL7c9shI75OfcYN3+m9GsQSDQgghwaAQ2VBVFdOubeBXEdzc7V6+Gdhe4UG80m/TxM/J7uWXd8kunpg02gLPKHbRa3DXawhPkSBcCCEkGBQiG+q5UxB+A+1DHYukVfC4Zw2inDxoG3kSrcb+5Zd7Gg1J7t64JRasZRD+W4lECCHudRIMCpEN867fwcMLpWFTu5dtULT85VefyikxBCTesHv594okdx+bE083bdqU+vXrW22r5KIlOtVEukmWpRNC3NskGBTiLubLF1CvXETTpgOK1v7Zl/b53s9tvTPtI44jbYIFd9vdJ6Ob2IY1hsPCwvjmm2+stvm7yCQSIYQACQaFyMK863dwcUXTtKXdy47Vu3LApy6B8VepllKw8W4iw20PX7RmE85J8QU6vqqLHoDrSQZ7VksIIcocCQaFuIN66zrquVNoWrZD0TvYt2xgW6UGaFSV9hHH7Vr2vShzRrF7QlSBjnfRZ6xEci1JWgaFEPc2WYFEiDuYdm0DB0c0LR7K97FqHt2VF9wqcdHNn/YRx3AzpRW0iuJfiZ4VAHCPjyKiSt1c982cBHT376iqq45zCemoqlokE4WEEKIskGBQiH+pMVGoJw6jad0Bxck538dnBhM3s+l2NGi0/FK7IR4pCVS5epqb/BeUVL/jmMqu+oJV/h6U7uhCuoMTbvEFaxkEqOqq52hMGrFpZnyctHasnRBClB3STSzEv0x//QEaLZrW7exe9uEqDUh2cKHl5TA0yOxVu1AUEj39CtxNDBktgyDjBoUQ9zYJBoUA1MR41MMH0DRujuLmYdeyI119OF2xLgGR56mYVPC8eCKrRI8KuBeiZdDPSYujVuG6jBsUQtzDSkUwuGHDBnr37k2fPn144oknOHr0KACLFi2ia9euPPLII8yfP98y3icmJobnnnuO7t2707NnTw4dOlSS1RflgHnPdjCb0TzU0a7lmhSFfTWDcTGk0Pj6MbuWLeC2px+Oacno05ILdLyiKFR11UnLoBDinlbiweCFCxf4+OOPWbp0KRs2bOD555/npZdeYvv27WzZsoXQ0FA2btzIvn372Lx5MwDTpk2jWbNmbNq0iY8//piXX36ZlJSUEn4noqxSU5IxH9yD0qAxirevXcs+7l+feGdPWlw+hN4srU/2lujpB1Co1sGqrnoiU02kGs32qpYQQpQpJR4MOjg48P7771OxYkUAGjRoQFRUFFu2bKFnz564uLjg6OhIv379+PHHHzEajfz5558MHDgQgMDAQGrVqsXOnTtL8m2IMsy8fxcY0tE+FGLXcuOcPDjuH0jNmCtUTbhl17JFBvsEgxnjBm8kS7AuhLg3lfhs4mrVqlGtWjUgI+3DjBkz6NSpExERETz88MOW/fz9/QkPDyc2Nhaz2YyPj4/ltUqVKnHrlvyxFfmnpqdh3rcTJeABlEqV7VauGdhXMxi92UCzq//YrVxhLdnFE5NWl+ckki+++ILLly9n+1oVFz0KcC3JwH0e9s0tKYQQZUGJB4OZkpOTmTRpErdu3WLp0qW88sorWfbRaDSYzdl35Wi1+UsLcexY9uO3wsLC8lWOyFlZuJZ+F05RJSWZs35VSc6mvsHBwcTHxdlUlqeXFwAGo5Ezle4nys2Xluf2oE1NJq8RaQZjZquU/o5/3/1a7ttyOt72Y/8rw7b98i7bljKy3yfn9xEfb73iSLyrF07RN7NsB0/LPRgcHExwcHCO96Sb4s+Jm6m43jiVZ31F2fhslxVyLe2jvF/H4ODgIi2/VASDN27cYMyYMdSpU4evv/4aJycnKleuTGRkpGWf8PBw/P398fXNGNMVHx+Pp6en5bVKlSrl65wNGjTA0dHRaltYWFiRX/B7RVm4lqrJiHHHzyg17yPwka457pcZ5NkqxdWLo9WDqBJ/kzrx11F0eX/M9Hfsc+e/DUaj1c/Z7ZOf12x5Pb/75XaMLWXktE9O2zM/95mSvSvhHXU9y3awfoDmdk8m3khif3gKDYKa4Kgt8dEzpVpZ+GyXFXIt7UOuY+GV+FMvLi6OIUOG0KVLF2bPno2TkxMAISEh/PjjjyQnJ5Oenk5oaCidO3dGp9PRoUMH1qxZA8CpU6c4f/48LVvafx1ZUb6Z/z4ACfFoHrbfWEGzqrKnVjO0qpmWl8OQNS2KXqKnHy5JcWiN6Tnus3jxYkJDQ3N8vba7HjNw5bbMKhZC3HtKvGVw1apV3Lx5k99++43ffvvNsn358uV06dKFxx9/HIPBQEhICH379gVg6tSpTJ48mZ49e6IoCjNnzsTd3b2E3oEoi1STEfPOrSjVaqLUqWe3cvdHpBDl5kebi/twMaTarVyRswSviiiAe1wkcX5Vs91n9OjRAEyfPj3b16u66tFr4GKCgfs9HbPdRwghyqsSDwaff/55nn/++WxfGzNmDGPGjMmy3c/Pj0WLFhV11UQ5o5qMKNqMWz6jVTAOTa/H7bYmbaSDOztvJlM99hq1Yq7apUyRt3jvjCEinnEROQaDedFpFKq76bmUKC2DQoh7T4kHg0IUF0Wrw7hqGarZDJfOgZMzpv27MB/4K8djdINH2FS2CYXNVZriqFVoceWQdA8Xo2Q3b4w6PR6x4YUqp7a7A78nJBGXZsLLUdYpFkLcO0p8zKAQxS4hDoxG8K1gt1bBvX4BRDh50bW6G065jF0TRUBRSPCqiEdcYYPBjFnU0joohLjXSDAo7imq2QwxUeDkDC6udinzlpMXe30DeCD+KgFeMt6sJMR7VcIzNgL+XbKyIHydtLjrNVxMlGBeCHFvkWBQ3Fvs3CqYrmjZVLkprsY0OoUfLXz9RIEkeFfCIT0Fp5TEApehKAq13DPGDZoLEVQKIURZI8GguGeohnS7twr+WakBMQ5udL95CCezdC+WlATvjOUs7xw3aDLnP6C7z8OBNJPK9SRjgcsQQoiyRiaQiHuGec/2jFZB/6p2aRU87V6ZI161aBF9hhrJBV8bVxRegtd/M4ojqt4PgFaj8N35jFVJ1p6LIz4+3vJzTsyqigJsvJyIn5OWx+tkTWQthBDljbQMinuCmhiPedc2cHNHsUOrYILOmV/9G+OfEstDkbKEWUkzODiR7OpZ6BnFGkXBWadw22BGla5iIcQ9QoJBcU8wbdsMJhP45W/ZwuyYgU1VmqKi0PPGQbRI0FAaZMwojih0OW56DSYV0kzyexVC3BskGBTlnvnyBdR/DqBp1RbFwaHQ5e31DeCaix+dw4/gZUi2Qw2FPcR7V8I9PhKtMevYzYl92vPeUz1sKsdVp6AAt40SDAoh7g0SDIpyTTUaMW38Djy90bTvUujyrjv7sMevPg/EX+WBhGt2qKGwl1jfqmhUFc/YW1leu3j8MFdOHbOpHOkqFkLcayQYFOWaedfvEBWBtkd/FIfC5QBM1jqwsUowHoZkQsKP2KmGwl7ifKsA4BV9o9BluekyuoozZxULIUR5JsGgKLfM169g3rEVpWETNPcHFq4s4OcqwSRrHel9/QCOZgkSSptUF3dSXNzxjrpe6LJc9Rldxafi0gpfMSGEKOUkGBTlkpqehil0JXh4ou3ev9Dl/eUXyGXXinQOP0KltNzTk4iSE+tbBW87tAxqFAUXncLxmDSMkmtQCFHOSZ5BUe6oqorp5+8hJhrt08+jODkXqrwz7pXZ5xdAw7hLNIy/YqdaioJQTcZcc/+ZbtyHedtpBlTJWGf48TqepKQVLBm4h4OGm8kmTsel8aCPU4HKEEKIskCCQVHumPftRD0ShqbDo2hq1SlUWbeSjWyq3JTKKTGEyHJzJU7R6ri6dFGOrzukJuMDRCz7HF9fL24mGaj+3JgCnctZq+DtqOHvqFQJBoUQ5ZoEg6JcMV88i/nXn1DqN0DTrnOhyrqtc+L7Cwk4m9Lpe20/OtVsp1qKomL4d5KQ3mA91i9k0NOkp6fnqyxFUWji58y260lEpBip6CyPSyFE+SRPN1FuqOE3Ma1ZDn4V0fYdjKIUfEhsukbHD9Vakmoy8+S1fbiaZCJBWaBqtBh1evTpqVbbR0+fS3x8/sd6NvRxZPuNJP6JSqVLdTd7VVMIIUoVmUAiygU1Phbjt0vAwQHdU8+hOOavW+/OfHJGRcP6qi2IcPSgby0PKqQl2Lu6oggZ9E7o01PtkiPQWach0NuRYzFppJmkZVgIUT5JMCjKPDUpEeOKLyAtDd2TI1E8vfNdhqIo3EwycD3JQGjFJlxxrUCrSwep4+nAzSSDTf+J0iHd0Rmt2QSG/7qFLxz7h8snCzbmM9jPiXSzyt9RqXnvLIQQZZB0E4syTU1JxrhiMcTHoR06CsW/SoHLMgN7azXninc1ml49zH0xMnO4LEp3/Hf2eHIS6N0BmNS3AwBrz8Xlu7zKrnruc9ezLyKFpn7OOGgVO9VUCCFKB2kZFGWWmp6G6dulEBWO9oln0dS4r8BlmYE9tZpz0bcmja4fIzDirP0qKoqVSavHpNFCiv3WjX64sgspRpWwyBS7lSmEEKWFBIOiTFKNBkyrv0S9fhVt/6Fo6tQrcFkmFH6uEswl35oEXT9Gw1un7FhTUewUhXRHF0hOBjutLVzFVc99Hnr2R6TI2EEhRLkjwaAoc1SzGVPot6gXz6HtMwhNYMMCl5Wm0fF99dac9qhGk2tHaCCBYLmQ7ugMJiNao/3Gcj7s70KKSSUsUsYOCiHKFxkzKMoUVVUxb/4B9eQRNF16Yz5xGPOJwzYdqxs8wurnRJ0TP1RrSZSjB91uHMIn/EJRVFmUgHSHjHGDDun269at4qqnrqcDe8NTaODjiIeD1m5lCyFESZKWQVGmmHduxXxwN5o2HdC2bl/gcm44efNNrfbE6t147No+Hky4asdaipJm0ulBq8MhzX7jBgE6V3XFrKr8fj3JruUKIURJkmBQlBnmQ3sx/7EFpVEwms49ClSGChzxrMGaGg+hNxt56vIOaidF2LeiouQpCri44piajGo22a1YL0ctbfxdOB2XzoWE/K1oIoQQpZV0E4sywXz6GKaN61Dq1kfbe1CBVhdJM5nZUrkpJz2rU+t2BD1uHMTZLPkByy03dzSJ8ahXL/Hh+j+5ffu2XYptUdGZYzFp/Hr1NiMCvdFrJNWMEKJsk5ZBUSqoJmOOr5mvXsS0bgVK5WpoHx+Gos3/WK2bTt4sPx3HKY9qPBR5kn7X9kggWN65uqKioJ4+zn0NGlOzEBON7qTTKDxa3ZW4dDO/X5PuYiFE2Sctg6JUULQ6jKuWZdmupqXC1Uug1aE6OWP6/hvLa3dPCMmOQdHwl18gYT51cDPDwCt/UT0l2p5VF6WUotGS5uiM45kTUKfg40uzU9PdgVYVndkbkUJNdz2B3o52LV8IIYqTBIOi1FIN6XDtCigaqFYDRZe/2/Wasw+/VG5CrIMbQbEX6dQ+GO1RCQTvJalOrjhGR/LlhNEkoeGljz+3W9ltq7hwNcnA5iu38XfR4e0os4uFEGWTdBOLUkk1GuHaZVDNGYGg3iH3/e9ILpyuaPm9YkNW13gYEwqPX/mLR8KP4KiV2/1ek+bkCsCWH9aw84dVdi1bqyj0ruWOokDohQRSJRm1EKKMkpZBUeqoJhNcvwxGI1SrieLolOcxiqJwM8nALfcK7KsZzG1HN+pFnCXo+jH0ZhM3gerAzaTsxwlWdtXb902IUsGs06NUqV7g401mFW0uE0Q8HbT0reXOd+cTWH8xkcfv87DaP6/jhRCiNJBgUJQqqskI169AWhpUrYHi7GLTcWkaHftqNOBchTq4pybyyOk/qXg7qohrK8oCJahZgY/VahS+Ox+f536+TlouJRr4/EQMFZ20KEpGAPh4Hc8Cn1sIIYqLBIOi1FANhoxA0JAOVaqjuLrZdNxF14r86h/EbZ0zgbfO0OjGcXSq/XLLibJN07BpkZ/Dw0GDUVWJTTOjwYyfk8YSEAohRGknwaAoFcyXzsOVC6CqGS2CLq55HpNqNPObfxOOe9XAJy2RLqf/wC8pphhqK8qSO1uXNSYjZm3RPPa8HTSoKsSlm1FRqeAkE0qEEGWDBIOiRKlGA+YdWzHv2gY6XUaLoGPeaTrOufmz9WQcSZ7VaBl1htbRp4lMSiuGGouyzP/aaW7UfLBIylYUBR/HjElKGQGhScYMCiHKBAkGRYlQVTPqiaOYtm2CmCiURsGoSbfzTCidotGzrVJDTnpWp4JOoe/ZHfin5T2mS9zbaj8YhEdcBHVP7uVGjQcylqsrApkBoUaBmDQza88n8Fhtd5x0MpNdCFF6STAoipWakoz56CHMB3dDZDj4VUQ7dDSa+wKyTTptOQ447V6FbZUakqp1oE3kKR7q/BDqPxIIirx9tGE7lQ7vwufoNircvEBklTpFdi5FUfB21KJVFK4mGfjmbDz97/OQPIRCiFJLgkFR5FTVjHrpPOZD+1BPHgWTEfyrou33FMqDjVE0ubeaJOic2erfiAtu/lRKiePxq3uokJaAVvMwOS9iJ4S1c9UDCTp/kPpHdxBZ+b4iax3M5OGgoXM1V364mMjy03H0rOnG/Z6yUokQovSRYFDYnWoyEhwcjJp8G/M/BzCH7YWYKHByRtO0JZomLVAqV8uzHDPwt/d97KoQCEDH8KM0ib0gmdJFgZi1Os4+2IagA1uocOtiRkBYxGq6O/BMPS9+uJjA9xcSaVXRSNvKLjKOUAhRqkgwKOwvLpakpXNxSEvNmB3s7Az+VcDNAzU+FtOfv1jtrqoq+iefs9oW6ejBL/6NueXsTe3b4XS+dRhPY0pxvgtRjgys6wXAutNR1D2xl0b7N/NHz9FFNrP4Tl6OWoYGePHbtdvsjUjh0m0DvWu64yOzjYUQpYQEg8Ju1IR4TH9sRj18EL0KeHmBp3eeK4hk5mO7mWTAqGg4WvkBTvoH4GhM56ELe6kZe41kIPmu4+5cUURWEBG2MGt1/NOqBw/9vpJ6R7ZzsklIsZxXp1HoVsOd2h4ObLlym/+djqVtZVeaVXBCI/kIhRAlTIJBUWiqyYR595+Yd24FswlNy3bEnzuFh4+v7WWoKle9qnCoWiNuO7pRJ+oiTa4dwdGU/fJxQhRUZOU6XK7TmPtP7OZWtQBiKxR8ubr8qu/lSFUXHVuu3mbb9SSOx6TSrYY7/i7yKBZClBx5AolCMV+/gunHtRBxE6V+Q7RdeqF4+6JeOGtzGRGOHvx5LoErddrgmRJPyOnt+N+OLMJai3vdsaaP4Bt+mVZ/rmHHo8+S5GH7F5fCcnfQMuA+D07FpbP12m2+Oh1HswpOtK3sioNWWgmFEMVPgkFRIGp6GuZtmzHv2wXu7mgHPYumfoN8lZGkdeAvv0COetXEKcVI88uHqBt1EQ1qEdVaiAwGR2f2hDxFuy1f0mbbt/zVeQjJbt52P09OSacVRSHQ25Ha7nr+vJHMgchUTsel07GqK/W9HKyWspPE1UKIoibBoMg389mTmH7+HuJj0TRrgyakO4qTs83HJ2kdOeBTh8PetTEpGprGXqBt++ZE7r9QhLUWwlqSuw97Og6mzbaVdNi0hIMP9yOiSl27nkOrUfjufN65MKu4aIlKNbHhUiJbtAq+jhqc/01U/XgdT7vWSQjxH3t82SoPX9gkGBR5Uk1GFK0ONSEO0y8bUE8cAb9KaJ8di6ZGbZvLSdQ5ccCnLke8amFSNNRPuEbr6DP4pN9Gp2tZhO9AiOzF+VXlz27P0XL7Wtps+5brNQI51agDiV4VirUezjoN1VwVEg0qMWkmbiSbcNaa8XaUREpCFCVbv7Dlpjx8YZNgUNhAwbBgJkRHZPzoWwG8vDH/tQ3zX1n31g0eYfVzhKMHf3vX5oRHdVRF4YH4q7SMPou3IakY6i4EjHp/Dikp2acmSnb3YUfX4dx/fDd1Tu6l6pWTxPlU5ma1AMxqAI4pbqQ5uxV5kmpFUfBwUHDTKySkm4lLN3Mj2cSKM3E0r+hMgKeDzDwWQhQJCQZFjlRVRT1zHNPvmyEqHFzdoKI/it4hz2PTtXqOetbgqGcNbrj4ojObaBB/hRYxZ/E0SL5AUXxUk5Ev3no57x3r9UFNCsH09z68Tx3D68j/t3fv0VHUd+PH33PZSzbZTUgIJE24hQq2QUBoLRaQnwWLPxSo/nge0UeRX1X6eKuttadiUdB66lFE9NBy1FL704J4O1C09tRHVFDksaV9KqLcNCQYQGJCQi6b7GVmvr8/ZrOQAMmGYC7k8zpnzmR2Z2a/89nJzGe/3+/MbMb+aDP/G8D0QL8ctOwctKwc6JeNlpgmK5uIDX8+0PrmR6dH1zSyfAYhr0593CEcd/hTaT0hj87oHD+jc3yEvHKPQiHEmSPJoDgpp/RTnLf+gjr4OWT3h68NQssItrlMk+6hNGMg+0rr+HT8v2PrJv1iDfyvio8prv2cNEduEyO6nmaYlK96krhl4THbP+QNuvE/Kd/9KVp+EQMNm7r6RgwrjlFfj1FTjWHvRFctL3Iyg5lMSssknNGPxox+hIP9CGdkofJ098brp1Gjp2samV6D/1MU4tPaGP+qirDlcCNbDjdSmG4yIsvHiEwvWfLMYyG6lFIKBzib6uklGRRJSjmoT3fj/PcmVFkJhDIxZv4b2thvY7/4/06cH6jyhdiXPoB9GXkcSstGaRrpDXFG1ZZTXFtOfqTmrPqHEb3Tmnffw7Yd5l08JeVllG6gBfw0qlY14UqhO7abINpxDCtOaPhwOFTBgMP7SGusT85qvQGXmx4iaUEs04vl8WKZXhzDxNENlK7j6IY7GCYxbxqRtHSi/gwiaRnUZ+aiaxojs3yMzPJxNGrzcXWUPUejvH0wzNsHwwxMMxiR5WNIhoe8gInZyzuyi67R1kUPSikaLIe6WPNgUx93iNqKuKOwE7+FDA28uobP0Ej36GR5DbJ8Blk+Hb/R+/u7KqWIO9BkO8RsiDoK67jtb7a9KsKY/m0/XKGnk2RQoOJx1Ef/xP5gM1R9CaFM9Omz0b91IZp57MkeMd3kC38WB9OyOZSWzRdp2UQN9/1+jTUUf7GLwtovGHP1v1P61Fo00+RwCp8vTw8RX7WFf3weoEPJ4ClpmpvMGSZx3Kvo+10+h/zEo+2UFYej1aiaalTNEbzVR/CE6yAWSwxRVLQBbLvlYMUhFj3h4+L9ctAG5KENzCc0uIiJhUOYlN+PmqjN3qNR9tbGeO+LRt7DPTnnB0wGZXgoSPcwIM0g6NFxFL3+akdxZhm6xkufHcVWEHMUUVsRcxQxWxFzTpxfA3TNHTQg02vgJJaN2A7xVsukGRrZfoO8gEl+wCQvYJLtM3p8v1fbUTTZikbLodE6lvjpgNfQCJgahqbRfEtQBQwL9f5zmCSDfUzzlcEAquIQzj8/wNnxPxBpgvxCjCv/A/vc0VRbUFVvU9kU5suITWXRNGq96YmVKPpH6zm37gBfa6phcGMlDbXHakO0Hv7PLsSZ1twU3Vp+uif5yMTEnKD5GXTzT046P8pBd2x028awLcx4jGA8TrxkL+aencBGlKZRmzWQ6txB2LmD6DdgEGZGiIitiNiKyiabA2ELaGr+RPICJv39Bjl+NznM8Oike3QyTB2focn/bAe1rjlrshzijptMxR2IJTIIQwczkTiYupZIJnQCpjuOouMo1SUJUpPlUBWxqYpYVDbZVEXc/cQ5rparuaYv06vhSZTd1DVMHYxWZbxyWKjFD4yo7XA06nA0ZnM0alMTdaiKWHx0JMI/E88Q8OoaAwMGBekeCtJN8tJMgt3c/9VyFE2JxK/JUkQTAdGBtMT3lGZqmNqpz21nQx9eSQb7msYwkdW/Q9XXYTaFcTSNIzkF7Bv5Dfb1H0ptJIPwx0eTs2vKITvNQ17kKKNqPycvUkN+Uw1+x2qx2oYu3gwhzkqajmPoOIYHC4imQSjdQ1U4TuG11/Putp1kV5aTU1nO4H0fUrR3GwCNgRDVuYVU5w7iSO4gjmYNJKK0RE2PexIurYuzo/rEmkdDA4+uJQdTd6ebaz/c2iD3ZGjoGkbuCP6rvAFdcxMEn6HhNzT8po7f0EhLNBlmePQWSU5vvBdb1E4kUE02lRGLqohNTdRtMnVOcW98TyJ+GmApd7utU91HXx/Eux8eIWBqpJtugn5srLWY9hnud2I2fx86OIpjSWiiZq/RUtTFbOriDvUxh9qYW+bwcYXw6hr9/Qbppvv9eROJauuEry2p3JLFZ2gMSjeJOxCx3drHLxttyhuOnT/6+XS+FnCTw4J0D/3TjA6Vo6NspfgibPF5Q5z99XEOhuPJ78dvaPTz6QQMrc/9SOrVyeCmTZtYtmwZsViMkSNH8utf/5qMjIzuLlaXU+rYQSAcdwhbzrFxzIGaKjIPfEbB/p3kVX2OAVQGc9kx6kJ2FRYT9aQRtJrIjIUZ1lBBVryRzHiY7GgDObF6/HP/L9aH67p7M4Xo2zwepkweD4wHQDk2VHyB83kp6eVlBMpLKdy/053X9EC/bJxQFmVGkK8X5oI/gOX1ETG9NBo+woaPet1LveYhapjE0LESNVtuvzCF5bgnT0fB0ZiNUuA4Dug6KLeJrK3nBRlaomZMd5/LnOHRCXp0Ql6dkNetpewJfRzjjuJIxKayyU34KiMWVU1uQtXMo0OOz01YDoXjmMfVnBnHNZ+eLIFQiQuOHAUXF6QTsd1aqLDl0Gg5hOMqedyuicYJx51TJ5Ap0oGg1411UchLf79Bf79J/zSDkEdH0zp/f71UaJqG16DFoxYd5SaGI7J8HAxblNbH+KTG/aGia5DjMxiQZpKbZpDrN8n0uj8uOpKg2UrREHeS32vlcd9vc9Nvrt9gbH8/Bxri+M2OJcNnm16bDFZXV7Nw4ULWrl3L0KFDWbp0KY8++ihLlizp7qKdEUod128h7o7DiaExceBoiDcfSI4dOEwrRk59Fbn1lRQcKae4aj/BpjoA6jNz2T/uYupt0D1ehlsRxpVvJVxbi6FOPPI4QCUwCFo1dbUkff6E+Oqdqik6KTgAPa0f3lgETyyCUd+A3zAoOvo5zl63yVgHAomhf+vlDRN8PvC6g+bzQXoGWnoQMoL8T5NJ1J/BERuMAflE0oIoXUcpt1+Vo44ljrZym9+sREJpOYqdNVEirXve4zbFhTw6Qa+bpIS8bsIY9BjJ5lS/qXWqKVUpt/mvLlFT5o4daqJus2lN9FjSZ2iQ4zcozPCQ6zfof1xC0pyIdDSJal7O0GBgwJNcvra2lszMYzcsTjM10kyDHJ/7UM7mZLw5vgqFUlCc7UfhJp9eXcPTXLuna/hNjZDXrVHsqf3zdE0jzdSYMDAAuN9PbczhYDjOl4la2M8b4skEsZmpkax19uoahq6hAzVaLiUltcQStY/u+bLlvhb06PT3G4zP9fK1dJPBGR4Ciaf8dEVS3NP12mRwy5YtnHfeeQwdOhSAq6++mtmzZ7N48eIeXbWrlALb4ou6CFUNEaLROJFonFjMHeLxOPFYHDsed/sOKQfdcdyxctAcB7/mkKdBQMVxwg0Eoo2kxRoJNdSQET6avHo36kujauBQSvImUZk3jHAoh38bntnihNIv3UPkJImgEKL3cUwPEdNDJODeBmrQjf9J+aonyUszwHHASVys0vy347QYtKJzIJ64kCUaQVVXoT4vhcYwY0/4NA0ygqiMEF940omkhWgKBIkE3HHc48fyeLFND7bHy+Xn5KB0g/qYQ33cTcjq405ibFMbtSlviBM9ScII7gUJftNtyvbqbo2cJ1Erp3Dv4NM8thM1TxFbEbFsnGgETzyGNx7FZ0XxWlHSrBjZWAw1IcPUCJoaGV6dgN+LbnkhYoLlgYgHvF7w+VFen5swn+btglKlaW4zs9sV7cTP+W5e4Kx6aoaWuLdmls+g+LjXI5ZDZcSmPubQkKgACcfdcdRWWJbbXK8HgjRaCq/uNvN+Ld1M1EIbZPsNcv1G8vGO4uR6bTJ4+PBh8vLyktN5eXk0NDQQDofbbCpurq6PxWInfT8aPbFPTUepeAzn1RdR9XWJKwUtsGxwLLDdX6CZiaHTn+Xx0OQNEPP5qc4fSnkwh4ZQNg3BbBqD/Tj+QGLacaLRKLb32K0yYqYH29v2Qa31Mq21Xoc7vw/dTK1Tbcz0YLfzGe2Vq7Pbcarl29v2E9ZxhrfD1o0T4ni620F+vrvOdsp3JrYjPz+a0medalvO2Pfh1YhGo+SnuO2tPyOV/aqtcp3p7Tjd7yTmMVP6kWxcMgv71ZdANyEtwx3656GUoiocRbctlBXHi0K3bXTHIpDVD19NLaHaSrzxUx8/o4Cj6xiaTqamE9J0lK6hNN2tYdR0HDT3djta86ChdA0HHa9pYGsajqZjJ+a1cWvQDMfCtC0MJ45h2Zh2HI8dx7BieKyO3d80lTPADE2D9AysH/wHbzd0/DsxbbdMXmUn/z7ddZyuzq7jTJShsSnSZj9SDRjgcQe3XvvkCd2LHx8iGAy6vwYsiFgQiUAVUNpOGWYMCZ6RWHYFr9f7lVV2aUr1zmqhJ598kkOHDvHAAw8AYFkWxcXF/Otf/yIQCJxyufr6evbu3dtVxRRCCCGE6LRRo0bh8/m+knX32prB/Px8tm/fnpyuqKggMzOzzUQQID09nREjRuDxeHp0c7IQQgghRDNvB1sGOqLXJoOTJk3i4YcfpqysjKFDh/LCCy8wderUdpfTdd2tThZCCCGEEL23mRhg8+bNLFu2jHg8zuDBg3n44YfJysrq7mIJIYQQQvQavToZFEIIIYQQnSPXWgshhBBC9GGSDAohhBBC9GGSDAohhBBC9GGSDAohhBBC9GGSDAohhBBC9GF9OhnctGkTM2fOZPr06fz4xz+moaHhpPPt2bOH6667jh/84AdceeWVfPzxx11c0p4v1Vg227hxI+PGjeui0vUeqcZxw4YNzJo1i9mzZzN37lx27NjRxSXtmVKJX0f31b4olRjJPpiajuxvclw8tVTiKOfqTlB91JEjR9SECRNUaWmpUkqpRx55RC1evPiE+RobG9XEiRPVpk2blFJKvfnmm2r69OldWNKeL9VYNistLVXTpk1TY8eO7ZoC9hKpxrGkpERNnDhRVVRUKKWU2rRpk5oyZUrXFbSHSiV+Hd1X+6JUYiT7YGo6sr/JcfHUUomjnKs7p88mgxs2bFA33XRTcrq8vFyNGzdOOY7TYr4333xTzZ07NzntOI7atWtXl5WzN0g1lkq5/7Bz5sxRGzdulINeK6nGsby8XL3zzjvJ6aqqKlVcXKyi0WhXFbVHSiV+HdlX+6pUYiT7YGpS3d/kuNi2VOIo5+rO6bWPo0vV5s2bufnmm094/ZZbbiEvLy85nZeXR0NDA+FwmIyMjOTrpaWl5Obmcs8997B7925CoRA///nPu6TsPU1nYwlw3333cdVVVzFy5MivvLw9VWfjWFhYSGFhIQBKKR566CG+973vfaXPrewNDh8+3G78Upmnr0slRrIPpibV/U2Oi21LJY5yru6csz4ZnDJlCjt37jzh9SeffPKk8+t6y26UlmWxefNmnnvuOcaMGcPGjRtZsGAB77zzTp878HU2lmvWrME0TebMmcOBAwe+kjL2Bp2NY7PGxkbuvvtuDh8+zKpVq85oGXsjx3FO+vrx8Utlnr6uIzGSfbBtqcRSjovtSyWOcq7unD57BMzPz6eysjI5XVFRQWZmJoFAoMV8AwYMoKioiDFjxgAwbdo0bNumvLy8S8vbk6Uay/Xr17Njxw5mz57NggULiEQizJ49m4qKiq4uco+UahwBDh06xNy5czEMg+eee45QKNSVRe2RUolfR2LcV6UaI9kH25dKLOW42L5U4ijn6s7ps8ngpEmT2L59O2VlZQC88MILTJ069YT5LrroIg4ePJi8Kmnbtm1ompZsIhGpx/KVV17hz3/+Mxs2bODpp5/G7/ezYcMGBg4c2MUl7plSjePRo0e59tpr+f73v8/y5cvx+/1dXNKeKZX4pRrjviyVGMk+mJpUYinHxfalEkc5V3dSd3da7E6bNm1SM2fOVJdeeqlasGCBqqmpUUop9dFHH6lZs2Yl5/v73/+u5syZoy677DJ1xRVXqG3btnVTiXuuVGPZrLy8XDpKn0QqcVy5cqU699xz1axZs1oM1dXV3VjynuFk8Wu9D54qxuKY9uIo+2DqUtknm8lx8dRSiaOcq0+fppRS3Z2QCiGEEEKI7tFnm4mFEEIIIYQkg0IIIYQQfZokg0IIIYQQfZgkg0IIIYQQfZgkg0IIIYQQfZgkg0KIbmdZFitXruSSSy5h1KhRTJ48mXvvvZcjR44k57nuuutYvnw5AHfffTd33XUXACtWrODqq6/ulnIrpVi7dm3yCQmxWIwXXnjhtNeXShyEEOJMk2RQCNHtli1bxuuvv86SJUt44403WL58OXv37uWmm26i+e5XK1asYMGCBd1c0pa2bdvGkiVLksng66+/zsqVK097fanEQQghzjRJBoUQ3W7dunXcfvvtTJw4kYKCAr71rW/x6KOP8sknn7B9+3YAsrKySE9P7+aSttQ6QetswpZKHIQQ4kyTZFAI0SN88MEH2LadnB40aBB/+ctfOPfcc4GWzcStWZbFgw8+yPjx47nwwgtZtWpV8j3HcVi1ahXTpk1j9OjRXHvttezevTv5/siRI9m6dWtyet26dVx00UXJ6U8//ZR58+YxevRoLrnkEp555hmUUhw4cIB58+YBUFxczN/+9jcWLlxIRUUFI0eO5MCBAyilWLlyJZMnT2b8+PHccMMNyUdqnW4cbNvmiSeeYPLkyYwbN46bb76ZL7/8MuVtffzxx5kwYQLz588H4B//+Adz5sxh9OjRXHbZZfzpT39qs3xCiLOPJINCiG43b9481q5dy8UXX8yiRYt4/fXXqaurY/jw4Sk99/ajjz4CYP369fzoRz9i6dKl7NmzB4Df/va3PPPMMyxcuJD169dTWFjIjTfeSENDQ7vrjUQi3HjjjYwdO5ZXX32VRYsW8eyzz7J69Wry8/NZsWIFAO+++y7nn38+99xzD7m5uWzZsoX8/HxWr17Nhg0beOSRR3jppZcYMmQI119/PU1NTacdhxUrVvDyyy/z4IMP8vLLLxONRvnFL36R8ra+9dZbPP/88/zyl7+ksrKSBQsWMHPmTF577TVuvfVWHnzwQd5+++12YyOEOHtIMiiE6Ha33nory5cvZ/Dgwaxbt44777yTSZMmtajha0tubi733HMPgwcPZv78+YRCIfbs2YNSitWrV3PbbbcxdepUhg8fzq9+9StM02TDhg3trve1114jMzOTO++8k6FDhzJlyhR+8pOf8Oyzz2IYBpmZmQDk5OTg9XoJBoPouk5ubi6GYbBq1SruuusuLrzwQoYPH869996LaZq88cYbpxUHpRQvvvgid9xxB1OmTGH48OEsWbKE8847D9u2U9rWq666iqKiIs455xzWrFnDd77zHa6//nqGDBnCjBkzmD9/Ps8++2xKcRdCnB3M7i6AEEIAzJgxgxkzZlBXV8fWrVt58cUXWbp0KcOGDWPq1KltLltQUICuH/ttGwwGiUajHDlyhKNHjzJmzJjkex6Ph1GjRlFSUtJumfbt28dnn33G+eefn3zNcRxisRixWKzNZcPhMIcPH+auu+5qUbZoNNpmU3FbcTj//POprq6muLg4Of/gwYO58847qaqqSmlbCwoKWmzfe++912L7LMsiOzu77cAIIc4qkgwKIbrV7t27eeWVV1i0aBEAoVCISy+9lOnTpzNnzhzef//9dpPB45Ot452qidm27Rb98lq/18yyLC644ALuv//+E+YzzbYPn83reeyxx/j617/e4r1gMHjC/KnE4YILLjjl56W6rT6fL/m3ZVlcdtll3HLLLS2WOVU8hRBnJ/mPF0J0K9u2+eMf/8iHH37Y4nVN0wgGg52qpcrIyCA3N7fFlbjxeJxPPvmEYcOGAW7tWTgcTr5fXl6e/HvYsGGUlZVRUFDAkCFDGDJkCLt27eJ3v/sduq6jadoJZW4WCoXIycmhsrIyuWxhYSGPPfZYsj9jR+PQPN65c2fy/bKyMr773e9iWVa729rasGHD2L9/f7J8Q4YMYcuWLbzyyitthVUIcZaRZFAI0a2Ki4u5+OKLue2221i/fj3l5eXs2LGD5cuXs2vXLubMmdOp9f/whz/kN7/5DW+99RYlJSXcd999RKNRLr/8cgDOO+881qxZQ1lZGe+88w7r1q1LLjtr1ixisRiLFi2ipKSE999/nwceeCDZVzAQCACwc+dOotEogUCA+vp6SktLsSyL+fPn88QTT7Bx40b279/P/fffz9atWykqKjrtOMybN48VK1bw/vvvU1JSwgMPPMA3v/lNsrKy2t3W1q655hp27tzJsmXLKCsr469//StLly5l4MCBnYq5EKJ3kWZiIUS3e/zxx3n66ad56qmnWLx4MV6vl29/+9usWbOGvLy8Tq17/vz5NDQ0sHjxYurr6xk7dizPPfcc/fv3B+Dee+9l0aJFXH755YwaNYo77rgjeZVwRkYGq1at4qGHHuKKK64gFApxxRVX8NOf/hSAESNGMGnSJK655hoee+wxJkyYQFFREbNmzeL555/nhhtuoKmpifvvv5+6ujq+8Y1v8Pvf//6UyVYqcbjpppuoq6vjZz/7GfF4nEmTJnHfffeltK2tFRQU8NRTT/Hoo4/yhz/8gdzcXG6//XauueaaTsVcCNG7aEpuay+EEEII0WdJM7EQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB8myaAQQgghRB/2/wFZ/GhKuV1a4wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Example data: Two sets of values\n",
    "values_set_1 = embeddings_cluster_silhouette_values  # Replace with your actual data\n",
    "values_set_2 = embeddings_cluster_context_silhouette_values  # Replace with your second set of values\n",
    "\n",
    "mean_set_1 = np.mean(values_set_1)\n",
    "mean_set_2 = np.mean(values_set_2)\n",
    "# Set the style of the visualization\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Create the figure and axis\n",
    "plt.figure(figsize=(10, 6))\n",
    "\n",
    "# Plot the first set of values\n",
    "sns.histplot(values_set_1, color='skyblue', kde=True, label='w.o. context', bins=30, alpha=0.7)\n",
    "\n",
    "# Plot the second set of values\n",
    "sns.histplot(values_set_2, color='salmon', kde=True, label='with context', bins=30, alpha=0.7)\n",
    "\n",
    "# Add a vertical line at x=0 for reference\n",
    "plt.axvline(x=0, color='black', linestyle='--', linewidth=2)\n",
    "\n",
    "plt.axvline(x=mean_set_1, color='blue', linestyle='-', linewidth=2, label=f'Mean  ({mean_set_1:.2f})')\n",
    "plt.axvline(x=mean_set_2, color='red', linestyle='-', linewidth=2, label=f'Set 2 Mean ({mean_set_2:.2f})')\n",
    "\n",
    "# Customize the plot\n",
    "plt.title('Comparison of Silhouette Score between column embeddings with and w.o. context', fontsize=16)\n",
    "plt.xlabel('Silhouette Score', fontsize=14)\n",
    "plt.ylabel('Frequency', fontsize=14)\n",
    "plt.legend()\n",
    "plt.xticks(fontsize=12)\n",
    "plt.yticks(fontsize=12)\n",
    "\n",
    "# Remove the top and right spines for a cleaner look\n",
    "sns.despine()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/bert-base-uncased-fromscratch-semi1-poolv0-unlabeled8-randTrue-bs16-ml64-ne50-do0.1_eval.json\", 'r') as f:\n",
    "    results = json.load(f)\n",
    "# bert-base-uncased-fromscratch-semi1-poolv0-unlabeled8-randTrue-bs16-ml64-ne50-do0.1_best_f1_micro.pt\n",
    "# labels_train = np.array(results['train']['tr_true_list'])\n",
    "# preds_train = np.array(results['train']['tr_pred_list'])\n",
    "# class_f1_train = np.array(results['train']['tr_class_f1'])\n",
    "\n",
    "labels = np.array(results['f1_macro']['true_list'])\n",
    "preds = np.array(results['f1_macro']['pred_list'])\n",
    "class_f1 = np.array(results['f1_macro']['ts_class_f1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/307592194.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  best_state_dict = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/bert-base-uncased-fromscratch-semi1-Repeat@5-AttnMask-UnlabelValid-max-unlabeled@8-poolv0-unlabeled8-randFalse-bs16-ml128-ne50-do0.1_best_f1_macro.pt\", map_location=device)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_state_dict = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/bert-base-uncased-fromscratch-semi1-AttnMask-max-unlabeled@8-poolv0-unlabeled8-randFalse-bs16-ml128-ne50-do0.1_best_f1_micro.pt\", map_location=device)\n",
    "best_state_dict = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/bert-base-uncased-fromscratch-semi1-Repeat@5-AttnMask-UnlabelValid-max-unlabeled@8-poolv0-unlabeled8-randFalse-bs16-ml128-ne50-do0.1_best_f1_macro.pt\", map_location=device)\n",
    "model.load_state_dict(best_state_dict, strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_train = []\n",
    "labels_train = []\n",
    "for batch_idx, batch in enumerate(train_dataloader):\n",
    "    cls_indexes = torch.nonzero(\n",
    "                    batch[\"data\"].T == tokenizer.cls_token_id)\n",
    "    embs = model.bert(batch[\"data\"].T)\n",
    "    embs = extract_cls_tokens(embs[0], cls_indexes)\n",
    "    ft_embs_train.append(embs.detach().cpu())\n",
    "    labels_train.append(batch[\"label\"].cpu())\n",
    "ft_embs_train = torch.cat(ft_embs_train, dim=0)\n",
    "labels_train = torch.cat(labels_train, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataloader = DataLoader(test_dataset,\n",
    "                                batch_size=16,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test_origin = []\n",
    "logits_test_origin_16 = []\n",
    "cls_indexes_all_16 = []\n",
    "for batch_idx, batch in enumerate(test_dataloader):\n",
    "    cls_indexes = torch.nonzero(\n",
    "                    batch[\"data\"].T == tokenizer.cls_token_id)\n",
    "    # embs = model.bert(batch[\"data\"].T)\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,).detach().cpu()\n",
    "    # embs = extract_cls_tokens(embs[0], cls_indexes)\n",
    "    # ft_embs_test.append(embs.detach().cpu())\n",
    "    labels_test_origin.append(batch[\"label\"].cpu())\n",
    "    if logits.dim() == 1:\n",
    "        logits = logits.unsqueeze(0)\n",
    "    logits_test_origin_16.append(logits)\n",
    "    cls_indexes_all_16.append(cls_indexes.detach().cpu())\n",
    "# ft_embs_test = torch.cat(ft_embs_test, dim=0)\n",
    "\n",
    "labels_test_origin = torch.cat(labels_test_origin, dim=0)\n",
    "logits_test_origin_16 = torch.cat(logits_test_origin_16, dim=0)\n",
    "preds_test_origin_16 = torch.argmax(logits_test_origin_16, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.2349e+01,  2.2623e+00,  2.0195e+00,  4.7039e-01,  1.3064e+00,\n",
       "         1.2641e+00,  8.9482e-01,  2.4156e+00,  2.7143e+00, -5.1789e-01,\n",
       "         1.7351e+00,  1.0563e+00,  5.2661e-01,  2.3156e+00,  3.5534e-01,\n",
       "         4.9356e-03,  4.0803e-01, -1.2673e-01, -1.5701e+00,  4.4213e-01,\n",
       "         1.2484e+00, -6.4937e-02, -2.9976e-01, -1.0279e+00,  4.1979e-01,\n",
       "        -1.6945e+00,  1.1877e-01, -3.9443e-01, -2.4848e+00, -1.3368e+00,\n",
       "        -1.0973e+00, -1.7170e-01, -1.3608e+00,  1.0209e+00,  1.2189e+00,\n",
       "        -1.0656e+00, -4.0851e-01,  2.1805e+00, -1.4573e+00,  3.6495e-02,\n",
       "         1.2361e+00, -1.9280e+00, -1.0239e-01, -1.9924e+00, -1.8749e+00,\n",
       "         5.8734e-01, -3.8315e-01, -1.6866e+00,  8.4979e-02, -7.3993e-01,\n",
       "        -3.0850e+00, -2.9878e+00, -2.6023e+00, -2.5192e+00, -2.8550e+00,\n",
       "        -1.0333e+00,  1.0281e+00, -7.8085e-01, -8.5541e-01,  3.0408e-01,\n",
       "         1.1371e+00, -1.7338e+00, -2.1937e+00, -6.1994e-01, -1.8035e+00,\n",
       "        -3.0026e+00, -2.4853e+00, -1.7369e+00, -1.8136e+00, -1.0013e+00,\n",
       "        -1.6233e+00, -6.4445e-01, -3.5500e-01, -7.8222e-01, -2.6241e+00,\n",
       "        -1.8708e+00, -1.1988e+00, -7.2830e-01, -1.9388e+00, -1.0569e+00,\n",
       "        -2.4025e+00, -2.7821e+00, -1.1982e+00, -2.1431e+00,  1.5204e-01,\n",
       "        -2.4471e+00, -1.5189e+00, -3.1720e+00, -1.3798e+00, -1.6331e+00,\n",
       "        -2.2853e+00, -1.8026e+00, -2.6446e+00, -2.2680e+00, -2.5178e+00,\n",
       "        -2.3193e+00, -7.0378e-01, -1.0620e+00, -1.5850e+00,  8.6639e-01,\n",
       "        -1.9128e+00])"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_test_origin_16[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.2314e+01,  2.2582e+00,  2.0340e+00,  4.8279e-01,  1.2961e+00,\n",
       "         1.2751e+00,  9.4323e-01,  2.4833e+00,  2.7057e+00, -5.0379e-01,\n",
       "         1.7371e+00,  1.1412e+00,  5.4223e-01,  2.3659e+00,  3.5382e-01,\n",
       "        -3.1089e-03,  4.4818e-01, -1.2872e-01, -1.5742e+00,  4.2284e-01,\n",
       "         1.2316e+00, -8.0508e-02, -2.9698e-01, -1.0158e+00,  4.0950e-01,\n",
       "        -1.7014e+00,  1.4729e-01, -4.0749e-01, -2.4696e+00, -1.2981e+00,\n",
       "        -1.0998e+00, -1.8593e-01, -1.3521e+00,  1.0008e+00,  1.2162e+00,\n",
       "        -1.0663e+00, -4.2975e-01,  2.2044e+00, -1.4563e+00,  6.6450e-02,\n",
       "         1.1826e+00, -1.9540e+00, -1.3295e-01, -2.0091e+00, -1.8851e+00,\n",
       "         6.2226e-01, -3.7785e-01, -1.6972e+00,  7.0330e-02, -7.5263e-01,\n",
       "        -3.0694e+00, -2.9786e+00, -2.5552e+00, -2.5303e+00, -2.8545e+00,\n",
       "        -1.0462e+00,  1.0155e+00, -7.7138e-01, -8.7357e-01,  3.1614e-01,\n",
       "         1.0917e+00, -1.7528e+00, -2.2213e+00, -6.6158e-01, -1.8124e+00,\n",
       "        -2.9912e+00, -2.5006e+00, -1.7283e+00, -1.8156e+00, -1.0231e+00,\n",
       "        -1.6513e+00, -6.9392e-01, -3.7237e-01, -7.9804e-01, -2.6371e+00,\n",
       "        -1.8794e+00, -1.2159e+00, -7.2817e-01, -1.9247e+00, -1.0613e+00,\n",
       "        -2.4169e+00, -2.7907e+00, -1.1989e+00, -2.1343e+00,  1.2457e-01,\n",
       "        -2.4674e+00, -1.5457e+00, -3.2085e+00, -1.4093e+00, -1.6608e+00,\n",
       "        -2.2757e+00, -1.7809e+00, -2.6700e+00, -2.2951e+00, -2.4974e+00,\n",
       "        -2.3410e+00, -7.4908e-01, -1.0551e+00, -1.5980e+00,  8.9172e-01,\n",
       "        -1.9220e+00])"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_test_origin_1[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_test_origin_16.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_test_origin_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_indexes_all_16 = torch.cat(cls_indexes_all_16, dim = 0)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_indexes_all_16.equal(cls_indexes_all_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_indexes_all_1  = torch.cat(cls_indexes_all_1, dim = 0)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  0,   0],\n",
       "        [  1,   0],\n",
       "        [  2,   0],\n",
       "        [  2,  64],\n",
       "        [  3,   0],\n",
       "        [  4,   0],\n",
       "        [  5,   0],\n",
       "        [  5,  14],\n",
       "        [  6,   0],\n",
       "        [  7,   0],\n",
       "        [  8,   0],\n",
       "        [  8,  64],\n",
       "        [  9,   0],\n",
       "        [  9,  52],\n",
       "        [ 10,   0],\n",
       "        [ 11,   0],\n",
       "        [ 12,   0],\n",
       "        [ 12,  64],\n",
       "        [ 12, 128],\n",
       "        [ 12, 192],\n",
       "        [ 13,   0]], device='cuda:1')"
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataloader = DataLoader(test_dataset,\n",
    "                                batch_size=16,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test_origin = []\n",
    "logits_test_origin_1 = []\n",
    "cls_indexes_all_1 = []\n",
    "for batch_idx, batch in enumerate(test_dataloader):\n",
    "    cls_indexes = torch.nonzero(\n",
    "                    batch[\"data\"].T == tokenizer.cls_token_id)\n",
    "    \n",
    "    # embs = model.bert(batch[\"data\"].T)\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,).detach().cpu()\n",
    "    # embs = extract_cls_tokens(embs[0], cls_indexes)\n",
    "    # ft_embs_test.append(embs.detach().cpu())\n",
    "    labels_test_origin.append(batch[\"label\"].cpu())\n",
    "    if logits.dim() == 1:\n",
    "        logits = logits.unsqueeze(0)\n",
    "    logits_test_origin_1.append(logits)\n",
    "    cls_indexes_all_1.append(cls_indexes.detach().cpu())\n",
    "# ft_embs_test = torch.cat(ft_embs_test, dim=0)\n",
    "\n",
    "labels_test_origin = torch.cat(labels_test_origin, dim=0)\n",
    "logits_test_origin_1 = torch.cat(logits_test_origin_1, dim=0)\n",
    "preds_test_origin_1 = torch.argmax(logits_test_origin_1, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_equal = []\n",
    "for i in range(len(logits_test_origin_16)):\n",
    "    is_equal.append(torch.equal(logits_test_origin_16[i], logits_test_origin_1[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "True in is_equal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085, 101])"
      ]
     },
     "execution_count": 314,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_test_origin_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4507, ts_macro_f1=0.1865\n"
     ]
    }
   ],
   "source": [
    "ts_pred_list_origin = logits_test_origin_1.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "ts_micro_f1 = f1_score(labels_test_origin.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list_origin,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test_origin.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list_origin,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/1939110723.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test_origin_1 == labels_test_origin).float()\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "ood_score = F.softmax(logits_test_origin_1, dim=1).max(1).values\n",
    "ood_labels = torch.tensor(preds_test_origin_1 == labels_test_origin).float()\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(ood_labels, ood_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7740097583069132"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_entropy(probs):\n",
    "    # Adding a small value (1e-9) to avoid log(0)\n",
    "    if probs.dim() == 1:\n",
    "        return -torch.sum(probs * torch.log(probs))\n",
    "    else:\n",
    "        return -torch.sum(probs * torch.log(probs), dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4129975340.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test_origin_1 != labels_test_origin).float()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7723833738213859"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "ood_score = compute_entropy(F.softmax(logits_test_origin_1, dim=1))\n",
    "ood_labels = torch.tensor(preds_test_origin_1 != labels_test_origin).float()\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(ood_labels, ood_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD8CAYAAACYebj1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAASw0lEQVR4nO3df7DldX3f8ecLFtCJxhW53W72RxfHHVObRiUbgpJmDEw6SlKWpATIpLIy2M0kxOrQMcF0prad/pFMO1GxGXRHTJbECJRoWSmaEkCdTCJxUYIKGjaMdO8K7Iq6xtIkXX33j/PZr8fL3d1zL+d7ztl7n4+ZM+f7/Xw/5/t93+9y7ovP53u+56aqkCQJ4JRpFyBJmh2GgiSpYyhIkjqGgiSpYyhIkjqGgiSp02soJFmb5LYkX0zycJJXJTkzyV1JHmnPL2x9k+T6JPuSPJjknD5rkyQ9U98jhXcBH6uqHwReDjwMXAfcXVVbgbvbOsDrgK3tsRO4oefaJEkLpK+b15K8AHgAeHENHSTJl4DXVNXjSdYDH6+qlyZ5b1v+4MJ+vRQoSXqGNT3u+2zgEPC7SV4O3A+8GVg39Iv+CWBdW94A7B96/XxrO2YonHXWWbVly5Yxly1JK9v999//1aqaW2xbn6GwBjgHeFNV3ZfkXXx3qgiAqqokSxqqJNnJYHqJzZs3s3fv3nHVK0mrQpLHjrWtz2sK88B8Vd3X1m9jEBJPtmkj2vPBtv0AsGno9Rtb2/eoql1Vta2qts3NLRp0kqRl6i0UquoJYH+Sl7amC4GHgD3Ajta2A7i9Le8BrmyfQjoPOOz1BEmarD6njwDeBHwgyenAo8BVDILo1iRXA48Bl7W+dwIXAfuAp1tfSdIE9RoKVfUAsG2RTRcu0reAa/qsR5J0fN7RLEnqGAqSpI6hIEnqGAqSpI6hIEnqrNpQ2LBpM0nG8tiwafO0fxxJGou+71OYWV+Z38/l7/2zsezrll969Vj2I0nTtmpHCpKkZzIUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEmdXkMhyZeTfC7JA0n2trYzk9yV5JH2/MLWniTXJ9mX5MEk5/RZmyTpmSYxUvjJqnpFVW1r69cBd1fVVuDutg7wOmBre+wEbphAbZKkIdOYPtoO7G7Lu4FLhtpvqoFPAWuTrJ9CfZK0avUdCgX8ryT3J9nZ2tZV1eNt+QlgXVveAOwfeu18a5MkTcianvf/41V1IMk/AO5K8sXhjVVVSWopO2zhshNg8+bN46tUktTvSKGqDrTng8CHgXOBJ49OC7Xng637AWDT0Ms3traF+9xVVduqatvc3Fyf5UvSqtNbKCT5viTPP7oM/HPg88AeYEfrtgO4vS3vAa5sn0I6Dzg8NM0kSZqAPqeP1gEfTnL0OH9YVR9L8mng1iRXA48Bl7X+dwIXAfuAp4GreqxNkrSI3kKhqh4FXr5I+1PAhYu0F3BNX/VIkk7MO5olSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLU6T0Ukpya5LNJ7mjrZye5L8m+JLckOb21n9HW97XtW/quTZL0vSYxUngz8PDQ+m8B76iqlwBfB65u7VcDX2/t72j9JEkT1GsoJNkI/DTwvrYe4ALgttZlN3BJW97e1mnbL2z9JUkT0vdI4Z3ArwHfaesvAr5RVUfa+jywoS1vAPYDtO2HW//vkWRnkr1J9h46dKjH0iVp9ektFJL8DHCwqu4f536raldVbauqbXNzc+PctSStemt63Pf5wMVJLgKeA3w/8C5gbZI1bTSwETjQ+h8ANgHzSdYALwCe6rE+SdICvY0UquptVbWxqrYAVwD3VNUvAvcCl7ZuO4Db2/Ketk7bfk9VVV/1SZKeaRr3Kfw6cG2SfQyuGdzY2m8EXtTarwWum0JtkrSq9Tl91KmqjwMfb8uPAucu0udvgZ+fRD2SpMV5R7MkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqTNSKCQ5f5Q2SdLJbdSRwrtHbJMkncSO+zeak7wKeDUwl+TaoU3fD5zaZ2GSpMk7bigApwPPa/2eP9T+TeDSvoqSJE3HcUOhqj4BfCLJ71XVYxOqSZI0JScaKRx1RpJdwJbh11TVBX0UJUmajlFD4b8D7wHeB3y7v3IkSdM0aigcqaobeq1EkjR1o34k9SNJfiXJ+iRnHn30WpkkaeJGHSnsaM9vHWor4MXjLUeSNE0jhUJVnb3UHSd5DvBJ4Ix2nNuq6u1JzgZuBl4E3A+8vqr+PskZwE3AjwBPAZdX1ZeXelxJ0vKNFApJrlysvapuOs7L/g64oKq+leQ04E+TfBS4FnhHVd2c5D3A1cAN7fnrVfWSJFcAvwVcvoSfRZL0LI16TeFHhx7/DPgPwMXHe0ENfKutntYeBVwA3NbadwOXtOXtbZ22/cIkGbE+SdIYjDp99Kbh9SRrGUwBHVeSUxlMEb0E+B3gr4FvVNWR1mUe2NCWNwD72/GOJDnMYIrpqwv2uRPYCbB58+ZRypckjWi5X539f4ATXmeoqm9X1SuAjcC5wA8u83jD+9xVVduqatvc3Nyz3Z0kacio1xQ+wmDqBwZfhPePgVtHPUhVfSPJvcCrgLVJ1rTRwkbgQOt2ANgEzCdZA7yAwQVnSdKEjPqR1P86tHwEeKyq5o/3giRzwP9rgfBc4KcYXDy+l8GX6d3M4KOut7eX7Gnrf96231NV9YwdS5J6M+o1hU8kWcfgQjPAIyO8bD2wu11XOAW4taruSPIQcHOS/wx8Frix9b8R+P0k+4CvAVcs4eeQJI3BqNNHlwH/Bfg4EODdSd5aVbcd6zVV9SDwykXaH2VwfWFh+98CPz9a2ZKkPow6ffTvgB+tqoPQTQ39Cd/9aKkkaQUY9dNHpxwNhOapJbxWknSSGHWk8LEkfwx8sK1fDtzZT0mSpGk50d9ofgmwrqremuTngB9vm/4c+EDfxUmSJutEI4V3Am8DqKoPAR8CSPJP27Z/0WNtkqQJO9F1gXVV9bmFja1tSy8VSZKm5kShsPY42547xjokSTPgRKGwN8m/XtiY5I0MvuhOkrSCnOiawluADyf5Rb4bAtuA04Gf7bEuSdIUHDcUqupJ4NVJfhL4odb8P6vqnt4rkyRN3KjffXQvgy+ykyStYN6VLEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpE5voZBkU5J7kzyU5AtJ3tzaz0xyV5JH2vMLW3uSXJ9kX5IHk5zTV22SpMX1OVI4AvzbqnoZcB5wTZKXAdcBd1fVVuDutg7wOmBre+wEbuixNknSInoLhap6vKo+05b/BngY2ABsB3a3bruBS9ryduCmGvgUsDbJ+r7qkyQ900SuKSTZArwSuA9YV1WPt01PAOva8gZg/9DL5lvbwn3tTLI3yd5Dhw71V7QkrUK9h0KS5wF/BLylqr45vK2qCqil7K+qdlXVtqraNjc3N8ZKJUm9hkKS0xgEwgeq6kOt+cmj00Lt+WBrPwBsGnr5xtYmSZqQPj99FOBG4OGq+u2hTXuAHW15B3D7UPuV7VNI5wGHh6aZJEkTsKbHfZ8PvB74XJIHWttvAL8J3JrkauAx4LK27U7gImAf8DRwVY+1SZIW0VsoVNWfAjnG5gsX6V/ANX3VI0k6Me9oliR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1eguFJO9PcjDJ54fazkxyV5JH2vMLW3uSXJ9kX5IHk5zTV12SpGPrc6Twe8BrF7RdB9xdVVuBu9s6wOuAre2xE7ihx7okScfQWyhU1SeBry1o3g7sbsu7gUuG2m+qgU8Ba5Os76s2SdLiJn1NYV1VPd6WnwDWteUNwP6hfvOtTZI0QVO70FxVBdRSX5dkZ5K9SfYeOnSoh8okafWadCg8eXRaqD0fbO0HgE1D/Ta2tmeoql1Vta2qts3NzfVarCStNpMOhT3Ajra8A7h9qP3K9imk84DDQ9NMkqQJWdPXjpN8EHgNcFaSeeDtwG8Ctya5GngMuKx1vxO4CNgHPA1c1VddkqRj6y0UquoXjrHpwkX6FnBNX7VIkkbjHc3jcMoakozlsWHT5mn/NJJWsd5GCqvKd45w+Xv/bCy7uuWXXj2W/UjScjhSkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQmDXe8yBpirxPYdZ4z4OkKXKkIEnqGAqSpI6hoJPahk2bvQYjjZHXFHRS+8r8fq/BSGPkSEGS1DEUVjI/3roijHOKzH9LnYjTRyuZH29dmhai4/ADGzdxYP//Hsu+xjlFBqvk31LLZihoNDP6C3Osxhmiv/wTYztf0iQZChqNo46l8XzpJOU1BUlSx5GCJm+MU1GSxstQ0OQ5tbJibNi0ma/M7x/Lvmb2WtMqYyhIWjZvHlx5vKYgSeoYCpJmgzdbzgSnj6TVZlYv9HutaSYYCtJq4y9fHcdMTR8leW2SLyXZl+S6adcj6STlVNSyzcxIIcmpwO8APwXMA59OsqeqHppuZZJOOo6Glm2WRgrnAvuq6tGq+nvgZmD7lGuStNqNcdSx5vTnzPwIZmZGCsAGYPgumHngx6ZUiyQNjHnUMesjmFRVLzteqiSXAq+tqje29dcDP1ZVv7qg305gZ1t9KfClZR7yLOCry3xt32a1tlmtC6xtuaxt6Wa1Lhi9tn9UVXOLbZilkcIBYNPQ+sbW9j2qahew69keLMneqtr2bPfTh1mtbVbrAmtbLmtbulmtC8ZT2yxdU/g0sDXJ2UlOB64A9ky5JklaVWZmpFBVR5L8KvDHwKnA+6vqC1MuS5JWlZkJBYCquhO4c0KHe9ZTUD2a1dpmtS6wtuWytqWb1bpgHFPrs3KhWZI0fbN0TUGSNGUrPhRO9NUZSc5Ickvbfl+SLTNS1xuSHEryQHu8cRJ1tWO/P8nBJJ8/xvYkub7V/mCSc2akrtckOTx0zv79JOpqx96U5N4kDyX5QpI3L9JnWudtlNomfu6SPCfJXyT5y1bXf1ykz7Ten6PUNrX3aDv+qUk+m+SORbYt/7xV1Yp9MLhg/dfAi4HTgb8EXragz68A72nLVwC3zEhdbwD+25TO208A5wCfP8b2i4CPAgHOA+6bkbpeA9wxpXO2HjinLT8f+KtF/k2ndd5GqW3i566dh+e15dOA+4DzFvSZ+PtzCbVN7T3ajn8t8IeL/bs9m/O20kcKo3x1xnZgd1u+Dbgw6f17hWf6Kz2q6pPA147TZTtwUw18ClibZP0M1DU1VfV4VX2mLf8N8DCDu/SHTeu8jVLbxLXz8K22elp7LLzIOY3356i1TU2SjcBPA+87Rpdln7eVHgqLfXXGwjdD16eqjgCHgRfNQF0A/7JNM9yWZNMi26dl1Pqn4VVtyP/RJP9kGgW0oforGfzf5bCpn7fj1AZTOHdtCuQB4CBwV1Ud85xN8P05am0wvffoO4FfA75zjO3LPm8rPRROZh8BtlTVDwN38d3U17F9hsHt+y8H3g38j0kXkOR5wB8Bb6mqb076+Mdzgtqmcu6q6ttV9QoG32BwbpIfmsRxRzFCbVN5jyb5GeBgVd3fx/5XeiiM8tUZXZ8ka4AXAE9Nu66qeqqq/q6tvg/4kZ5rWoqRvpJk0qrqm0eH/DW45+W0JGdN6vhJTmPwS/cDVfWhRbpM7bydqLZpn7uq+gZwL/DaBZum8f4cqbYpvkfPBy5O8mUGU88XJPmDBX2Wfd5WeiiM8tUZe4AdbflS4J5qV2emWdeCueaLGcwDz4o9wJXt0zTnAYer6vFpF5XkHx6dN01yLoP/vifyC6Qd90bg4ar67WN0m8p5G6W2aZy7JHNJ1rbl5zL4WypfXNBtGu/PkWqb1nu0qt5WVRuraguD3x33VNW/WtBt2edtpu5oHrc6xldnJPlPwN6q2sPgzfL7SfYxuIh5xYzU9W+SXAwcaXW9oe+6jkryQQafRjkryTzwdgYX2qiq9zC46/wiYB/wNHDVjNR1KfDLSY4A/xe4YhK/QJrzgdcDn2vz0AC/AWweqm8q523E2qZx7tYDuzP4A1unALdW1R3Tfn8uobapvUcXM67z5h3NkqTOSp8+kiQtgaEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSer8fxYKz/tIuZ9hAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(ood_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAT9UlEQVR4nO3df/BddX3n8edLfkRHKEHzXYghNDTS3cHSYjZlSe3soIxbykwb26Vs3B0Jjm6cLa467XZG3dnV7qwdd8bqLtXBSYUxdKxIFTUqtqXIyDhT0ZAiX37INrEg+U6EABqwLtDge//4nhwuN/f7/d58ybn3fpPnY+bO99zP+Zxz3+feb84r55zPPd9UFZIkAbxo3AVIkiaHoSBJahkKkqSWoSBJahkKkqTW8eMu4IVYsWJFrVmzZtxlSNKScscddzxaVVOD5i3pUFizZg07duwYdxmStKQkeXCueZ4+kiS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1OguFJC9O8q0k30lyT5I/bNrPSnJ7kl1JPpPkxKZ9WfN8VzN/TVe1SZIG6/LLa08Dr6uqHyc5AfhGkq8Cvwd8pKquT/Jx4C3A1c3PH1bVK5NsAv4X8O86Kezpp9m5c+ch7evWrWPZsmVdvKQkLQmdhULN/vWeHzdPT2geBbwO+PdN+zbg/cyGwsZmGuCzwEeTpDr4K0A7d+7kHR/7AqesWtu27Z/ZzVVXwoYNG470y0nSktHpbS6SHAfcAbwS+BiwG/hRVR1ouuwBVjXTq4CHAKrqQJL9wMuBR/vWuQXYAnDmmWcuurZTVq1lxdpzF728JB2NOr3QXFXPVtV5wBnA+cC/OALr3FpV66tq/dTUwPs5SZIWaSSjj6rqR8CtwAZgeZKDRyhnADPN9AywGqCZfwrw2CjqkyTN6nL00VSS5c30S4DXA/cxGw6XNt02A19sprc3z2nmf62L6wmSpLl1eU1hJbCtua7wIuCGqvpyknuB65P8T+DvgGua/tcAf5ZkF/A4sKnD2iRJA3Q5+ugu4NUD2r/H7PWF/vangN/pqh5J0sL8RrMkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqdVZKCRZneTWJPcmuSfJO5v29yeZSXJn87ikZ5n3JNmV5P4kv9ZVbZKkwY7vcN0HgN+vqp1JTgbuSHJzM+8jVfWh3s5JzgE2Aa8CXgH8TZKfr6pnO6xRktSjsyOFqtpbVTub6SeB+4BV8yyyEbi+qp6uqn8AdgHnd1WfJOlQI7mmkGQN8Grg9qbp7UnuSnJtklObtlXAQz2L7WFAiCTZkmRHkh379u3rsmxJOuZ0HgpJTgI+B7yrqp4ArgbWAucBe4E/Ppz1VdXWqlpfVeunpqaOdLmSdEzrNBSSnMBsIHyqqm4EqKqHq+rZqvop8Kc8d4poBljds/gZTZskaUS6HH0U4Brgvqr6cE/7yp5uvwXc3UxvBzYlWZbkLOBs4Ftd1SdJOlSXo49eA7wJmE5yZ9P2XuCNSc4DCngAeBtAVd2T5AbgXmZHLl3pyCNJGq3OQqGqvgFkwKyb5lnmA8AHuqpJkjQ/v9EsSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWp1FgpJVie5Ncm9Se5J8s6m/WVJbk7y983PU5v2JLkqya4kdyVZ11VtkqTBujxSOAD8flWdA1wAXJnkHODdwC1VdTZwS/Mc4NeBs5vHFuDqDmuTJA3QWShU1d6q2tlMPwncB6wCNgLbmm7bgDc00xuB62rWN4HlSVZ2VZ8k6VAjuaaQZA3wauB24LSq2tvM+gFwWjO9CnioZ7E9TZskaUQ6D4UkJwGfA95VVU/0zquqAuow17clyY4kO/bt23cEK5UkdRoKSU5gNhA+VVU3Ns0PHzwt1Px8pGmfAVb3LH5G0/Y8VbW1qtZX1fqpqanuipekY1CXo48CXAPcV1Uf7pm1HdjcTG8GvtjTfnkzCukCYH/PaSZJ0ggc3+G6XwO8CZhOcmfT9l7gg8ANSd4CPAhc1sy7CbgE2AX8BHhzh7VJkgboLBSq6htA5ph90YD+BVzZVT2SpIX5jWZJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUstQkCS1DAVJUmuoUEjymmHaJElL27BHCn8yZJskaQmb9y6pSTYAvwJMJfm9nlk/AxzXZWGSpNFb6NbZJwInNf1O7ml/Ari0q6IkSeMxbyhU1deBryf5ZFU9OKKaJEljMuwf2VmWZCuwpneZqnpdF0VJksZj2FD4C+DjwCeAZ7srR5I0TsOGwoGqurrTSiRJYzfskNQvJfndJCuTvOzgo9PKJEkjN+yRwubm5x/0tBXwc0e2HEnSOA0VClV1VteFSJLGb6hQSHL5oPaquu7IliNJGqdhTx/9cs/0i4GLgJ2AoSBJR5FhTx/9597nSZYD13dRkCRpfBZ76+x/BLzOIElHmWFvnf2lJNubx1eA+4HPL7DMtUkeSXJ3T9v7k8wkubN5XNIz7z1JdiW5P8mvLXaDJEmLN+w1hQ/1TB8AHqyqPQss80ngoxx63eEjVdW7PpKcA2wCXgW8AvibJD9fVX57WpJGaKgjhebGeN9l9k6ppwLPDLHMbcDjQ9axEbi+qp6uqn8AdgHnD7msJOkIGfb00WXAt4DfAS4Dbk+y2Ftnvz3JXc3ppVObtlXAQz199jRtg2rZkmRHkh379u1bZAmSpEGGvdD8X4FfrqrNVXU5s/+L/2+LeL2rgbXAecBe4I8PdwVVtbWq1lfV+qmpqUWUIEmay7Ch8KKqeqTn+WOHsWyrqh6uqmer6qfAn/LcKaIZYHVP1zOaNknSCA27Y//LJH+V5IokVwBfAW463BdLsrLn6W8BB0cmbQc2JVmW5CzgbGZPV0mSRmihv9H8SuC0qvqDJL8N/Goz62+BTy2w7KeBC4EVSfYA7wMuTHIeszfTewB4G0BV3ZPkBuBeZkc3XenII0kavYWGpP5v4D0AVXUjcCNAknObeb8x14JV9cYBzdfM0/8DwAcWqEeS1KGFTh+dVlXT/Y1N25pOKpIkjc1CobB8nnkvOYJ1SJImwEKhsCPJf+xvTPJW4I5uSpIkjctC1xTeBXw+yX/guRBYD5zI7OghSdJRZN5QqKqHgV9J8lrgF5rmr1TV1zqvTJI0csP+PYVbgVs7rkWSNGaL/XsKkqSjkKEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKkVmehkOTaJI8kubun7WVJbk7y983PU5v2JLkqya4kdyVZ11VdkqS5dXmk8Eng4r62dwO3VNXZwC3Nc4BfB85uHluAqzusS5I0h85CoapuAx7va94IbGumtwFv6Gm/rmZ9E1ieZGVXtUmSBhv1NYXTqmpvM/0D4LRmehXwUE+/PU3bIZJsSbIjyY59+/Z1V6kkHYPGdqG5qgqoRSy3tarWV9X6qampDiqTpGPXqEPh4YOnhZqfjzTtM8Dqnn5nNG2SpBEadShsBzY305uBL/a0X96MQroA2N9zmkmSNCLHd7XiJJ8GLgRWJNkDvA/4IHBDkrcADwKXNd1vAi4BdgE/Ad7cVV2SpLl1FgpV9cY5Zl00oG8BV3ZViyRpOH6jWZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSa3jx13ApPjpgX9ienr6eW3r1q1j2bJlY6pIkkbPUGg8+fD3uerBpzh99+zz/TO7uepK2LBhw3gLk6QRMhR6nHz6GlasPXfcZUjS2IwlFJI8ADwJPAscqKr1SV4GfAZYAzwAXFZVPxxHfZJ0rBrnhebXVtV5VbW+ef5u4JaqOhu4pXkuSRqhSRp9tBHY1kxvA94wvlIk6dg0rlAo4K+T3JFkS9N2WlXtbaZ/AJw2ntIk6dg1rgvNv1pVM0n+GXBzku/2zqyqSlKDFmxCZAvAmWee2X2lknQMGcuRQlXNND8fAT4PnA88nGQlQPPzkTmW3VpV66tq/dTU1KhKlqRjwshDIclLk5x8cBr4N8DdwHZgc9NtM/DFUdcmSce6cZw+Og34fJKDr//nVfWXSb4N3JDkLcCDwGVjqE2SjmkjD4Wq+h7wSwPaHwMuGnU9kqTnTNKQVEnSmBkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqTWOP7y2pLw0wP/xPT09CHt69atY9myZWOoSJK6ZyjM4cmHv89VDz7F6bufa9s/s5urroQNGzaMrzBJ6pChMI+TT1/DirXnts8HHT30Hzk8/fTT7Ny5c94+kjSpDIXD0H/08MPv38/bXjvNuec+FxzT09Ns/fpulp+xFvDoQtLSYigcpt6jh/0zu7nqr+953immmTtvY/nZ//J5RxiStFQYCi9Q/ymm/TO75+ktSZPNIamSpJahIElqefpoDPpHKD3zzDMAnHjiic/rt9CopcWMdBq0TFevJWnpMRTGYOfOnbzjY1/glFWzI5Rm7ryN405+OaevfVXbZ5hRS/3rWcwyMHgUVf8OfzGvZZB0b7EhL81l4kIhycXA/wGOAz5RVR8cc0kvyKDvNkxPT/Mzr1j7vFFMxy8/fd4RS4P+8fevZ9hvYZ+yau0hF8d7R1HNNdS297WGre9wh+cOs5Mb5Y5w0Gv1H9kNOtIb1NbFd1oWG/L9jtQR5DDvxWKPjI+UpfaflVEH/0SFQpLjgI8Brwf2AN9Osr2q7h1vZYs36JvRB4etzqd/B9+/gx20nhfyLexhh9oupr7DGZ47aCfXvw3D7AiH2fEMs8Ofa7t6j+wGHen1tw3znZZBfRbaoQ4K62FCvv+9WOx72r8Nw7wXg/oM81rDBO0wFnPU22+Y351h6htmhz/Mv4kjaaJCATgf2FVV3wNIcj2wETjiodA/dPTH+2Y47qmnePSlLx34/AX1Ofnlh7z+kz94YN717L37b/mj25/k1JV3A/Do9+7mlLMO3bkesp4Br9W/8x5q2xeoeTH17Z/ZzYADmTlrnW8b+v3k8Yf5o+t2Pa+e415yMqeu/Nm2zz8+tpf/sun17Y5nenqaD11/My99+cq2T/9yc23X4eqvb9C65+rTX8+g+pLnXqv/8xu03kHvxUI1D3pPu3p/5nqt3rb+bRjWoG2d6/duvnUs9LszTH2D1jPMZ9OlVNVIX3A+SS4FLq6qtzbP3wT8q6p6e0+fLcCW5uk/B+5f5MutAB59AeWO21KvH5b+Nlj/+C31bRhX/T9bVVODZkzakcKCqmorsPWFrifJjqpafwRKGoulXj8s/W2w/vFb6tswifVP2vcUZoDVPc/PaNokSSMwaaHwbeDsJGclORHYBGwfc02SdMyYqNNHVXUgyduBv2J2SOq1VXVPRy/3gk9BjdlSrx+W/jZY//gt9W2YuPon6kKzJGm8Ju30kSRpjAwFSVLrqA+FJBcnuT/JriTvHjB/WZLPNPNvT7JmDGXOaYj6r0iyL8mdzeOt46hzLkmuTfJIkrvnmJ8kVzXbd1eSdaOucT5D1H9hkv097/9/H3WN80myOsmtSe5Nck+Sdw7oM+mfwTDbMLGfQ5IXJ/lWku809f/hgD6Tsx+qqqP2wezF6t3AzwEnAt8Bzunr87vAx5vpTcBnxl33YdZ/BfDRcdc6zzb8a2AdcPcc8y8BvgoEuAC4fdw1H2b9FwJfHned89S/EljXTJ8M/N8Bv0OT/hkMsw0T+zk07+tJzfQJwO3ABX19JmY/dLQfKbS3zaiqZ4CDt83otRHY1kx/Frgo6b1pwFgNU/9Eq6rbgMfn6bIRuK5mfRNYnmTlPP1Haoj6J1pV7a2qnc30k8B9wKq+bpP+GQyzDROreV9/3Dw9oXn0j/CZmP3Q0R4Kq4CHep7v4dBfprZPVR0A9gOH3vhnPIapH+DfNof9n02yesD8STbsNk6yDc2pga8medXC3cejOSXxamb/p9pryXwG82wDTPDnkOS4JHcCjwA3V9Wcn8G490NHeygcC74ErKmqXwRu5rn/bWg0djJ7H5lfAv4E+MJ4yxksyUnA54B3VdUT465nMRbYhon+HKrq2ao6j9m7NJyf5BfGXNKcjvZQGOa2GW2fJMcDpwCPjaS6hS1Yf1U9VlVPN08/Acx/T+7Js6RvbVJVTxw8NVBVNwEnJFkx5rKeJ8kJzO5MP1VVNw7oMvGfwULbsBQ+B4Cq+hFwK3Bx36yJ2Q8d7aEwzG0ztgObm+lLga9Vc7VnAixYf9+5399k9nzrUrIduLwZAXMBsL+q9o67qGElOf3gud8k5zP7b2pS/lNBU9s1wH1V9eE5uk30ZzDMNkzy55BkKsnyZvolzP69mO/2dZuY/dBE3ebiSKs5bpuR5H8AO6pqO7O/bH+WZBezFxQ3ja/i5xuy/nck+U3gALP1XzG2ggdI8mlmR4asSLIHeB+zF9qoqo8DNzE7+mUX8BPgzeOpdLAh6r8U+E9JDgD/D9g0Qf+pAHgN8CZgujmnDfBe4ExYGp8Bw23DJH8OK4Ftmf0jYi8CbqiqL0/qfsjbXEiSWkf76SNJ0mEwFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktT6/4DeuF4bEfQdAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(ood_score[ood_labels==0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAATUklEQVR4nO3dfZBd9X3f8feHJycT7GKHHaoKqQJX8YS4iaAbSkzMEBM3QF2IExfDpDykdoRjaO1xJqntTOM0M5nJtH5I81CIbBigxVjEQI1dnITB1EwmAWeFCQZjx4JCkSKjDbjgxh6ngm//uEeHy/qudCXtOWelfb9m7ujc3zl370dH3P1wHm+qCkmSAA4bOoAkafmwFCRJLUtBktSyFCRJLUtBktQ6YugAB+LYY4+tdevWDR1Dkg4qW7Zs+Zuqmpk076AuhXXr1jE3Nzd0DEk6qCR5YrF57j6SJLUsBUlSq7NSSLImyd1Jvpzk4STvasZfleTOJF9r/nxlM54kv5Nka5IHk5zSVTZJ0mRdbinsAn6pqk4CTgOuSHIS8F7grqpaD9zVPAc4B1jfPDYCV3WYTZI0QWelUFU7qur+ZvqbwCPAauB84PpmseuBn26mzwduqJF7gWOSrOoqnyTpu/VyTCHJOuBk4D7guKra0cz6OnBcM70aeHLsZduaMUlSTzovhSRHA7cA766q58bn1egWrft0m9YkG5PMJZmbn59fwqSSpE5LIcmRjArhxqq6tRl+avduoebPnc34dmDN2MuPb8Zeoqo2VdVsVc3OzEy89kKStJ+6PPsowDXAI1X14bFZtwOXNtOXAp8aG7+kOQvpNODZsd1MkqQedLmlcDpwMfCGJA80j3OB3wLemORrwE82zwHuAB4DtgIfBd7ZYTYAVq9ZS5JeHqvXrO36ryNJB6yz21xU1Z8CWWT2WROWL+CKrvJM8tfbnuStf/BnvbzX5stf18v7SNKB8IpmSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktSwFSVLLUpAktTorhSTXJtmZ5KGxsc1j39f8eJIHmvF1Sb49Nu/qrnJJkhbX2Xc0A9cBvwfcsHugqt66ezrJh4Bnx5Z/tKo2dJhHkrQXnZVCVd2TZN2keUkCXAC8oav3lyTtu6GOKbweeKqqvjY2dkKSLyb5fJLXL/bCJBuTzCWZm5+f7z6pJK0gQ5XCRcBNY893AGur6mTgPcDHk7xi0guralNVzVbV7MzMTA9RJWnl6L0UkhwB/AywefdYVX2nqp5uprcAjwI/0Hc2SVrphthS+EngK1W1bfdAkpkkhzfTJwLrgccGyCZJK1qXp6TeBPw58Jok25K8rZl1IS/ddQRwBvBgc4rqJ4F3VNUzXWWTJE3W5dlHFy0yftmEsVuAW7rKIkmajlc0S5JaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaloIkqWUpSJJaXX4d57VJdiZ5aGzs15NsT/JA8zh3bN77kmxN8tUkP9VVLknS4rrcUrgOOHvC+EeqakPzuAMgyUmMvrv5h5rX/Jckh3eYTZI0QWelUFX3AM9Mufj5wCeq6jtV9b+ArcCpXWWTJE02xDGFK5M82OxeemUzthp4cmyZbc3Yd0myMclckrn5+fmus0rSitJ3KVwFvBrYAOwAPrSvP6CqNlXVbFXNzszMLHE8SVrZei2Fqnqqqp6vqheAj/LiLqLtwJqxRY9vxiRJPeq1FJKsGnv6ZmD3mUm3AxcmeVmSE4D1wBf6zCZJgiO6+sFJbgLOBI5Nsg34AHBmkg1AAY8DlwNU1cNJbga+DOwCrqiq57vKJkmarLNSqKqLJgxfs4flfxP4za7ySJL2ziuaJUktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1LIUJEmtzkohybVJdiZ5aGzsPyX5SpIHk9yW5JhmfF2Sbyd5oHlc3VUuSdLiutxSuA44e8HYncBrq+qHgb8C3jc279Gq2tA83tFhLknSIjorhaq6B3hmwdifVNWu5um9wPFdvb8kad8NeUzhXwOfHXt+QpIvJvl8ktcv9qIkG5PMJZmbn5/vPqUkrSCDlEKSXwV2ATc2QzuAtVV1MvAe4ONJXjHptVW1qapmq2p2Zmamn8CStEL0XgpJLgPeBPxcVRVAVX2nqp5uprcAjwI/0Hc2SVrpei2FJGcDvwKcV1XfGhufSXJ4M30isB54rM9skiQ4oqsfnOQm4Ezg2CTbgA8wOtvoZcCdSQDubc40OgP4jST/D3gBeEdVPTPxB0uSOtNZKVTVRROGr1lk2VuAW7rKIkmajlc0S5JaloIkqWUpSJJaloIkqWUpSJJaU5VCktOnGZMkHdym3VL43SnHJEkHsT1ep5Dkx4DXATNJ3jM26xXA4V0GkyT1b28Xrx0FHN0s9/Kx8eeAt3QVSpI0jD2WQlV9Hvh8kuuq6omeMkmSBjLtbS5elmQTsG78NVX1hi5CSZKGMW0p/CFwNfAx4Pnu4kiShjRtKeyqqqs6TSJJGty0p6R+Osk7k6xK8qrdj06TSZJ6N+2WwqXNn788NlbAiUsbR5I0pKlKoapO6DqIJGl4U5VCkksmjVfVDUsbR5I0pGl3H/3o2PT3AGcB9wOWgiQdQqbdffRvxp8nOQb4xN5el+Ra4E3Azqp6bTP2KmAzo2seHgcuqKpvZPSlzf8ZOBf4FnBZVd0/7V9EknTg9vfW2X8LTHOc4Trg7AVj7wXuqqr1wF3Nc4BzgPXNYyPgKbCS1LNpjyl8mtHZRjC6Ed4PAjfv7XVVdU+SdQuGzwfObKavB/4n8O+a8RuqqoB7kxyTZFVV7ZgmoyTpwE17TOGDY9O7gCeqatt+vudxY7/ovw4c10yvBp4cW25bM/aSUkiykdGWBGvXrt3PCJKkSabafdTcGO8rjO6U+krg75bizZutgtrrgi99zaaqmq2q2ZmZmaWIIUlqTPvNaxcAXwD+JXABcF+S/b119lNJVjU/dxWwsxnfDqwZW+74ZkyS1JNpDzT/KvCjVXVpVV0CnAr8+/18z9t58QrpS4FPjY1fkpHTgGc9niBJ/Zr2mMJhVbVz7PnTTFEoSW5idFD52CTbgA8AvwXcnORtwBOMtjwA7mB0OupWRqek/vyU2SRJS2TaUvijJH8M3NQ8fyujX+J7VFUXLTLrrAnLFnDFlHkkSR3Y23c0/yNGZwv9cpKfAX68mfXnwI1dh5Mk9WtvWwq/DbwPoKpuBW4FSPKPm3n/osNskqSe7e24wHFV9aWFg83Yuk4SSZIGs7dSOGYP8753CXNIkpaBvZXCXJJfWDiY5O3Alm4iSZKGsrdjCu8Gbkvyc7xYArPAUcCbO8wlSRrAHkuhqp4CXpfkJ4DXNsP/o6o+13kySVLvpv0+hbuBuzvOIkka2P5+n4Ik6RBkKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKk17TevLZkkrwE2jw2dCPwaozuy/gIw34y/v6r2+u1ukqSl03spVNVXgQ0ASQ4HtgO3MfpO5o9U1Qf7ziRJGhl699FZwKNV9cTAOSRJDF8KFwI3jT2/MsmDSa5N8spJL0iyMclckrn5+flJi0iS9tNgpZDkKOA84A+boauAVzPatbQD+NCk11XVpqqararZmZmZPqJK0oox5JbCOcD9zXc2UFVPVdXzVfUC8FHg1AGzSdKKNGQpXMTYrqMkq8bmvRl4qPdEkrTC9X72EUCS7wPeCFw+Nvwfk2wACnh8wTxJUg8GKYWq+lvg+xeMXTxEFknSi4Y++0iStIxYCpKklqUgSWpZCpKklqUgSWpZCpKklqXQl8OOIElvj9Vr1g79N5Z0EBrkOoUV6YVdvPUP/qy3t9t8+et6ey9Jhw63FCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBB53Va9Z6IaDUES9e00Hnr7c96YWAUkcGK4UkjwPfBJ4HdlXVbJJXAZuBdYy+kvOCqvrGUBklaaUZevfRT1TVhqqabZ6/F7irqtYDdzXPJUk9GboUFjofuL6Zvh746eGiSNLKM2QpFPAnSbYk2diMHVdVO5rprwPHLXxRko1J5pLMzc/P95VVklaEIUvhx6vqFOAc4IokZ4zPrKpiVBwsGN9UVbNVNTszM9NTVKk/fZ5d5ZlVWmiwA81Vtb35c2eS24BTgaeSrKqqHUlWATuHyicNpc+zqzyzSgsNsqWQ5PuSvHz3NPDPgIeA24FLm8UuBT41RD5JWqmG2lI4Drgtye4MH6+qP0ryF8DNSd4GPAFcMFA+SVqRBimFqnoM+JEJ408DZ/WfSJIEy++UVEnSgLzNxaHqsCNods/14h8cv4btT/7v3t6vVz2vS2lIlsKh6oVd3h9oqbgutYK4+0iS1LIUJEktS0GS1LIUJEktS0GS1LIUJEktS0GS1PI6BS0NL/CSDgmWgpZGjxd4eXGX1B13H0mSWpaCJKllKUiSWpaCJKllKUiSWr2XQpI1Se5O8uUkDyd5VzP+60m2J3mgeZzbdzZJWumGOCV1F/BLVXV/kpcDW5Lc2cz7SFV9cIBM0srklzFpgd5Loap2ADua6W8meQRY3XcOSfT/BUK/eIYltMwNevFaknXAycB9wOnAlUkuAeYYbU18Y8B4kpaa32K37A12oDnJ0cAtwLur6jngKuDVwAZGWxIfWuR1G5PMJZmbn5/vK64krQiDlEKSIxkVwo1VdStAVT1VVc9X1QvAR4FTJ722qjZV1WxVzc7MzPQXWpJWgCHOPgpwDfBIVX14bHzV2GJvBh7qO5skrXRDHFM4HbgY+FKSB5qx9wMXJdkAFPA4cPkA2SRpRRvi7KM/BSadfnBH31kkSS/lFc2SpJalIElqWQqSpJalIElqWQqSpJalIElqWQqSpJalIElqWQqSDl3N90X08Vi9Zu3Qf9slMeitsyWpUz3eqvtQuU23WwqSpJalIElqWQqSpJalIElqWQqSpJalIElqWQqStBR6vCaiy+sivE5BkpZCj9dEQHfXRSy7LYUkZyf5apKtSd47dB5JWkmWVSkkORz4feAc4CTgoiQnDZtKklaOZVUKwKnA1qp6rKr+DvgEcP7AmSRpxUhVDZ2hleQtwNlV9fbm+cXAP62qK8eW2QhsbJ6+Bvjqfr7dscDfHEDcLplt3y3XXGC2/WW2fTdtrn9YVTOTZhx0B5qrahOw6UB/TpK5qppdgkhLzmz7brnmArPtL7Ptu6XItdx2H20H1ow9P74ZkyT1YLmVwl8A65OckOQo4ELg9oEzSdKKsax2H1XVriRXAn8MHA5cW1UPd/R2B7wLqkNm23fLNReYbX+Zbd8d+K715XSgWZI0rOW2+0iSNCBLQZLUOuRLYW+3zUjysiSbm/n3JVm3THJdlmQ+yQPN4+195Gre+9okO5M8tMj8JPmdJvuDSU5ZJrnOTPLs2Dr7tT5yNe+9JsndSb6c5OEk75qwzFDrbZpsg6y7JN+T5AtJ/rLJ9h8mLNP7Z3TKXIN9Rpv3PzzJF5N8ZsK8/V9nVXXIPhgdrH4UOBE4CvhL4KQFy7wTuLqZvhDYvExyXQb83kDr7QzgFOChReafC3wWCHAacN8yyXUm8JmB1tkq4JRm+uXAX034Nx1qvU2TbZB116yLo5vpI4H7gNMWLDPEZ3SaXIN9Rpv3fw/w8Un/bgeyzg71LYVpbptxPnB9M/1J4KwkWQa5BlNV9wDP7GGR84EbauRe4Jgkq5ZBrsFU1Y6qur+Z/ibwCLB6wWJDrbdpsg2iWRf/t3l6ZPNYePZL75/RKXMNJsnxwD8HPrbIIvu9zg71UlgNPDn2fBvf/WFol6mqXcCzwPcvg1wAP9vsZvhkkjUT5g9l2vxD+LFmk/+zSX5oiADNpvrJjP7vctzg620P2WCgddfsBnkA2AncWVWLrrceP6PT5ILhPqO/DfwK8MIi8/d7nR3qpXAw+zSwrqp+GLiTF1tfi7uf0T1dfgT4XeC/9x0gydHALcC7q+q5vt9/T/aSbbB1V1XPV9UGRncwODXJa/t67z2ZItcgn9EkbwJ2VtWWLn7+oV4K09w2o10myRHA3wOeHjpXVT1dVd9pnn4M+CcdZ9oXy/J2JFX13O5N/qq6AzgyybF9vX+SIxn90r2xqm6dsMhg621v2YZed837/h/gbuDsBbOG+IzuNdeAn9HTgfOSPM5o1/Mbkvy3Bcvs9zo71Ethmttm3A5c2ky/BfhcNUdnhsy1YF/zeYz2Ay8XtwOXNGfTnAY8W1U7hg6V5O/v3m+a5FRG/3338sujed9rgEeq6sOLLDbIepsm21DrLslMkmOa6e8F3gh8ZcFivX9Gp8k11Ge0qt5XVcdX1TpGvzs+V1X/asFi+73OltVtLpZaLXLbjCS/AcxV1e2MPiz/NclWRgcxL1wmuf5tkvOAXU2uy7rOtVuSmxidjXJskm3ABxgdaKOqrgbuYHQmzVbgW8DPL5NcbwF+Mcku4NvAhT0U/G6nAxcDX2r2QwO8H1g7lm+Q9TZltqHW3Srg+oy+YOsw4Oaq+szQn9Epcw32GZ1kqdaZt7mQJLUO9d1HkqR9YClIklqWgiSpZSlIklqWgiSpZSlIklqWgiSp9f8BQ3EJBSz4VUsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(ood_score[ood_labels==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7680)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0.25\n",
    "(ood_score[ood_labels==1] > threshold).sum()/sum(ood_score> threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7262)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0.1\n",
    "(ood_score[ood_labels==1] > threshold).sum()/sum(ood_score> threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.8188)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ood_score[ood_labels==0] < 0.99).sum()/len(ood_score[ood_labels==0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3378)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ood_score[ood_labels==1] < 0.99).sum()/len(ood_score[ood_labels==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3378)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ood_score[ood_labels==1] < 0.99).sum()/len(ood_score[ood_labels==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for test_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GittablesTablewiseIterateDataset(data.Dataset):\n",
    "\n",
    "    def __init__(\n",
    "            self,\n",
    "            cv: int,\n",
    "            split: str,\n",
    "            src: str,  # train or test\n",
    "            tokenizer: AutoTokenizer,\n",
    "            max_length: int = 256,\n",
    "            gt_only: bool = False,\n",
    "            device: torch.device = None,\n",
    "            base_dirpath: str = \"/data/zhihao/TU/GitTables/semtab_gittables/2022\",\n",
    "            base_tag: str = '', # blank, comma\n",
    "            small_tag: str = \"\",\n",
    "            train_ratio: float = 1.0,\n",
    "            max_unlabeled=8,\n",
    "            random_sample=False, # TODO\n",
    "            train_only=False): # TODO\n",
    "        if device is None:\n",
    "            device = torch.device('cpu')\n",
    "        basename = small_tag+ \"_cv_{}.csv\"\n",
    "    \n",
    "        if split in [\"train\", \"valid\"]:\n",
    "            df_list = []\n",
    "            for i in range(5):\n",
    "                if i == cv:\n",
    "                    continue\n",
    "                filepath = os.path.join(base_dirpath, basename.format(i))\n",
    "                df_list.append(pd.read_csv(filepath))\n",
    "                print(split, i)\n",
    "            df = pd.concat(df_list, axis=0)\n",
    "        else:\n",
    "            # test\n",
    "            filepath = os.path.join(base_dirpath, basename.format(cv))\n",
    "            df = pd.read_csv(filepath)\n",
    "            print(split)\n",
    "\n",
    "\n",
    "        if gt_only:\n",
    "            df = df[df[\"class_id\"] > -1]\n",
    "        if train_only and split != \"train\":\n",
    "            df = df[df[\"class_id\"] > -1]\n",
    "\n",
    "        \n",
    "        data_list = []\n",
    "        \n",
    "        df['class_id'] = df['class_id'].astype(int)\n",
    "        df.drop(df[(df['data'].isna()) & (df['class_id'] == -1)].index, inplace=True)\n",
    "        df['col_idx'] = df['col_idx'].astype(int)\n",
    "        df['data'] = df['data'].astype(str)\n",
    "        \n",
    "        num_tables = len(df.groupby(\"table_id\"))\n",
    "        valid_index = int(num_tables * 0.8)\n",
    "        num_train = int(train_ratio * num_tables * 0.8)        \n",
    "        \n",
    "        # df.drop(df[(df['data'] == '') & (df['class_id'] == -1)].index, inplace=True)\n",
    "        total_num_cols = 0\n",
    "        for i, (index, group_df) in enumerate(df.groupby(\"table_id\")):\n",
    "            if (split == \"train\") and ((i >= num_train) or (i >= valid_index)):\n",
    "                break\n",
    "            if split == \"valid\" and i < valid_index:\n",
    "                continue\n",
    "            #     break\n",
    "            labeled_columns = group_df[group_df['class_id'] > -1]\n",
    "            unlabeled_columns = group_df[group_df['class_id'] == -1]\n",
    "            num_unlabeled = min(max(max_unlabeled-len(labeled_columns), 0), len(unlabeled_columns)) if max_unlabeled > 0 else len(unlabeled_columns)\n",
    "            unlabeled_columns = unlabeled_columns.sample(num_unlabeled) if random_sample else unlabeled_columns[0:num_unlabeled]\n",
    "            # group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns.sample(min(10-len(labeled_columns), len(unlabeled_columns)))])\n",
    "            # group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns[0:min(max(10-len(labeled_columns), 0), len(unlabeled_columns))]])\n",
    "            group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns]) # TODO\n",
    "            group_df.sort_values(by=['col_idx'], inplace=True)\n",
    "\n",
    "            if max_length <= 128:\n",
    "                cur_maxlen = min(max_length, 512 // len(list(group_df[\"class_id\"].values)) - 1)\n",
    "            else:\n",
    "                cur_maxlen = max(1, max_length // len(list(group_df[\"class_id\"].values)) - 1)\n",
    "                \n",
    "            token_ids_list = group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "                tokenizer.cls_token + \" \" + x, add_special_tokens=False, max_length=cur_maxlen, truncation=True)).tolist(\n",
    "                )\n",
    "            token_ids = torch.LongTensor(reduce(operator.add,\n",
    "                                                token_ids_list)).to(device)\n",
    "            for col_i in range(len(token_ids_list)):\n",
    "                if group_df[\"class_id\"].values[col_i] == -1:\n",
    "                    continue\n",
    "                target_col_mask = []\n",
    "                cls_index_value = 0\n",
    "                context_id = 1\n",
    "                for col_j in range(len(token_ids_list)):\n",
    "                    if col_j == col_i:\n",
    "                        target_col_mask += [0] * len(token_ids_list[col_j])\n",
    "                    else:\n",
    "                        target_col_mask += [context_id] * len(token_ids_list[col_j])\n",
    "                        context_id += 1\n",
    "                    if col_j < col_i:\n",
    "                        cls_index_value += len(token_ids_list[col_j])\n",
    "                cls_index_list = [cls_index_value] \n",
    "                for cls_index in cls_index_list:\n",
    "                    assert token_ids[\n",
    "                        cls_index] == tokenizer.cls_token_id, \"cls_indexes validation\"\n",
    "                cls_indexes = torch.LongTensor(cls_index_list).to(device)\n",
    "                class_ids = torch.LongTensor(\n",
    "                    [group_df[\"class_id\"].values[col_i]]).to(device)\n",
    "                target_col_mask = torch.LongTensor(target_col_mask).to(device)\n",
    "                data_list.append(\n",
    "                    [index,\n",
    "                    len(group_df), token_ids, class_ids, cls_indexes, target_col_mask])                \n",
    "        print(split, len(data_list))\n",
    "        self.table_df = pd.DataFrame(data_list,\n",
    "                                     columns=[\n",
    "                                         \"table_id\", \"num_col\", \"data_tensor\",\n",
    "                                         \"label_tensor\", \"cls_indexes\", \"target_col_mask\"\n",
    "                                     ])\n",
    "        \"\"\"\n",
    "        # NOTE: msato contains a small portion of single-col tables. keep it to be consistent.  \n",
    "        if multicol_only:\n",
    "            # Check\n",
    "            num_all_tables = len(self.table_df)\n",
    "            self.table_df = self.table_df[self.table_df[\"num_col\"] > 1]\n",
    "            assert len(self.table_df) == num_all_tables\n",
    "        \"\"\"\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.table_df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            \"data\": self.table_df.iloc[idx][\"data_tensor\"],\n",
    "            \"label\": self.table_df.iloc[idx][\"label_tensor\"],\n",
    "            \"cls_indexes\": self.table_df.iloc[idx][\"cls_indexes\"],\n",
    "            \"target_col_mask\": self.table_df.iloc[idx][\"target_col_mask\"],\n",
    "        }\n",
    "        #\"idx\": torch.LongTensor([idx])}\n",
    "        #\"cls_indexes\": self.table_df.iloc[idx][\"cls_indexes\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_fn(pad_token_id, data_only=True):\n",
    "    '''padder for input batch'''\n",
    "\n",
    "    def padder(samples):    \n",
    "        data = torch.nn.utils.rnn.pad_sequence(\n",
    "            [sample[\"data\"] for sample in samples], padding_value=pad_token_id)\n",
    "        if not data_only:\n",
    "            label = torch.nn.utils.rnn.pad_sequence(\n",
    "                [sample[\"label\"] for sample in samples], padding_value=-1)\n",
    "        else:\n",
    "            label = torch.cat([sample[\"label\"] for sample in samples])\n",
    "        batch = {\"data\": data, \"label\": label}\n",
    "        if \"idx\" in samples[0]:\n",
    "            batch[\"idx\"] = [sample[\"idx\"] for sample in samples]\n",
    "        if \"cls_indexes\" in samples[0]:\n",
    "            cls_indexes = torch.nn.utils.rnn.pad_sequence(\n",
    "                [sample[\"cls_indexes\"] for sample in samples], padding_value=0)\n",
    "            batch[\"cls_indexes\"] = cls_indexes\n",
    "        if \"target_col_mask\" in samples[0]:\n",
    "            target_col_mask = torch.nn.utils.rnn.pad_sequence(\n",
    "                [sample[\"target_col_mask\"] for sample in samples], padding_value=-1)\n",
    "            batch[\"target_col_mask\"] = target_col_mask\n",
    "        if \"table_embedding\" in samples[0]:\n",
    "            table_embeddings = [sample[\"table_embedding\"] for sample in samples]\n",
    "            batch[\"table_embedding\"] = torch.stack(table_embeddings, dim=0)\n",
    "        return batch\n",
    "        \n",
    "    return padder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "test 1085\n"
     ]
    }
   ],
   "source": [
    "src = None\n",
    "test_dataset_iter = GittablesTablewiseIterateDataset(cv=cv,\n",
    "                            split=\"test\", src=src,\n",
    "                            tokenizer=tokenizer,\n",
    "                            max_length=max_length,\n",
    "                            gt_only='all' not in task,\n",
    "                            device=device,\n",
    "                            base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "                            small_tag=\"semi1\")\n",
    "padder = collate_fn(tokenizer.pad_token_id)\n",
    "test_dataloader_iter = DataLoader(test_dataset_iter,\n",
    "                                batch_size=1,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "test 1085\n"
     ]
    }
   ],
   "source": [
    "src = None\n",
    "test_dataset_iter_all = GittablesTablewiseIterateDataset(cv=cv,\n",
    "                            split=\"test\", src=src,\n",
    "                            tokenizer=tokenizer,\n",
    "                            max_length=max_length,\n",
    "                            gt_only='all' not in task,\n",
    "                            device=device,\n",
    "                            max_unlabeled=-1,\n",
    "                            base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "                            small_tag=\"semi1\")\n",
    "padder = collate_fn(tokenizer.pad_token_id)\n",
    "test_dataloader_iter_all = DataLoader(test_dataset_iter_all,\n",
    "                                batch_size=1,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols_all = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter_all):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    num_cols_all.append(batch[\"target_col_mask\"].max().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAATVklEQVR4nO3dfbBkdX3n8fcHCCYRBYTZKZ5mBzajWdfdDNaNmGiEaLJBdyIx5SLUrCGiXK2VRSMkUbO1ZteyKsliTFg3uFeZRasUYXlYiTFGlhDIVkXijFAzg0AEAus8yEyGOFqagox+948+99Azty9zh7ndp+/t96uqq/t8z+nu76kD9zPndx46VYUkSQBHdN2AJGl8GAqSpJahIElqGQqSpJahIElqHdV1A4fjxBNPrNWrV3fdhiQtKZs2bfq7qloxaN6SDoXVq1ezcePGrtuQpCUlyWPzzXP4SJLUMhQkSS1DQZLUMhQkSS1DQZLUMhQkSS1DQZLUGlooJNmQZFeSrX2165Pc2zweTXJvU1+d5B/65n1sWH1JkuY3zIvXrgU+CnxqtlBVb5p9neTDwN6+5R+uqrVD7EeSdBBDC4WquivJ6kHzkgQ4H3j1sL5/sU1fdgU79uzdr3byCccyc9WVHXUkSYuvq9tc/AzweFV9va92epJ7gG8D/7Gq/nLQG5NMA9MAq1atGnqjs3bs2csxZ1+8f+3ODSP7fkkaha4ONF8IXNc3vRNYVVVnAu8BPpPk+YPeWFUzVTVVVVMrVgy8n5Mk6VkaeSgkOQr4ZeD62VpVPVlVe5rXm4CHgReOujdJmnRd7Cn8HPBAVW2bLSRZkeTI5vUZwBrgkQ56k6SJNsxTUq8D/gp4UZJtSd7azLqA/YeOAF4FbG5OUb0ReEdVPTGs3iRJgw3z7KML56n/6oDaTcBNw+pFkrQwXtEsSWoZCpKk1pL+Oc6ubd2yhXXrL5lT96I2SUuVoXAYnqoj5lzQBl7UJmnpcvhIktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJLUNBktQyFCRJraGFQpINSXYl2dpX++0k25Pc2zxe1zfvfUkeSvJgkl8YVl+SpPkNc0/hWuDcAfWPVNXa5vEFgCQvBi4A/kXznj9KcuQQe5MkDTC0UKiqu4AnFrj4ecBnq+rJqvpb4CHgZcPqTZI0WBfHFC5NsrkZXjq+qZ0CfKNvmW1NbY4k00k2Jtm4e/fuYfcqSRPlqBF/39XAB4Fqnj8MXHwoH1BVM8AMwNTUVC12g9OXXcGOPXvn1O+7/0HOOnuxv02SxstIQ6GqHp99neTjwOebye3AaX2LntrURm7Hnr0cc/bcnHpy8+UddCNJozXS4aMkJ/VNvgGYPTPpVuCCJM9JcjqwBvjrUfYmSRrinkKS64BzgBOTbAM+AJyTZC294aNHgbcDVNV9SW4AvgbsA95ZVd8fVm+SpMGGFgpVdeGA8jXPsPyHgA8Nqx9J0sF5RbMkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJahoIkqTW0UEiyIcmuJFv7av81yQNJNie5JclxTX11kn9Icm/z+Niw+pIkzW+YewrXAuceULsNeElV/Svgb4D39c17uKrWNo93DLEvSdI8hhYKVXUX8MQBtS9V1b5m8svAqcP6fknSoevymMLFwJ/2TZ+e5J4kdyb5mfnelGQ6ycYkG3fv3j38LiVpgnQSCkl+C9gHfLop7QRWVdWZwHuAzyR5/qD3VtVMVU1V1dSKFStG07AkTYiRh0KSXwXWAeurqgCq6smq2tO83gQ8DLxw1L1J0qQbaSgkORf4DeD1VfW9vvqKJEc2r88A1gCPjLI3SRIcNawPTnIdcA5wYpJtwAfonW30HOC2JABfbs40ehXwX5L8I/AD4B1V9cTAD5YkDc3QQqGqLhxQvmaeZW8CbhpWL5KkhfGKZklSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSa2h3SZ1kW7dsYd36S/arnXzCscxcdWVHHUnSwhgKQ/BUHcExZ1+8X23HnRs66kaSFs7hI0lSy1CQJLUMBUlSy1CQJLUMBUlSa0GhkOQVC6kNWGZDkl1JtvbVXpDktiRfb56Pb+pJclWSh5JsTvLSQ1kRSdLhW+iewn9bYO1A1wLnHlB7L3B7Va0Bbm+mAV4LrGke08DVC+xNkrRInvE6hSQ/Bfw0sCLJe/pmPR848mAfXlV3JVl9QPk84Jzm9SeBvwB+s6l/qqoK+HKS45KcVFU7F7AekqRFcLA9haOBY+iFx/P6Ht8G3vgsv3Nl3x/6bwIrm9enAN/oW25bU9tPkukkG5Ns3L1797NsQZI0yDPuKVTVncCdSa6tqscW+8urqpLUIb5nBpgBmJqaOqT3SpKe2UJvc/GcJDPA6v73VNWrn8V3Pj47LJTkJGBXU98OnNa33KlNTZI0IgsNhf8FfAz4BPD9w/zOW4GLgN9pnj/XV780yWeBs4C9Hk+QpNFaaCjsq6pDPhsoyXX0DiqfmGQb8AF6YXBDkrcCjwHnN4t/AXgd8BDwPeAth/p9kqTDs9BQ+OMk/x64BXhytlhVTzzTm6rqwnlmvWbAsgW8c4H9SJKGYKGhcFHz/Ot9tQLOWNx2JEldWlAoVNXpw25EktS9BYVCkl8ZVK+qTy1uO5KkLi10+Ogn+17/ML1jAl8FDAVJWkYWOnz0H/qnkxwHfHYYDUmSuvNsb539XcDjDJK0zCz0mMIf0zvbCHo3wvvnwA3DakqS1I2FHlO4su/1PuCxqto2hH6Wra1btrBu/SVz6iefcCwzV1054B2SNHoLPaZwZ5KVPH3A+evDa2l5eqqO4JizL55T33Hnhg66kaTBFvrLa+cDfw38W3q3pbg7ybO9dbYkaUwtdPjot4CfrKpdAElWAP8HuHFYjUmSRm+hZx8dMRsIjT2H8F5J0hKx0D2FLyb5M+C6ZvpN9O5qKklaRg72G80/Ru/nM389yS8Dr2xm/RXw6WE3J0karYPtKfwB8D6AqroZuBkgyb9s5v3iEHuTJI3YwY4LrKyqLQcWm9rqoXQkSerMwULhuGeY9yOL2IckaQwcLBQ2JplzGW6StwGbhtOSJKkrBzum8G7gliTreToEpoCjgTcMsS9JUgeeMRSq6nHgp5P8LPCSpvwnVfXnQ+9MkjRyC7330R3AHYvxhUleBFzfVzoD+E/0jl9cAuxu6u+vKq+FkKQRWujFa4umqh4E1gIkORLYDtwCvAX4SFV5y1BJ6kjXt6p4DfBwVT3WcR+SJLoPhQt4+tYZAJcm2ZxkQ5Lju2pKkibVyIePZiU5Gng9zRXTwNXAB+n9wtsHgQ8Dc36AIMk0MA2watWqkfQ6TIN+fMcf3pHUlc5CAXgt8NXmDKfZM50ASPJx4POD3lRVM8AMwNTUVA1aZikZ9OM7/vCOpK50OXx0IX1DR0lO6pv3BmDryDuSpAnXyZ5CkucCPw+8va/8e0nW0hs+evSAeZKkEegkFKrqu8AJB9Te3EUvkqSndX32kSRpjBgKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqSWoSBJahkKkqRWl7/RrHls3bKFdesvmVM/+YRjmbnqyg46kjQpDIUx9FQdwTFnXzynvuPODR10I2mSOHwkSWoZCpKklqEgSWp1dkwhyaPAd4DvA/uqairJC4DrgdXAo8D5VfX3XfUoSZOm6z2Fn62qtVU11Uy/F7i9qtYAtzfTkqQRGbezj84DzmlefxL4C+A3u2pm3Aw6VdXTVCUtpi5DoYAvJSngf1TVDLCyqnY2878JrDzwTUmmgWmAVatWjarXsTDoVFVPU5W0mLoMhVdW1fYk/wS4LckD/TOrqprA4ID6DDADMDU1NWe+JOnZ6+yYQlVtb553AbcALwMeT3ISQPO8q6v+JGkSdRIKSZ6b5Hmzr4F/DWwFbgUuaha7CPhcF/1J0qTqavhoJXBLktkePlNVX0zyFeCGJG8FHgPO76g/SZpInYRCVT0C/MSA+h7gNaPvSJIE3V+nIEkaI4aCJKllKEiSWoaCJKllKEiSWoaCJKllKEiSWoaCJKllKEiSWoaCJKllKEiSWoaCJKk1bj/HqUUyfdkV7Nizd7+aP90p6WAMhWVqx569/nSnpEPm8JEkqeWewhK3dcsW1q2/ZE79vvsf5KyzO2hI0pJmKCxxT9URc4aJAJ7cfPmc2nwB4rEGSbMMhQkyX4B4rEHSLI8pSJJahoIkqTXyUEhyWpI7knwtyX1J3tXUfzvJ9iT3No/Xjbo3SZp0XRxT2AdcXlVfTfI8YFOS25p5H6kqj3hKUkdGHgpVtRPY2bz+TpL7gVNG3Yckaa5OjykkWQ2cCdzdlC5NsjnJhiTHz/Oe6SQbk2zcvXv3qFqVpInQWSgkOQa4CXh3VX0buBr4Z8BaensSHx70vqqaqaqpqppasWLFqNqVpInQSSgk+SF6gfDpqroZoKoer6rvV9UPgI8DL+uiN0maZF2cfRTgGuD+qvr9vvpJfYu9Adg66t4kadJ1cfbRK4A3A1uS3NvU3g9cmGQtUMCjwNs76E2SJloXZx/9XyADZn1h1L1IkvbnFc2SpJahIElqGQqSpJahIElqGQqSpJahIElqGQqSpJY/x6mxMH3ZFezYs3dO3d+PlkbLUNBY2LFnr78fLY0Bh48kSS33FLQoBg3/OPQjLT2GghbFoOEfh36kpcdQ0NBs3bKFdesvmVN3D0IaX4aChuapOmLgweMvffTX5oTFffc/yFlnz/2MQcFiqEjDYyho5AaFxZObL1/wsoNCBQwLaTEYClpy5tsD8RiGdPg8JVWS1HJPQYdkviuP5zsmIGlpMRR0SAdz57vyeL5jAqN0uAelR32rDW/toXFkKGjgGP1SHJ8/3PUY9a02vLWHxpGhoIk06F/pDoFpHI16j3LsQiHJucAfAkcCn6iq3+m4pYk034VnS+0P5zOuxzt+d7/afENggz7jkb95gDNe+ONzll2M/1GHNQw2qGeHqsbfqPcoxyoUkhwJ/Hfg54FtwFeS3FpVX+u2s8kz32mf43Ds4FAsxnoM+oxvbb58aP+jDmsYbFDPDlXpQGMVCsDLgIeq6hGAJJ8FzgOGEgoOIWipWM63DFmM4ZGldtB+nPtNVXXaQL8kbwTOraq3NdNvBs6qqkv7lpkGppvJFwEPHsZXngj83WG8fylwHZePSVjPSVhH6H49/2lVrRg0Y9z2FA6qqmaAmcX4rCQbq2pqMT5rXLmOy8ckrOckrCOM93qO2xXN24HT+qZPbWqSpBEYt1D4CrAmyelJjgYuAG7tuCdJmhhjNXxUVfuSXAr8Gb1TUjdU1X1D/MpFGYYac67j8jEJ6zkJ6whjvJ5jdaBZktStcRs+kiR1yFCQJLUmMhSSnJvkwSQPJXlv1/0shiSnJbkjydeS3JfkXU39BUluS/L15vn4rntdDEmOTHJPks8306cnubvZptc3JyosWUmOS3JjkgeS3J/kp5bjtkzya81/r1uTXJfkh5f6tkyyIcmuJFv7agO3XXquatZ1c5KXdtd5z8SFQt+tNF4LvBi4MMmLu+1qUewDLq+qFwMvB97ZrNd7gdurag1wezO9HLwLuL9v+neBj1TVjwF/D7y1k64Wzx8CX6yqHwd+gt66LqttmeQU4DJgqqpeQu/kkgtY+tvyWuDcA2rzbbvXAmuaxzRw9Yh6nNfEhQJ9t9KoqqeA2VtpLGlVtbOqvtq8/g69PyKn0Fu3TzaLfRL4pU4aXERJTgX+DfCJZjrAq4Ebm0WW9HomORZ4FXANQFU9VVXfYhluS3pnQP5IkqOAHwV2ssS3ZVXdBTxxQHm+bXce8Knq+TJwXJKTRtLoPCYxFE4BvtE3va2pLRtJVgNnAncDK6tqZzPrm8DKrvpaRH8A/Abwg2b6BOBbVbWvmV7q2/R0YDfwP5shsk8keS7LbFtW1XbgSuD/0QuDvcAmlte2nDXfthu7v0eTGArLWpJjgJuAd1fVt/vnVe/84yV9DnKSdcCuqtrUdS9DdBTwUuDqqjoT+C4HDBUtk215PL1/KZ8OnAw8l7nDLsvOuG+7SQyFZXsrjSQ/RC8QPl1VNzflx2d3R5vnXV31t0heAbw+yaP0hv5eTW/8/bhmCAKW/jbdBmyrqrub6RvphcRy25Y/B/xtVe2uqn8Ebqa3fZfTtpw137Ybu79HkxgKy/JWGs24+jXA/VX1+32zbgUual5fBHxu1L0tpqp6X1WdWlWr6W27P6+q9cAdwBubxZb0elbVN4FvJHlRU3oNvdvHL6ttSW/Y6OVJfrT573d2PZfNtuwz37a7FfiV5iyklwN7+4aZOjGRVzQneR29cenZW2l8qNuODl+SVwJ/CWzh6bH299M7rnADsAp4DDi/qg48CLYkJTkHuKKq1iU5g96ewwuAe4B/V1VPdtjeYUmylt6B9KOBR4C30PtH3LLalkn+M/AmemfP3QO8jd6Y+pLdlkmuA86hd3vsx4EPAP+bAduuCcOP0hs2+x7wlqra2EHbrYkMBUnSYJM4fCRJmoehIElqGQqSpJahIElqGQqSpJahIElqGQqSpNb/B+H6Ey5trIEhAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(num_cols_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1\n",
      "train 2\n",
      "train 3\n",
      "train 4\n",
      "train 3463\n"
     ]
    }
   ],
   "source": [
    "train_dataset_iter = GittablesTablewiseIterateDataset(cv=cv,\n",
    "                            split=\"train\", src=src,\n",
    "                            tokenizer=tokenizer,\n",
    "                            max_length=max_length,\n",
    "                            gt_only='all' not in task,\n",
    "                            device=device,\n",
    "                            base_dirpath=os.path.join(args.data_path, \"GitTables/semtab_gittables/2022\"),\n",
    "                            small_tag=\"semi1\")\n",
    "padder = collate_fn(tokenizer.pad_token_id)\n",
    "train_dataloader_iter = DataLoader(train_dataset_iter,\n",
    "                                batch_size=1,\n",
    "                            #   collate_fn=collate_fn)\n",
    "                            collate_fn=padder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_test_origin \n",
    "logits_test_origin \n",
    "preds_test_origin "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_test_origin.equal(logits_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_test.equal(labels_test_origin )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 507,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 tensor(0.0009)\n"
     ]
    }
   ],
   "source": [
    "print((num_cols==0).sum().item(), (num_cols==0).sum()/len(num_cols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.5355, ts_macro_f1=0.2745\n",
      "ts_micro_f1=0.5351, ts_macro_f1=0.2745\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "num_cols = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits, embs = model(batch[\"data\"].T, cls_indexes=cls_indexes, get_enc=True)\n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    ft_embs_test.append(embs.detach().cpu().reshape(1, -1))\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "ft_embs_test = torch.cat(ft_embs_test, dim=0)\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MahalanobisDistance(object):\n",
    "    def __init__(self, ftrain, labels_train):\n",
    "        self.xc = [ftrain[labels_train == i] for i in np.unique(labels_train)]\n",
    "        self.cov_inv = torch.linalg.pinv(torch.Tensor(np.cov(ftrain.T, bias=True))).numpy()\n",
    "    def __call__(self, ftest):\n",
    "        dtest = [\n",
    "            np.sum(\n",
    "                (ftest - np.mean(x, axis=0, keepdims=True))\n",
    "                * (\n",
    "                    self.cov_inv.dot(\n",
    "                        (ftest - np.mean(x, axis=0, keepdims=True)).T\n",
    "                    )\n",
    "                ).T,\n",
    "                axis=-1,\n",
    "            )\n",
    "            for x in self.xc\n",
    "        ]\n",
    "\n",
    "        dtest = np.min(dtest, axis=0)\n",
    "\n",
    "        return dtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/4049685968.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ft_embs_train = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/training_embeddings.pt\")\n",
      "/tmp/ipykernel_3443360/4049685968.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  labels_train = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/training_labels.pt\")\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sns' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m mah_dist \u001b[38;5;241m=\u001b[39m MahalanobisDistance(F\u001b[38;5;241m.\u001b[39mnormalize(ft_embs_train)\u001b[38;5;241m.\u001b[39mnumpy(), labels_train\u001b[38;5;241m.\u001b[39mnumpy())\n\u001b[1;32m      4\u001b[0m mah_ood_score \u001b[38;5;241m=\u001b[39mmah_dist(ft_embs_test\u001b[38;5;241m.\u001b[39mnumpy())\n\u001b[0;32m----> 5\u001b[0m \u001b[43msns\u001b[49m\u001b[38;5;241m.\u001b[39mhistplot(mah_ood_score)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sns' is not defined"
     ]
    }
   ],
   "source": [
    "ft_embs_train = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/training_embeddings.pt\")\n",
    "labels_train = torch.load(\"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/training_labels.pt\")\n",
    "mah_dist = MahalanobisDistance(F.normalize(ft_embs_train).numpy(), labels_train.numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAATJElEQVR4nO3df6zd9X3f8ecrNpCosALlznLs65mkXjvarg5yGCFR1UGTEKTJZKKJoymxVlqjBaZEy6rBIm2JNKR0asKWqSOBwnC6LEATItyKNSUENarSQAx1CD9CcQip7RjsBkLSSaMzee+P87lfjsz19bU533PO9X0+pKPzPZ/v93zv63651y++P873pqqQJAngVZMOIEmaHpaCJKljKUiSOpaCJKljKUiSOisnHeCVOOuss2r9+vWTjiFJS8oDDzzwN1U1M9+8JV0K69evZ+fOnZOOIUlLSpLvHWmeh48kSR1LQZLUsRQkSR1LQZLUsRQkSR1LQZLUsRQkSR1LQZLUsRQkSZ1lWwprZteRZCSPNbPrJv3tSNJILOnbXLwS39+7h3d/+msjWddtV1wwkvVI0qQt2z0FSdLLWQqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnq9FYKSV6d5P4k30zySJKPtvGzk9yXZHeS25Kc3MZPaa93t/nr+8omSZpfn3sKLwAXVtUvAxuBi5OcD/wOcF1V/SzwHHB5W/5y4Lk2fl1bTpI0Rr2VQg38bXt5UnsUcCHw+Ta+Hbi0TW9ur2nzL0qSvvJJkl6u13MKSVYk2QUcAO4GvgP8sKoOtUX2Amva9BpgD0Cb/zzwM/Osc1uSnUl2Hjx4sM/4krTs9FoKVfViVW0E1gLnAT8/gnXeUFWbqmrTzMzMK12dJGnIWK4+qqofAvcCbwJOT7KyzVoL7GvT+4BZgDb/p4EfjCOfJGmgz6uPZpKc3qZfA7wVeIxBOVzWFtsK3Nmmd7TXtPlfqarqK58k6eVWHn2R47Ya2J5kBYPyub2q/jjJo8CtSf4T8JfATW35m4A/SLIbeBbY0mM2SdI8eiuFqnoIeMM8408yOL9w+Pj/BX69rzySpKPzE82SpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnqWAqSpI6lIEnq9FYKSWaT3Jvk0SSPJPlAG/9Ikn1JdrXHJUPvuSbJ7iSPJ3l7X9kkSfNb2eO6DwEfqqoHk5wGPJDk7jbvuqr63eGFk5wDbAF+AXgt8OUk/7CqXuwxoyRpSG97ClW1v6oebNM/Bh4D1izwls3ArVX1QlV9F9gNnNdXPknSy43lnEKS9cAbgPva0FVJHkpyc5Iz2tgaYM/Q2/YyT4kk2ZZkZ5KdBw8e7DO2JC07vZdCklOBLwAfrKofAdcDrwc2AvuBjx/L+qrqhqraVFWbZmZmRh1Xkpa1XkshyUkMCuGzVXUHQFU9U1UvVtVPgBt56RDRPmB26O1r25gkaUz6vPoowE3AY1X1iaHx1UOLvRN4uE3vALYkOSXJ2cAG4P6+8kmSXq7Pq4/eDLwX+FaSXW3s3wPvSbIRKOAp4AqAqnokye3AowyuXLrSK48kabx6K4Wq+nMg88y6a4H3XAtc21cmSdLC/ESzJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOpaCJKljKUiSOr2VQpLZJPcmeTTJI0k+0MbPTHJ3kifa8xltPEk+mWR3koeSnNtXNknS/PrcUzgEfKiqzgHOB65Mcg5wNXBPVW0A7mmvAd4BbGiPbcD1PWaTJM2jt1Koqv1V9WCb/jHwGLAG2Axsb4ttBy5t05uBz9TA14HTk6zuK58k6eXGck4hyXrgDcB9wKqq2t9mPQ2satNrgD1Db9vbxg5f17YkO5PsPHjwYH+hJWkZ6r0UkpwKfAH4YFX9aHheVRVQx7K+qrqhqjZV1aaZmZkRJpUk9VoKSU5iUAifrao72vAzc4eF2vOBNr4PmB16+9o2Jkkakz6vPgpwE/BYVX1iaNYOYGub3grcOTT+vnYV0vnA80OHmSRJY7CoUkjy5sWMHebNwHuBC5Psao9LgI8Bb03yBPBr7TXAXcCTwG7gRuD9i/sWJEmjsnKRy/034PDPDcw31qmqPwdyhNkXzbN8AVcuMo8kqQcLlkKSNwEXADNJ/s3QrL8HrOgzmCRp/I62p3AycGpb7rSh8R8Bl/UVSpI0GQuWQlX9GfBnSW6pqu+NKZMkaUIWe07hlCQ3AOuH31NVF/YRSpI0GYsthT8EPgX8PvBif3EkSZO02FI4VFXeoE6STnCL/fDaHyV5f5LV7dbXZyY5s9dkkqSxW+yewtwnkH97aKyA1402jiRpkhZVClV1dt9BJEmTt6hSSPK++car6jOjjSNJmqTFHj5649D0qxncpuJBwFKQpBPIYg8f/evh10lOB27tI5AkaXKO99bZ/wfwPIMknWAWe07hj3jpL6StAP4RcHtfoSRJk7HYcwq/OzR9CPheVe3tIY8kaYIWdfio3Rjv2wzulHoG8Hd9hpIkTcZi//Lau4D7gV8H3gXcl8RbZ0vSCWaxh48+DLyxqg4AJJkBvgx8vq9gkqTxW+zVR6+aK4TmB8fwXknSErHYPYU/SfIl4HPt9buBu/qJJEmalKP9jeafBVZV1W8n+efAW9qsvwA+23c4SdJ4HW1P4b8A1wBU1R3AHQBJfqnN+2c9ZpMkjdnRzgusqqpvHT7YxtYv9MYkNyc5kOThobGPJNmXZFd7XDI075oku5M8nuTtx/h9SJJG4GilcPoC815zlPfeAlw8z/h1VbWxPe4CSHIOsAX4hfae/55kxVHWL0kasaOVws4kv3X4YJLfBB5Y6I1V9VXg2UXm2AzcWlUvVNV3gd3AeYt8ryRpRI52TuGDwBeT/AteKoFNwMnAO4/za17V/j7DTuBDVfUcsAb4+tAye9vYyyTZBmwDWLdu3XFGkCTNZ8E9hap6pqouAD4KPNUeH62qN1XV08fx9a4HXg9sBPYDHz/WFVTVDVW1qao2zczMHEcESdKRLPbvKdwL3PtKv1hVPTM3neRG4I/by33A7NCia9uYJGmMxvqp5CSrh16+E5i7MmkHsCXJKUnOBjYwuNeSJGmMFvuJ5mOW5HPArwJnJdkL/EfgV5NsZPC3GZ4CrgCoqkeS3A48yuDW3FdW1Yt9ZZMkza+3Uqiq98wzfNMCy18LXNtXHknS0XlTO0lSx1KQJHUsBUlSx1KQJHUsBUlSx1KQJHUsBUlSx1IYhVetJMlIHmtmvcmfpMnp7cNry8pPDvHuT39tJKu67YoLRrIeSToe7ilIkjqWgiSpYylIkjqWgiSpYylIkjqWgiSpYylIkjqWgiSpYylIkjqWgiSpYylIkjqWgiSpYylIkjq9lUKSm5McSPLw0NiZSe5O8kR7PqONJ8knk+xO8lCSc/vKJUk6sj73FG4BLj5s7GrgnqraANzTXgO8A9jQHtuA63vMJUk6gt5Koaq+Cjx72PBmYHub3g5cOjT+mRr4OnB6ktV9ZZMkzW/c5xRWVdX+Nv00sKpNrwH2DC23t429TJJtSXYm2Xnw4MH+kkrSMjSxE81VVUAdx/tuqKpNVbVpZmamh2SStHyNuxSemTss1J4PtPF9wOzQcmvbmCRpjMZdCjuArW16K3Dn0Pj72lVI5wPPDx1mkiSNSZ+XpH4O+Avg55LsTXI58DHgrUmeAH6tvQa4C3gS2A3cCLy/r1zLyZrZdSQZ2WPN7LpJf0uSerayrxVX1XuOMOuieZYt4Mq+siwpr1pJkpGt7t2f/trI1nXbFReMbF2SplNvpaDj9JNDI/uH3H/EJR0rb3MhSepYCpKkjqUgSepYClryRnmVlVdYabnzRLMWb4RXRr127Sz79vz1SNb1/b17PDkvjYiloMXzyijphOfhI0lSxz0FTcaIP6QnaTQsBU2Gh6KkqeThI0lSx1KQJHUsBUlSx1KQJHUsBUlSx1KQJHUsBUlSx1KQJHUsBWlY+6S1d1zVcuUnmqVhftJay5x7CpKkjqUgSepM5PBRkqeAHwMvAoeqalOSM4HbgPXAU8C7quq5SeSTpOVqknsK/7SqNlbVpvb6auCeqtoA3NNeS5LGaJoOH20Gtrfp7cClk4siScvTpEqhgD9N8kCSbW1sVVXtb9NPA6smE02Slq9JXZL6lqral+TvA3cn+fbwzKqqJDXfG1uJbANYt87rwCVplCayp1BV+9rzAeCLwHnAM0lWA7TnA0d47w1VtamqNs3MzIwrsiQtC2MvhSQ/leS0uWngbcDDwA5ga1tsK3DnuLNJ0nI3icNHq4Avtj/avhL4X1X1J0m+Adye5HLge8C7JpBNkpa1sZdCVT0J/PI84z8ALhp3HknSS6bpklRJR7Bmdp036tNYeEM8qS/tjquj4o36NA6WgtQX77iqJcjDR5KkjqUgSepYCpKkjqUgSepYCpKkjqUgSepYCpKkjqUgSepYCpKkjqUgSepYCpKkjqUgSepYCtJy0+7e6m24NR/vkiotN969VQtwT0GS1LEUJB0/D0WdcDx8JOn4eSjqhOOegiSpYylIkjqWgqQTzprZdZ7rOE5Td04hycXAfwVWAL9fVR+bcCRJS8z39+6ZynMda2bX8f29e0ayrteunWXfnr8eybqGTVUpJFkB/B7wVmAv8I0kO6rq0ckmk9S7diXTiWxay2rYVJUCcB6wu6qeBEhyK7AZsBSkE920Xsm0DMpqWKpq0hk6SS4DLq6q32yv3wv8k6q6amiZbcC29vLngMeP40udBfzNK4w7LkslqzlHb6lkXSo5Yelk7TvnP6iqmflmTNuewlFV1Q3ADa9kHUl2VtWmEUXq1VLJas7RWypZl0pOWDpZJ5lz2q4+2gfMDr1e28YkSWMwbaXwDWBDkrOTnAxsAXZMOJMkLRtTdfioqg4luQr4EoNLUm+uqkd6+FKv6PDTmC2VrOYcvaWSdankhKWTdWI5p+pEsyRpsqbt8JEkaYIsBUlSZ9mVQpKLkzyeZHeSq6cgz1NJvpVkV5KdbezMJHcneaI9n9HGk+STLftDSc7tOdvNSQ4keXho7JizJdnaln8iydYx5fxIkn1tu+5KcsnQvGtazseTvH1ovNefjSSzSe5N8miSR5J8oI1P1TZdIOc0btNXJ7k/yTdb1o+28bOT3Ne+7m3twhWSnNJe727z1x/te+g55y1Jvju0TTe28Yn9PlFVy+bB4OT1d4DXAScD3wTOmXCmp4CzDhv7z8DVbfpq4Hfa9CXA/wYCnA/c13O2XwHOBR4+3mzAmcCT7fmMNn3GGHJ+BPi38yx7Tvvvfgpwdvt5WDGOnw1gNXBumz4N+KuWZ6q26QI5p3GbBji1TZ8E3Ne21e3Aljb+KeBften3A59q01uA2xb6HsaQ8xbgsnmWn9jv03LbU+huo1FVfwfM3UZj2mwGtrfp7cClQ+OfqYGvA6cnWd1XiKr6KvDsK8z2duDuqnq2qp4D7gYuHkPOI9kM3FpVL1TVd4HdDH4uev/ZqKr9VfVgm/4x8BiwhinbpgvkPJJJbtOqqr9tL09qjwIuBD7fxg/fpnPb+vPARUmywPfQd84jmdjv03IrhTXA8C0K97LwD/s4FPCnSR7I4BYeAKuqan+bfhpY1aanIf+xZptk5qvarvfNc4dkFsgz1pztsMUbGPwf49Ru08NywhRu0yQrkuwCDjD4R/I7wA+r6tA8X7fL1OY/D/zMOLIenrOq5rbptW2bXpfklMNzHpan95zLrRSm0Vuq6lzgHcCVSX5leGYN9hmn8rrhac4GXA+8HtgI7Ac+PtE0Q5KcCnwB+GBV/Wh43jRt03lyTuU2raoXq2ojgzsgnAf8/GQTze/wnEl+EbiGQd43Mjgk9O8ml3BguZXC1N1Go6r2tecDwBcZ/FA/M3dYqD0faItPQ/5jzTaRzFX1TPsl/AlwIy8dCphoziQnMfiH9rNVdUcbnrptOl/Oad2mc6rqh8C9wJsYHG6Z+3Du8NftMrX5Pw38YJxZh3Je3A7VVVW9APwPpmCbLrdSmKrbaCT5qSSnzU0DbwMebpnmrirYCtzZpncA72tXJpwPPD902GFcjjXbl4C3JTmjHW54Wxvr1WHnWt7JYLvO5dzSrkI5G9gA3M8YfjbaseubgMeq6hNDs6Zqmx4p55Ru05kkp7fp1zD4WyyPMfhH97K22OHbdG5bXwZ8pe2dHel76DPnt4f+ZyAMznsMb9PJ/D6N8qz1UngwOKv/VwyOO354wllex+CKh28Cj8zlYXCM8x7gCeDLwJn10hUMv9eyfwvY1HO+zzE4TPD/GBy7vPx4sgG/weDE3W7gX44p5x+0HA8x+AVbPbT8h1vOx4F3jOtnA3gLg0NDDwG72uOSadumC+Scxm36j4G/bJkeBv7D0O/W/W37/CFwSht/dXu9u81/3dG+h55zfqVt04eB/8lLVyhN7PfJ21xIkjrL7fCRJGkBloIkqWMpSJI6loIkqWMpSJI6loIkqWMpSJI6/x+uRnq9DSaqkAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "mah_ood_score =mah_dist(F.normalize(ft_embs_test).numpy())\n",
    "sns.histplot(mah_ood_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/327555800.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n",
      "/tmp/ipykernel_3443360/327555800.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  roc_auc_score(ood_labels, torch.tensor(ood_score))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8561098134032731"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "ood_score = F.softmax(logits_test, dim=1).max(1).values\n",
    "ood_labels = torch.tensor(preds_test == labels_test).float()\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(ood_labels, torch.tensor(ood_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.9998, 0.9996, 0.7101,  ..., 0.3492, 0.3524, 0.9998])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ood_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8493997924626198 0.4416666666666667 0.3 0.9373271889400921\n",
      "0.8312882376159464 0.4367283950617284 0.5 0.8506912442396314\n",
      "0.7862074634226532 0.5657214196540039 0.8 0.7152073732718894\n",
      "0.770558580787884 0.6570758928571429 0.9 0.6405529953917051\n",
      "0.7224813002123927 0.7077736825318071 0.99 0.4976958525345622\n"
     ]
    }
   ],
   "source": [
    "for threshold in [0.3, 0.5, 0.8, 0.9, 0.99]:\n",
    "    print(roc_auc_score(ood_labels[ood_score>threshold], ood_score[ood_score>threshold]), roc_auc_score(ood_labels[ood_score<threshold], ood_score[ood_score<threshold]), threshold, len(ood_score[ood_score>threshold])/len(ood_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.8592)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ood_score[ood_labels==0] < 0.99).sum()/sum(ood_labels==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7678)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "(ood_score[ood_labels==0] < threshold).sum()/sum(ood_score<threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4406, ts_macro_f1=0.1616\n",
      "ts_micro_f1=0.4400, ts_macro_f1=0.1616\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "# Single column results\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "num_cols = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, 0]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T[:, target_col_mask[0]==0], cls_indexes=cls_indexes,)\n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4903, ts_macro_f1=0.2121\n",
      "ts_micro_f1=0.4974, ts_macro_f1=0.2227\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# Cheat drop one-column\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "logits_init = []\n",
    "logits_correct = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = batch[\"data\"].T[target_col_mask!=col_i].reshape(1, -1)\n",
    "            target_col_mask_temp = target_col_mask[target_col_mask!=col_i].reshape(1, -1)\n",
    "            cls_indexes = (target_col_mask_temp == 0).nonzero()[0].reshape(1, -1)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits_init.append(logits.detach().cpu())\n",
    "                logits = logits_temp.clone()\n",
    "                \n",
    "                corrected += 1\n",
    "\n",
    "            \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cheat drop one-column\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "logits_init = []\n",
    "logits_correct = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = batch[\"data\"].T[target_col_mask!=col_i].reshape(1, -1)\n",
    "            target_col_mask_temp = target_col_mask[target_col_mask!=col_i].reshape(1, -1)\n",
    "            cls_indexes = (target_col_mask_temp == 0).nonzero()[0].reshape(1, -1)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits_init.append(logits.detach().cpu())\n",
    "                logits_correct.append(logits_temp.detach().cpu())\n",
    "                corrected += 1\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cheat brute force permutation\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "logits_init = []\n",
    "logits_correct = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = batch[\"data\"].T[target_col_mask!=col_i].reshape(1, -1)\n",
    "            target_col_mask_temp = target_col_mask[target_col_mask!=col_i].reshape(1, -1)\n",
    "            cls_indexes = (target_col_mask_temp == 0).nonzero()[0].reshape(1, -1)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits_init.append(logits.detach().cpu())\n",
    "                logits_correct.append(logits_temp.detach().cpu())\n",
    "                corrected += 1\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cheat drop one-column\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "logits_init = []\n",
    "logits_correct = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = batch[\"data\"].T[target_col_mask!=col_i].reshape(1, -1)\n",
    "            target_col_mask_temp = target_col_mask[target_col_mask!=col_i].reshape(1, -1)\n",
    "            cls_indexes = (target_col_mask_temp == 0).nonzero()[0].reshape(1, -1)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits_init.append(logits.detach().cpu())\n",
    "                logits_correct.append(logits_temp.detach().cpu())\n",
    "                corrected += 1\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(logits_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "314"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits_init = torch.stack(logits_init, dim=0)\n",
    "logits_correct = torch.stack(logits_correct, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "ood_score_init = F.softmax(logits_init, dim=1).max(1).values\n",
    "ood_score_correct = F.softmax(logits_correct, dim=1).max(1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(23)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((ood_score_correct - ood_score_init)>0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqW0lEQVR4nO3deZgd9X3n+/e36uzn9L5pbbUkQEhg1mZ3WMxijIOIHceOb+Jgx7Gem4x9M3HiTJjMnQyTmcfZxuNMMjO58ozNgD3eCE7wNQZjGywWIRCrQAK0tlpSq/f9rFX1nT9OA0JoaUl9lu7zfT3PebpPnerz+3ap9ak6v/rVr0RVMcYYUzucShdgjDGmvCz4jTGmxljwG2NMjbHgN8aYGmPBb4wxNSZU6QJmo7W1Vbu6uipdhjHGzCvPP//8kKq2Hb18XgR/V1cXW7durXQZxhgzr4hIz7GWW1ePMcbUGAt+Y4ypMRb8xhhTYyz4jTGmxljwG2NMjbHgN8aYGmPBb4wxNcaC3xhjaowFvzHG1BgLfmNMzenq7ERE5sWjq7Nzzn//eTFlgzHGzKWe3l68xx6udBmzErrh1jl/TzviN8aYGmPBb4wxNcaC3xhjaowFvzHG1BgLfmOMqTEW/MYYU2Ms+I0xpsZY8BtjTI0pWfCLyNdFZEBEXj3Ga38oIioiraVq3xhjzLGV8oj/HuA9l5yJyHLgFmB/Cds2xhhzHCULflXdBIwc46X/DPwxoKVq2xhjzPGVtY9fRO4ADqrqy+Vs1xhjzDvKNkmbiCSAf02xm2c2628ANgB0lmB2OmOMqVXlPOJfDawEXhaRfcAy4AURWXSslVV1o6p2q2p3W1tbGcs0xpiFrWxH/Kq6DWh/6/lM+Her6lC5ajDGGFPa4ZzfBjYDa0TkgIh8tlRtGWOMmb2SHfGr6idP8npXqdo2xhhzfHblrjHG1BgLfmOMqTEW/MYYU2Ms+I0xpsZY8BtjTI2x4DfGmBpjwW+MMTXGgt8YY2qMBb8xxtQYC35jjKkxFvzGGFNjLPiNMabGWPAbY0yNseA3xpgaY8FvjDE1xoLfGGNqjAW/McbUGAt+Y4ypMRb8xhhTY0p5s/Wvi8iAiLx6xLK/FpHXReQVEfmBiDSWqn1jjDHHVsoj/nuAW49a9ihwvqpeALwJ3FXC9o0xxhxDyYJfVTcBI0ct+4mqejNPnwGWlap9Y4wxx1bJPv7fBn58vBdFZIOIbBWRrYODg2UsyxhjFraKBL+I/CngAd863jqqulFVu1W1u62trXzFGWPMAhcqd4Mi8mngl4EbVVXL3b4xxtS6sga/iNwK/DFwnaqmy9m2McaYolIO5/w2sBlYIyIHROSzwN8DdcCjIvKSiPxDqdo3xhhzbCU74lfVTx5j8f8sVXvGGGNmx67cNcaYGmPBb4wxNcaC3xhjaowFvzHG1BgLfmOMqTEW/MYYU2Ms+I0xpsZY8BtjTI2x4DfGmBpjwW+MMTXGgt8YY2qMBb8xxtQYC35jjKkxFvzGGFNjLPiNMabGWPAbY0yNseA3xpgaY8FvjDE1xoLfGGNqTClvtv51ERkQkVePWNYsIo+KyM6Zr02lat8YY8yxlfKI/x7g1qOW/QnwM1U9G/jZzHNjjDFlVLLgV9VNwMhRi+8A/tfM9/8L+JVStW+MMebYQmVur0NV+2a+Pwx0HG9FEdkAbADo7OwsQ2lmoetc0UXv/p5KlzEryztXsL9nX6XLWJA6V3QBcPfdd1e2kFmob2goyfuWO/jfpqoqInqC1zcCGwG6u7uPu54xs9W7v4dNh6YqXcasXLskVekSFqy3dv6f+eJdFa7k5L7xlS+X5H3LPaqnX0QWA8x8HShz+8YYU/PKHfwPAnfOfH8n8M9lbt8YY2peKYdzfhvYDKwRkQMi8lngL4CbRWQncNPMc2OMMWVUsj5+Vf3kcV66sVRtGmOMOTm7ctcYY2qMBb8xxtQYC35jjKkxFvzGGFNjLPiNMabGWPAbY0yNseA3xpgaY8FvjDE1xoLfGGNqjAW/McbUGAt+Y4ypMbMKfhG5ZjbLjDHGVL/ZHvH/3SyXGWOMqXInnJ1TRK4CrgbaROSLR7xUD7ilLMwYY0xpnGxa5giQmlmv7ojlE8DHSlWUMcaY0jlh8KvqL4BfiMg9qjo/7lJtjDHmhGZ7I5aoiGwEuo78GVX9QCmKMsYYUzqzDf7vA/8A/A/AL105xhhjSm22we+p6n+fq0ZF5A+A3wEU2AZ8RlWzc/X+xhhjjm+2wzl/KCK/JyKLRaT5rcfpNCgiS4H/B+hW1fMpjg769dN5L2OMMadutkf8d858/dIRyxRYdQbtxkWkACSAQ6f5PsYYY07RrIJfVVfOVYOqelBE/gbYD2SAn6jqT45eT0Q2ABsAOjs756p5Y4ypebMKfhH5rWMtV9V7T7VBEWkC7gBWAmPA90XkN1X1m0e990ZgI0B3d7eeajvGGGOObbZdPZcd8X0MuBF4ATjl4AduAvaq6iCAiDxA8ergb57wp4wxxsyJ2Xb1fOHI5yLSCHznNNvcD1wpIgmKXT03AltP872MMcacotOdlnmaYlfNKVPVLcD9FD8xbJupYeNp1mGMMeYUzbaP/4cUR/FAcfjlWuB7p9uoqv4Z8Gen+/PGGGNO32z7+P/miO89oEdVD5SgHmOMMSU2q66emcnaXqc4Q2cTkC9lUcYYY0pntnfg+jjwLPBrwMeBLSJi0zIbY8w8NNuunj8FLlPVAQARaQN+SvEkrTHGmHlktqN6nLdCf8bwKfysMcaYKjLbI/6HReQR4Nszzz8BPFSakowxxpTSye65exbQoapfEpGPAu+feWkz8K1SF2eMMWbuneyI/6vAXQCq+gDwAICIvG/mtdtLWJsxxpgSOFnwd6jqtqMXquo2EekqTUlmvuvq7KSnt7fSZRzTtUtS73q+eOkyvvvc6xWqxpjKOFnwN57gtfgc1mEWkJ7eXrzHHq50Ge9x991385kv3vWuZavWr69QNWa+c7w8oWyacDaNm8vi+AUcz0OCAHUExCFwXfxIDC8Sw4vGKcRT4FR+XMzJgn+riHxOVb925EIR+R3g+dKVZYwx1cUp5IhNjBCdHCc6NUYo/87dYhUhCIUI3BDquIgGoIrje7he4Z31RMgn6sgnG8g0tpJP1oNI2X+XkwX/vwR+ICK/wTtB3w1EgI+UsC5jjKk4p5AnMTpAfGyQyNQ4AvhuiHyqkam2pRRiCbxYAj8SO26AS+Dj5rOEM2ki0xNEpidIDR6gbqAXPxQm09TGVOsSvHjqmD9fCicMflXtB64WkRuA82cW/0hVf17yyowxphJUiU6NkRw6RHxsCFGlEEsysbiLbEMrhXjylI7S1XHxYkm8WJJMUxsA4nvExoeJjw2RHOojNXiIXLKBqfalZBrbSv4pYLbz8T8GPFbSSowxppJUiY8OUte/n0hmisANMdW2lOmWxXjx5Nw25YbINHeQae7A8QokhvtIDR2iZe92CrEEE4u7ijuAEpntBVzGGLMwqZIY6afucA/hXIZCNM5I5xrSze3guCVvPgiFmeroZKp9OfHRAer7emjZu518PEVnQ31J2rTgN8bUJlViEyPUH9pDJDNNPp5iaOV5ZBtbK3LCFZHip4CmduKjAzQc2sOdF5/Hj3fumfOmLPiNMTVndXMjrbu3EZsYwYvGGO5aS6apvTKBf7S3dgCNrWy//z5+9KYFvzHGnDYtFPi311/FH73/MiJT44wtW81U69KqGFv/Ho7LlgN95Dx/7t96zt/RGGOqULDrdbz/9lf8m+uv4gfbd3L4vMuZal9enaFfYhX5jUWkUUTuF5HXRWSHiFxViTqMMQufZjN4//wd/G99DUJhbr7n+9z5wI8JwtFKl1Yxlerq+VvgYVX9mIhEgESF6jDGLGDB7jfwH/wuTE7gvP9GnOtu4Ref/1eVLqviyh78ItIAXAt8GkBV89g9fI0xc0jzOYJHHiR44Rlo7cD97KdxlnZWuqyqUYkj/pXAIPANEbmQ4lQQv6+q00euJCIbgA0AnZ32D2aMmR0d6sf73r0w2I9z9fU4N9yKhMKVLquqVCL4Q8AlwBdUdYuI/C3wJ8D/e+RKqroR2AjQ3d2tZa/SLBgKpCWEs3ItvfFmsk6ErBvGE4f1/+ov2DWeRwBHBEcg5AgRByKOEHWLD6mGYX7mpIJXX8R/8HsQDuN+agPOqnMqXVJVqkTwHwAOqOqWmef3Uwx+Y86YAiNOjD43QZ+bpN9NMOLGyEmI2Bcu5Om3V1RCGnDhBz/CSM5HFQIgOMYhhgPEQ0Ii5FAXdkhFHJIhwbGdQdVQzyP4yYMEzz2FLO/C/dinkPrGSpdVtcoe/Kp6WER6RWSNqr4B3AhsL3cdZuHIiMueUAP7QvX0hOqZdoof6yPq0+GnWZcfoSnI8tB9X+eO9b9CLCgQCTwcivPxbzo09fZ7qSq+Qj5Q8r6S9ZWMp2S8gLG8z2C2OKbaEWgIOzREXZqiDolQ7Q0JrBY6NoJ//33owf04V16Hc9OHEbf0Uy3MZ5Ua1fMF4FszI3r2AJ+pUB1mnsqIy+vhZt4MN9Lr1qEiJIICK7xJOr0JlvrTNAdZjjwm//9ff4HG2z54wvcVEUIz3T2Jo/53qCr5QJksKON5n7FcwOhkgX2TkAgJLVGX1phLPGRdQ+US7NyB/4P/DUGA+/E7cdZeUOmS5oWKBL+qvkRxXn9jZi0AekJ1bAu3sivciC8OzX6WK3KHObswRkeQppRxK/JWnz+0xopHlDk/YCQbMJT16Z326J32iLtCe9ylPR4i4toOoBQ0CAgef4TgiZ9CxxJCH78TaW6tdFnzhk3ZYKpeWkK8FGllW6SVCSdKLPC4KD/I+flh2oNMRWuLug6Lkw6LkyHyvjKc8xnK+PRMefRMeTRHHRYlQjRGHPsUMEd0ahL/gW+ie3chF1+O+6GPImEbtXMqLPhN1Rp1omyNtPNqpBVPHFYUJrgue5CzCmOEqL6BXhFXWJwIsTgRIuMF9Gd8BjIeI6N5oo6wOOnSEQ8RcmwHcLqCnj34998H2TTu+k/gXHx5pUualyz4TdXpd+Jsji1mZ6gRF2VdYYTLcv20BNmT/3CViIccuuocOlMhRnIBfdMe+yY9eqc8OuIhliRdoq6dEJ4tVSXY/DjBTx+CpmZCv/E5ZNGSSpc1b1nwm6ox6MR4KraEneEmoupxRe4wl+QHSKlX6dJOmyNCa6x40neyEHBo2uNQuvhoi7ksT4WI24igE9JsBv+fv4O+/iqy9gLc9R9HYvFKlzWvWfCbiht1ojwVXcyOcDMRAq7OHqI710+UoNKlzam6sMOaxggr/OIOoD9dHB7aFnNZlgrZkNBj0L4DeN+/F8ZHcT54B84Vv2TnSuaABb+pmIy4PBVdwkuRNkIEXJE7zGX5fuI69/OPV5OY67CqPsKypHIo7dGX9hjM+rTGXJYnQyTCtgNQVfTFLfgP/QCSSdxP/x7O8pWVLmvBsOA3ZecjvBhp4+nYYvK4XJgf5Kpc37zu0jkdEVfoqguzNBni4HRxBzCU9WmJubSvrN2pBrSQx//RP6Ivb0VWnYP70d9AkqlKl7WgWPCbstodqufnseWMuTG6CuPckD1A6zw6aVsKYeedHcBb5wB+//tP8MN9k1yzKEFzrHauQtXhQbzv3QMD/TjX3YJz7c1IDd4opdRsi5qymJYQD8ZX8kDybFyUj03v5NfSu2o+9I8UdoQVdWG622I8ce/f88ZYjq/tGOVHPZOM5eZH91dXZyciclqPj513DiP/6W6G9u3ltvvuJ3TDrTiue9rvd7yHsSN+U2IKvBpu4fHYMgri8P7sQS7P9eNW4Tj8ahF2hIf/y59z/9/8Ozb3p3lxKMtrIzkuaIlx9aI49ZHq/QTQ09uL99jDp/QzGgTonjfh4H6ob0DWXsjDH7qjRBXC3XffzZ8/vrlk7z8fWPCbkhl1ovwk3sn+UD3LvEluyfTQEuQqXda8kQw73LQsxRXtcTb3Z3hpOMu2kSwXtca4qiNBagGcBNZsFt3+MkyOw9JOZNU51rVTBhb8Zs45oRBbIh08HVuCg3JzpocL80MlnUdnIauLuNyyPMUVHXGeOpzmhcEsLw9luaQtzpXt8Xk7CkiHB9HXXwUNkHUXIG2LKl1SzbDgN3PqcNrjX9z7EzbFl3F2YZSbMr2ktFDpshaEhojLbZ11XNWR4Mm+NM8NZHhxKEN3W5zL2+Pz5kIwDQJ07y44sA9SdcjaC5BEstJl1RQLfjMn8r7y5OFiGKVa2rhjejfneGOVLmtBaoq63N5Vx9WL4jzZl2Zzf4YXBrNc1h6nuz1GrIqngtBsFt3xMkyMw5JlyOo1iFO95ywWKgt+c8b2TuR5uHeK8XzARS0xPnLtNfz5D79X6bIWvJZYiDtW1nNVxuPJvjRPHk6zdTDDFe1xLm2LV92U0O/q2ln7PqR9caVLOi4fYToUZdqNMh2KknMi5JwQeSeE5zgoQiAOoLga4GpAOPCJ+QViQYGEn6POy5L0slU5dNKC35y2jBfws4PTvDqSoznq8n+d3UBnKkx2aqLSpdWU9niIj66q53Da44m+aX7Rl+a5mR3ARa2xik8Gp4E/07XTA8m6Yn9+lXTt+AiT4Thj4QRj4SRj4QTj4QRZN/KedcOBRyTwCGmAaICDAoIvDr445MUl7757emhHA+q8LM35KZrzk7Tmp2golPa+EbNhwW9OmaqyfTTHTw9Ok/OUqzviXL0oYdMNV9iiRIhfW93AwekCT/SleexQmqcPZ7iwNUZ3W6wiw0B1ahJ9fRtMT8GS5cjqcyrateMhOKvO4wNnX81jresYjtbhS3HH6GhAQyHNouwYKS9H0s+S9HIk/RwxvzAT9CcWIGSdMOlQhIlQnMlQnPFwgoOxJvYm2wGI+Xk6suMsyo6xJDtKpAJTlFjwm1Mynvd5ZP8UeyYLLEmEuPWsFO1x+zOqJkuTYX79rAb6pgs8O5DhuYEMWwcyrG2K0t0WY1EiVPILmVQVDu5H9+yEUAg5/2Kkpa2kbR6Lh9DnJukN1dEbSnHITRH7/CXcGATkvTSrpw7TnJ+isZCmzsuccbeMg5II8iTyeVrzR9zLGZh2owxG6zkca+RwrJGeZBuOBizKjrE8PcyyzAihMk1MWLH/sSLiAluBg6r6y5Wqw8xOoMrzg1k29U0DcNPSJJe0xXDsSsiqtTgZ5o6VYa7L+WwdzPDKcI7XRnN0xF0uao2xrilakm4gzWWLffljI9DShpyzDolE57ydYzlW0HvigCrtQYYL84M8/c2v8Rf33sP2b91XlpoABEj5OVLpQVamB1FgOJKiN95Cb7yFQy3NvBB4rEgPsWq6n6ZCuqT1VPJQ7feBHUB9BWswszCQ8fjx/in60h6r68PcsjxFQxVfPWrerTHqctOyFO9fnGD7SI4Xh7I80jvNzw9Os7YpynlNUZanwme8E1dVPnH+GnTrZgh85Ox1sHhpST9deAiH3ST7Qyl6Q3XHDPpOb5Jl3hQxil0qT7z2LNnJ8ZLVNBsCtOanaM1PcdF4D4PRenYn29mTbGdXahHt2XHWTB2CEm27igS/iCwDPgz8R+CLlajBnJwXKE8dTrOlP0MsJKxfUcfapojNdzJPxVyHS9riXNwaoy/t8dJwlh2jOV4ZzpEMCWsao6xtirIseepdQToxjv+j+7nvYx+GeBw5930lOYFbmDmiP3AKQV/tBGjPTdCemyDn7GNvop03U4t4onUtsS/9HUsP//act1mpI/6vAn8M1FWofXMS+ycLPNw7xUjO5/zmKDcuTc6bC4TMiYkIS5JhliTD3LQ0xZ6JPDvGcrwynOWFoSyJkNBVF2FlXZiuujB1J/h09/a8+T/5Ifgef/zIL/jr//gf5uzgYFpCHHRTHAylOOim6HcTBCLvCvrl3iTL/KkFcR+HaOBx7tQhzpnqY3+8hadzAeMDfXPeTtmDX0R+GRhQ1edF5PoTrLcB2ADQ2dlZnuIMWS/gsUPTvDycoyHi8InV9aysf+/QNoDOFV307u855mt33313Kcs0cyTiCuc2RTm3KUrOD9g1nmf3RIF9k3m2jxbnVWqNuSxNhlicCLM4EaIl5hJyBB3qx3/oAXTvLmTFatz1H+er/+av+JvTCP0AmJAIA26CATfO4MzXCad4bsDVgMX+NJflDrPUn2KJP70ggv54HJSuzBCP/e2XmRoemPP3r8QR/zXAehG5DYgB9SLyTVX9zSNXUtWNwEaA7u5um8qxxFSVN8bzPNo7RdpTrmiP8/7FCcInGKLZu7+HTYem3rP82iUpPvPFu0pZ7mn5xle+XOkSqlrUdTivOcZ5zTFUlYGMz77JPD2TBd4Yy/PycHFHEPHyXLf7ac7fuYUgFOHgL91O7sLLiYdCdF7QzZATe3vooyIoM8McxSUjIbISIuOEmJQwY06UMSfKhBMtHskDokpzkGWpN8XF/iBL/Sk6/DQhm9F1zpQ9+FX1LuAugJkj/j86OvRNeU3kfR49MM3O8TwdcZdfW13HooQN0axlIkJHIkRHIsQVHcUDg7Gcz+QrL9G86UdEpyfY1XUBv1h7A2PhBOwvjkL53Xt+zDdm2UZUPRqDHB1+mjWFURqDHG1BhlY/Q9hCvqTsf3cNe2uI5hN9aQJVbliS4LL2uA3RNO+hB/dT9+gPSe3fCx1LcD/+W6ztXMlaIOcHTBUCMp5y4wdv5YG/+jIBMnOsXzx5KarE1CeuHnH1iKln4V5BFQ1+VX0ceLySNdSqw2mPh/dPcTjjsaquOESzMWpDNM276fAg/s8eQne8Ask6nNt+FefSK9519W3Udd6+HuDNp3/OuYXRSpVrZsmO+GtM3lc29U3z/GBx9MYdXXWc22hDNM276dgIwZM/J3hxC7ih4v1vr76+bBdimdKy4K8hO8dzPNo7zUQh4OLWGNctThCr5BDNwMfxfSTwEQ2QIABVEEFFZr46qOMSuCGwOzOVnA4P4j/5M/SV5wHBueRKnOtuRlJ2neVCYsFfA0ZzPj89MMXuiQJtMZdPrWxgaTJ88h88E6q4+SyhXOZdD8cr4BbyOF4BJzi14XiB4xC4IYJQGD8cLT4iM1+jMbxoHD8cLdnVjgtZcHA/wTOb0NdeAtfF6b4G55rrkfrGSpdmSsCCfwHL+8oz/Wm2DGRwRbhhSYLu9jjuXAejBsRHh0gOHSQ51Mdjv/0Jlrz8BE7wzoRTgTj40Rh+OEouWU8QjhCEwgRuCHWcmSN7B2auwhTV4rztqkjg43gejl/A8b3iziOfI5KexPXefXevQBy8WBwvGseLJijEkxTiSbxYYm5/5wVAvQL62ssEzz6JHuqFSBTnqutxrroOSdm1lQuZBf8CpKq8MZbn5weL3TrnNUW5fmmCuvDcnLxVDWCwn9+7/CLO/sl3qO/bRzhXHM7nhyLsVmW6ZXExcKMJvGicIBwpzZF4EOAWcoRyWUK5dPGTRTZDODNNfGz47ZElKsLnui+gbt8OCvHUzA4hNff1VDlVhf5DBK88T/DyVkhPQ2s7zoc+gnNhNxKNVbpEUwYW/AvMQMbjZwem6Zkq0B53ub2rgeWpM+/W0UIe3bOT4M3t6JvbYWqCr972AbLDhxhdsYaJJV1MtS8jW9/CB5bVs+cjZbo0w3Hwo3H8aJwcTUcVHRDKpglnpglnppgceoW2yVGSI/1vr7LvDzcQ//F9pFsWMd2ymHTrYrL1TcVPHguIjo0QbHuRYNsLMHgYHAc5Zx3OZdcgK8+2k/s1xoJ/gZgs+DzRl2bbcI6oK9yyLMlFrWc2bbLmsugbrxG89hK6503wPIhEkbPOxTnrXFZ94Gbu235oDn+LOSYOXjyFF0+RoYPvfPd7fObmj+B4+eLOID3FT3/0IL+6YjUNB3bjaLFryg9FSLd0FHcEb+0QmtvRUInPi8whDQL0QA+6cwfBzh3QX/x3kuVdyId/FWfdhVVzFyxTfhb881zeV7YMpHl2IIOv0N0W4+pFidOeUE29AvrmDoLXXiwe2Xse1DfiXHIVsmYdsmIV4hb/bHrG5uctFoNQhFxdhFxdE7/zT49wzn/7R8T3iI8OkhzuIzF8mORwH607Xya0/VkAVBwyja1Mtyx6Z2fQsggvXh3hqb6HHj6E7t+L9u5D9+2CTBrEQTq7kJs+XAz7ppZKl2qqwIIP/hNNJFaNlneuYH/PvpOuF6iybSTHE4fSTHkB5zZGuG5JkqbTvAhL+w8RvPAswStbIZuBZArn4iuKd05avgJZYF0fR1M3RLq12NXzzsKA6OTY2zuCxNBh6vv20bbrlbdXKUTjZBtayNa3kG2c+drQTD5ZTyGWnLMhqF2dnfT09gLQnkywrr2FtW0trGtr4bz2Fi5Z0kEiXPxEsnd0nCd7DvDwzr08uruHsWxuTmqYLZugr/ot+OA/3kRi1eraJSc+4fjWidsnDqcZzvosTYb4yKq60xqeqbks+uqLBC9sKY7qcF3k3PfhXHw5svKsit4btSqIQ66+mVx9M6Mr1729OJSdJjF8mMTwYeLjI8TGh2Z2CC+/68dVHPKJFIVEHflEHYVE6t1DUMNR1HWLp5/f6pJTxS3k+YOrL8V/7GHI59DpSf7HjVdy3do7IZeFI0ZLEQpBIgV19UhDI9Q3sjoaYzVwZ6m3zxHuvvtuPvPFu1i1fn1VTtB3JJusrwaCf6FQVXZPFNjUN81Axqc15vIrK+tY03DqV93qQB/BlieLJ/oKeWhfhPPBO3AuuNT6fWfBiyWZWLqaiaWr37Xc8fJEJ0aJTYwQmZ4knJ4gMj1JJD1JdHKUuv5e3EIOx/dO2sZf3nIdwaZHIRyBZKo4BDdVX7yVYTQGyVQx8CN21bU5dRb8VU5V6ZkssKkvzaG0R2PE4fYVKdY2RU/pxK0GAbpzO8GWJ9C9u2ZugH0JzqVXIks7LTzmQBCKkGnuINPcccL1JPBx8jlChdw7VysDxVtyC344ws3nrWAim327i+0DIni/Nfd3YjK1yYK/SqkqeyYKbO5Pc2Daoz7s8KHlKc5viZ7SBViazRC8+CzBs08Wb35d34hz44dxLrnCju4rRB0XP5bAP8FFZVP5woI/r2Iqx4K/yojj8Ppojqf70wxkfOrDDjcvS3JhS4zQCW6KcjSdnCDYsolg62bIZZHOVTg3346ce5713RtT4yz4q0SgymDG5w/uf4p/2jdJc9Tlts4U5zVFcU8l8IcH8Z9+HH35OQgCZN0FuFffgCxZXsLqjTHziQV/hRUCpT/t0Zf2yAeQz6T5la46zmmMnFIffnBwP8FTj6E7thUn2bro8uI0us2tJazeGDMfWfBXSNoL6Jv2GMj4BEBDxOGsZIi7fuNG/k5nd2ciVUV3v1EM/H27IBbHef8HcK74JZtkyxhzXBb8ZaSqjOWLgT+aDxCgLe6yJBEiGZ79iTwNfPS1l/GffgwOH4K6Bpybby+O0LFJtowxJ2HBXwZ5XxnIePRnfLK+EnZgeSrEoniIiHsK/feFfHGEzuZfFEfotLbjrv8EcsElb0+jYIwxJ1P2tBCR5cC9QAfFgcsbVfVvy11Hqakq4/mA/ozPcNZHgfqwQ2cqREvMPbUx+Jk0wXNPEWx5AtLTyLIVOB9cj6w5z4b8GWNOWSUOEz3gD1X1BRGpA54XkUdVdXsFaplzhUAZyPj0pz0yvuIKLEq4LEqESJzixGk6MUaweRPBC89APoecvRbnmhuQzlV2wZUx5rSVPfhVtQ/om/l+UkR2AEuBeRv8gSqjuYCBjMdoLkCBurDD2TNH96d6xysd6sd/6jH0lReKd6M6/yLca25AOpaU5hcwxtSUinYMi0gXcDGw5RivbQA2AHR2dpa3sFlQVaa9Yt/9YMbHUwg7sDgRoj3untLJ2rekBnr53q+vJ//3f0nW8/jGC6/y1c3Pvz39sSuCP8sRP+VysknlzOkR133Ppzqb9dLMlYoFv4ikgH8E/qWqvmdid1XdCGwE6O7urpq0y/kBg1mfwYxP2lMEaI65tMddmiLOqXfBqNJwYBdLXnqChr59LFmxjEOXXs/h867k0n+R5L4jVr12SYo9Dz44l7/OGfnGV758zJkYV61fX4FqFhb1/XfNKnvtklRVz3ppM17OLxUJfhEJUwz9b6nqA5Wo4VTkfWU46zOU9ZkoFKfErQs7rK4P0RpzT2kqhbeIV6B11yss3raZxOgAuWQ9+668lYvXf5gf//5fzfWvYIwxb6vEqB4B/iewQ1W/Uu72Z8sLimE/mPUZzxfDPhESOlPFsD/dO1yF05N0bH+Wju3PEc6mmW5ZxK7rP8rw6vNRN8R0vjCXv4YxxrxHJY74rwE+BWwTkZdmlv1rVX2oArW8ixcoo7li2I/NnKSNucKyZIi2mEviNPrt35IYPszibU/TsmsbEgSMrljD4fddxcTirnduwmGMMWVQiVE9TwJVk3SFQBnJ+gzn3gn7iCMsToRoi7skQ3LaQyfF92jeu4OOHc9S39eDH4owsLabw+dfSbbB7n1qjKmMmrzcM+e/E/ZvdeNE3WLYt8Qc6sKncZL2CNGJUdpf30r76y8Qzk6TrWui54pbGDj3UvxofK5+DWOMOS01E/wZL2A45zOS9ZksFAcJxUPFbpyW2Jkd2QMQBDTtf5P2Hc/R2LsLBEY719C/7jLGl60Gu8LWGFMlFnTwD2U8btzwR7w4lCXtFcM+OXOCtiXmnvKVtMcSGxui7c2XaN35EtHpCfKJOg5ech0D515KPtVwxu9vjDFzbUEH//NDWT6w4UuERFhZF6I55hBzzzzs3XyWlt2v0vbmi9T196IijC9bzb6rb2NsxRrU7nBljKliCzr4r+qI87GLVvHQq3vP+L3E92js3UnL7ldp3rcDx/dIN7bRc/nNDJ19IYVk/RxUbIwxpbegg78+4jI1MnjaPy+BT8OB3bTseZWmvTsIFXIUogkG1lzM4DmXMN22xIZiGmPmnQUd/KdDvAINB/fQvG8HTft2EM5l8CIxRlauY3j1+UwsXWVdOcaYec2CHwhlp2nqeZOmntdpOLAL1yvghaOMrVjD0OrzGV92Fmo3OjHGLBC1nWYasPahe0n17sJ1HHrHJ/nuG7v54Ru7+cW+Xgp+UJGybMbL8nFFqnZ7V2tdZv6r7eAXh3RjG3/3rW/xuf/780g8xe03CLdXsKTjzXgJNutlKfiqVTXj6VuO/juwf3szl2r+qqKeaz7Mv398M4VEnZ2oNcbUhJoPfmOMqTUW/MYYU2Ms+I0xpsZY8BtjTI2x4DfGmBpjwW+MMTXGgt8YY2qMBb8xxtSYigS/iNwqIm+IyC4R+ZNK1GCMMbWq7MEvIi7wX4EPAeuAT4rIunLXYYwxtaoSR/yXA7tUdY+q5oHvAHdUoA5jjKlJoqrlbVDkY8Ctqvo7M88/BVyhqp8/ar0NwIaZp2uAN8pa6Im1AkOVLqJK2bY5Pts2x2fb5vjOZNusUNW2oxdW7eycqroR2FjpOo5FRLaqanel66hGtm2Oz7bN8dm2Ob5SbJtKdPUcBJYf8XzZzDJjjDFlUIngfw44W0RWikgE+HWg+iZEN8aYBarsXT2q6onI54FHABf4uqq+Vu46zlBVdkFVCds2x2fb5vhs2xzfnG+bsp/cNcYYU1l25a4xxtQYC35jjKkxFvwncLKpJUTkiyKyXUReEZGficiKStRZCbOddkNEflVEVERqZqjebLaNiHx85m/nNRH53+WusVJm8X+qU0QeE5EXZ/5f3VaJOstNRL4uIgMi8upxXhcR+S8z2+0VEbnkjBpUVXsc40HxxPNuYBUQAV4G1h21zg1AYub73wW+W+m6q2XbzKxXB2wCngG6K113tWwb4GzgRaBp5nl7peuuom2zEfjdme/XAfsqXXeZts21wCXAq8d5/Tbgx4AAVwJbzqQ9O+I/vpNOLaGqj6lqeubpMxSvSagFs51248+BvwSy5SyuwmazbT4H/FdVHQVQ1YEy11gps9k2CtTPfN8AHCpjfRWjqpuAkROscgdwrxY9AzSKyOLTbc+C//iWAr1HPD8ws+x4Pktxj1wLTrptZj6KLlfVH5WzsCowm7+bc4BzROQpEXlGRG4tW3WVNZtt8++A3xSRA8BDwBfKU1rVO9U8OqGqnbJhPhGR3wS6gesqXUs1EBEH+Arw6QqXUq1CFLt7rqf4KXGTiLxPVccqWVSV+CRwj6r+JxG5CrhPRM5X1aDShS0kdsR/fLOaWkJEbgL+FFivqrky1VZpJ9s2dcD5wOMiso9in+SDNXKCdzZ/NweAB1W1oKp7gTcp7ggWutlsm88C3wNQ1c1AjOIkZbVuTqe6seA/vpNOLSEiFwP/H8XQr5V+WjjJtlHVcVVtVdUuVe2ieP5jvapurUy5ZTWbKUn+ieLRPiLSSrHrZ08Za6yU2Wyb/cCNACKylmLwD5a1yur0IPBbM6N7rgTGVbXvdN/MunqOQ48ztYSI/Htgq6o+CPw1kAK+LyIA+1V1fcWKLpNZbpuaNMtt8whwi4hsB3zgS6o6XLmqy2OW2+YPga+JyB9QPNH7aZ0Z1rKQici3KR4MtM6c3/gzIAygqv9A8XzHbcAuIA185ozaq4Ftaowx5gjW1WOMMTXGgt8YY2qMBb8xxtQYC35jjKkxFvzGGFNjLPiNMabGWPAbY0yN+T/SjiDogoQnVwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(ood_score_init, color='skyblue', kde=True)\n",
    "sns.histplot(ood_score_correct, color='salmon', kde=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34 314 0.10828025477707007\n"
     ]
    }
   ],
   "source": [
    "print(corrected, total_mistakes, corrected/total_mistakes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4323, ts_macro_f1=0.2168\n",
      "ts_micro_f1=0.4939, ts_macro_f1=0.2391\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# Cheat add\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = batch[\"data\"].T[(target_col_mask==col_i) + (target_col_mask==0)].reshape(1, -1)\n",
    "            target_col_mask_temp = target_col_mask[(target_col_mask==col_i) + (target_col_mask==0)].reshape(1, -1)\n",
    "            cls_indexes = (target_col_mask_temp == 0).nonzero()[0].reshape(1, -1)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits = logits_temp.clone()\n",
    "                corrected += 1\n",
    "                break\n",
    "            \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4396, ts_macro_f1=0.2099\n",
      "ts_micro_f1=0.5078, ts_macro_f1=0.2336\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# Cheat add\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        for col_i in target_col_mask.unique():\n",
    "            if col_i == 0 or col_i == -1:\n",
    "                continue\n",
    "            new_batch_data = torch.cat([batch[\"data\"].T[target_col_mask==0].reshape([1, -1]), batch[\"data\"].T[target_col_mask==col_i].reshape([1, -1])], dim=1)\n",
    "            cls_indexes = torch.tensor([[0, 0]]).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits = logits_temp.clone()\n",
    "                corrected += 1\n",
    "                break\n",
    "            \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n",
      "Mistake constitution tensor(0.5437) tensor(0.7332)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4138, ts_macro_f1=0.1909\n",
      "ts_micro_f1=0.4593, ts_macro_f1=0.2035\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.9****************************\n",
      "Mistake constitution tensor(0.6453) tensor(0.6741)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1887\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.2009\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.95****************************\n",
      "Mistake constitution tensor(0.7308) tensor(0.6721)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4147, ts_macro_f1=0.1843\n",
      "ts_micro_f1=0.4610, ts_macro_f1=0.1966\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.99****************************\n",
      "Mistake constitution tensor(0.8592) tensor(0.6309)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4138, ts_macro_f1=0.1839\n",
      "ts_micro_f1=0.4593, ts_macro_f1=0.1959\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.999****************************\n",
      "Mistake constitution tensor(0.9982) tensor(0.5437)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.1960\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n",
      "Mistake constitution tensor(1.) tensor(0.5171)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3797198874.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3797198874.py:40: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.1960\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# Greedy (MSP)\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "\n",
    "threshold = 0.8\n",
    "log = {}\n",
    "for threshold in [0.8, 0.9, 0.99 1.0]:\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    num_cols = []\n",
    "    log[threshold] = {}\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    \n",
    "    print(\"Mistake constitution\", (ood_score[ood_labels==0] < threshold).sum()/sum(ood_labels==0), \n",
    "          (ood_score[ood_labels==0] < threshold).sum()/sum(ood_score< threshold))\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        if 1 in target_col_mask:\n",
    "            ood_score_max = F.softmax(logits.detach()).max().item()\n",
    "            if ood_score_max < threshold:\n",
    "                log[threshold][batch_idx] = []\n",
    "                log[threshold][batch_idx].append([ood_score_max])\n",
    "                current_drop = 1\n",
    "                while True:\n",
    "                    improved = False\n",
    "                    log[threshold][batch_idx].append([])\n",
    "                    for col_i in target_col_mask.unique():\n",
    "                        if col_i == 0 or col_i == -1:\n",
    "                            continue\n",
    "                        target_col_mask_temp = target_col_mask.clone()\n",
    "                        target_col_mask_temp[target_col_mask_temp==col_i] = -1\n",
    "                        new_batch_data = batch[\"data\"].T[target_col_mask_temp!=-1].reshape(1, -1)\n",
    "                        cls_indexes = (target_col_mask_temp[target_col_mask_temp!=-1].reshape(1, -1) == 0).nonzero()[0].reshape(1, -1)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n",
    "                        if ood_score_temp > ood_score_max:\n",
    "                            logits = logits_temp.clone()\n",
    "                            temp_best_target_col_mask = target_col_mask_temp.clone()\n",
    "                            # print(f\"{batch_idx} col_{col_i}\", ood_score_max, ood_score_temp)\n",
    "                            ood_score_max = ood_score_temp\n",
    "                            log[threshold][batch_idx][current_drop].append(ood_score_max)\n",
    "                            improved = True\n",
    "                    if not improved:\n",
    "                        break\n",
    "                    else:\n",
    "                        target_col_mask = temp_best_target_col_mask.clone()\n",
    "                        current_drop += 1\n",
    "                    \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n",
      "tensor(0.8111) tensor(0.5621)\n",
      "*********************Threshold: 0.85****************************\n",
      "tensor(0.7956) tensor(0.6007)\n",
      "*********************Threshold: 0.9****************************\n",
      "tensor(0.7850) tensor(0.6493)\n",
      "*********************Threshold: 0.95****************************\n",
      "tensor(0.7614) tensor(0.7013)\n",
      "*********************Threshold: 0.99****************************\n",
      "tensor(0.6604) tensor(0.8188)\n",
      "*********************Threshold: 1.0****************************\n",
      "tensor(0.5493) tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "for threshold in [0.8, 0.85, 0.9, 0.95, 0.99, 1.0]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print((ood_score[ood_labels==0] < threshold).sum()/sum(ood_score< threshold), (ood_score[ood_labels==0] < threshold).sum()/sum(ood_labels==0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.01****************************\n",
      "tensor(0.6113) tensor(0.9984)\n",
      "*********************Threshold: 0.1****************************\n",
      "tensor(0.7398) tensor(0.8777)\n",
      "*********************Threshold: 0.2****************************\n",
      "tensor(0.7771) tensor(0.8307)\n",
      "*********************Threshold: 0.5****************************\n",
      "tensor(0.8098) tensor(0.7006)\n",
      "*********************Threshold: 1.0****************************\n",
      "tensor(0.8594) tensor(0.5940)\n",
      "*********************Threshold: 1.5****************************\n",
      "tensor(0.8734) tensor(0.4216)\n",
      "*********************Threshold: 2.0****************************\n",
      "tensor(0.9053) tensor(0.2696)\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.1\n",
    "for threshold in [0.01, 0.1, 0.2, 0.5, 1.0, 1.5, 2.0]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print((ood_score[ood_labels==1] > threshold).sum()/sum(ood_score> threshold), (ood_score[ood_labels==1] > threshold).sum()/sum(ood_labels==1))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.01****************************\n",
      "Mistake constitution tensor(0.5412) tensor(0.9892)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/154180484.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/154180484.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4101, ts_macro_f1=0.1833\n",
      "ts_micro_f1=0.4523, ts_macro_f1=0.1965\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.1****************************\n",
      "Mistake constitution tensor(0.7262) tensor(0.7939)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/154180484.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/154180484.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4111, ts_macro_f1=0.1834\n",
      "ts_micro_f1=0.4541, ts_macro_f1=0.1963\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.2****************************\n",
      "Mistake constitution tensor(0.7555) tensor(0.7366)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/154180484.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/154180484.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4101, ts_macro_f1=0.1834\n",
      "ts_micro_f1=0.4523, ts_macro_f1=0.1962\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.5****************************\n",
      "Mistake constitution tensor(0.7845) tensor(0.6004)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/154180484.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/154180484.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4092, ts_macro_f1=0.1882\n",
      "ts_micro_f1=0.4506, ts_macro_f1=0.2012\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n",
      "Mistake constitution tensor(0.8297) tensor(0.4803)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/154180484.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/154180484.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4111, ts_macro_f1=0.1908\n",
      "ts_micro_f1=0.4541, ts_macro_f1=0.2044\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# Greedy (Entropy)\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "\n",
    "threshold = 0.8\n",
    "log = {}\n",
    "for threshold in [0.01, 0.1, 0.2, 0.5, 1.0]:\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    num_cols = []\n",
    "    log[threshold] = {}\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    \n",
    "    print(\"Mistake constitution\", (ood_score[ood_labels==1] > threshold).sum()/sum(ood_score> threshold), (ood_score[ood_labels==1] > threshold).sum()/sum(ood_labels==1))\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        if 1 in target_col_mask:\n",
    "            ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
    "            if ood_score_min > threshold:\n",
    "                log[threshold][batch_idx] = []\n",
    "                log[threshold][batch_idx].append([ood_score_min])\n",
    "                current_drop = 1\n",
    "                while True:\n",
    "                    improved = False\n",
    "                    log[threshold][batch_idx].append([])\n",
    "                    for col_i in target_col_mask.unique():\n",
    "                        if col_i == 0 or col_i == -1:\n",
    "                            continue\n",
    "                        target_col_mask_temp = target_col_mask.clone()\n",
    "                        target_col_mask_temp[target_col_mask_temp==col_i] = -1\n",
    "                        new_batch_data = batch[\"data\"].T[target_col_mask_temp!=-1].reshape(1, -1)\n",
    "                        cls_indexes = (target_col_mask_temp[target_col_mask_temp!=-1].reshape(1, -1) == 0).nonzero()[0].reshape(1, -1)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n",
    "                        if ood_score_temp < ood_score_min:\n",
    "                            logits = logits_temp.clone()\n",
    "                            temp_best_target_col_mask = target_col_mask_temp.clone()\n",
    "                            ood_score_min = ood_score_temp\n",
    "                            log[threshold][batch_idx][current_drop].append(ood_score_min)\n",
    "                            improved = True\n",
    "                    if not improved:\n",
    "                        break\n",
    "                    else:\n",
    "                        target_col_mask = temp_best_target_col_mask.clone()\n",
    "                        current_drop += 1\n",
    "                    \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.01****************************\n",
      "Mistake constitution tensor(0.5412) tensor(0.9892)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/2503683824.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/2503683824.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4120, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4558, ts_macro_f1=0.1950\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.1****************************\n",
      "Mistake constitution tensor(0.7262) tensor(0.7939)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/2503683824.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/2503683824.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.1948\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.2****************************\n",
      "Mistake constitution tensor(0.7555) tensor(0.7366)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/2503683824.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/2503683824.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4120, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4558, ts_macro_f1=0.1947\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.5****************************\n",
      "Mistake constitution tensor(0.7845) tensor(0.6004)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/2503683824.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/2503683824.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4111, ts_macro_f1=0.1886\n",
      "ts_micro_f1=0.4541, ts_macro_f1=0.1997\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n",
      "Mistake constitution tensor(0.8297) tensor(0.4803)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3005789/2503683824.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3005789/2503683824.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4120, ts_macro_f1=0.1910\n",
      "ts_micro_f1=0.4558, ts_macro_f1=0.2026\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# Drop 1 (Entropy)\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "\n",
    "threshold = 0.8\n",
    "log = {}\n",
    "for threshold in [0.01, 0.1, 0.2, 0.5, 1.0]:\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    num_cols = []\n",
    "    log[threshold] = {}\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    \n",
    "    print(\"Mistake constitution\", (ood_score[ood_labels==1] > threshold).sum()/sum(ood_score> threshold), (ood_score[ood_labels==1] > threshold).sum()/sum(ood_labels==1))\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        if 1 in target_col_mask:\n",
    "            ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
    "            if ood_score_min > threshold:\n",
    "                log[threshold][batch_idx] = []\n",
    "                log[threshold][batch_idx].append([ood_score_min])\n",
    "                current_drop = 1\n",
    "\n",
    "                log[threshold][batch_idx].append([])\n",
    "                for col_i in target_col_mask.unique():\n",
    "                    if col_i == 0 or col_i == -1:\n",
    "                        continue\n",
    "                    target_col_mask_temp = target_col_mask.clone()\n",
    "                    target_col_mask_temp[target_col_mask_temp==col_i] = -1\n",
    "                    new_batch_data = batch[\"data\"].T[target_col_mask_temp!=-1].reshape(1, -1)\n",
    "                    cls_indexes = (target_col_mask_temp[target_col_mask_temp!=-1].reshape(1, -1) == 0).nonzero()[0].reshape(1, -1)\n",
    "                    logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                    ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n",
    "                    if ood_score_temp < ood_score_min:\n",
    "                        logits = logits_temp.clone()\n",
    "                        temp_best_target_col_mask = target_col_mask_temp.clone()\n",
    "                        ood_score_min = ood_score_temp\n",
    "                        log[threshold][batch_idx][current_drop].append(ood_score_min)\n",
    "                    \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([101])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n",
      "Mistake constitution tensor(0.5437) tensor(0.7332)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4157, ts_macro_f1=0.1926\n",
      "ts_micro_f1=0.4627, ts_macro_f1=0.2052\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.9****************************\n",
      "Mistake constitution tensor(0.6453) tensor(0.6741)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4147, ts_macro_f1=0.1904\n",
      "ts_micro_f1=0.4610, ts_macro_f1=0.2026\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.95****************************\n",
      "Mistake constitution tensor(0.7308) tensor(0.6721)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4166, ts_macro_f1=0.1861\n",
      "ts_micro_f1=0.4645, ts_macro_f1=0.1983\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.99****************************\n",
      "Mistake constitution tensor(0.8592) tensor(0.6309)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4157, ts_macro_f1=0.1856\n",
      "ts_micro_f1=0.4627, ts_macro_f1=0.1975\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 0.999****************************\n",
      "Mistake constitution tensor(0.9982) tensor(0.5437)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4147, ts_macro_f1=0.1855\n",
      "ts_micro_f1=0.4610, ts_macro_f1=0.1976\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n",
      "Mistake constitution tensor(1.) tensor(0.5171)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/920578023.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/920578023.py:41: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4147, ts_macro_f1=0.1855\n",
      "ts_micro_f1=0.4610, ts_macro_f1=0.1976\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# One column\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "\n",
    "threshold = 0.8\n",
    "log = {}\n",
    "changed_log = {}\n",
    "for threshold in [0.8, 0.9, 0.95, 0.99, 0.999, 1.0]:\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    num_cols = []\n",
    "    log[threshold] = {}\n",
    "    changed = [0, 0, 0]\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Mistake constitution\", (ood_score[ood_labels==0] < threshold).sum()/sum(ood_labels==0), \n",
    "          (ood_score[ood_labels==0] < threshold).sum()/sum(ood_score< threshold))\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        pred_init = logits.argmax().item()\n",
    "        changed_batch = False\n",
    "        if 1 in target_col_mask:\n",
    "            ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
    "            if ood_score_max < threshold:\n",
    "                log[threshold][batch_idx] = []\n",
    "                log[threshold][batch_idx].append([ood_score_max])\n",
    "                current_drop = 1\n",
    "                log[threshold][batch_idx].append([])\n",
    "                for col_i in target_col_mask.unique():\n",
    "                    if col_i == 0 or col_i == -1:\n",
    "                        continue\n",
    "                    target_col_mask_temp = target_col_mask.clone()\n",
    "                    target_col_mask_temp[target_col_mask_temp==col_i] = -1\n",
    "                    new_batch_data = batch[\"data\"].T[target_col_mask_temp!=-1].reshape(1, -1)\n",
    "                    cls_indexes = (target_col_mask_temp[target_col_mask_temp!=-1].reshape(1, -1) == 0).nonzero()[0].reshape(1, -1)\n",
    "                    logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                    ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n",
    "                    if ood_score_temp > ood_score_max:\n",
    "                        logits = logits_temp.clone()\n",
    "                        temp_best_target_col_mask = target_col_mask_temp.clone()\n",
    "                        # print(f\"{batch_idx} col_{col_i}\", ood_score_max, ood_score_temp)\n",
    "                        ood_score_max = ood_score_temp\n",
    "                        log[threshold][batch_idx][current_drop].append(ood_score_max)\n",
    "                        improved = True\n",
    "                        changed_batch = True\n",
    "        if changed_batch:\n",
    "            pred_current = logits.argmax().item()\n",
    "            if pred_current == pred_init:\n",
    "                changed[0] += 1\n",
    "            elif pred_current == batch[\"label\"].item():\n",
    "                changed[1] += 1\n",
    "            else:\n",
    "                changed[2] += 1\n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "    changed_log[threshold] = changed\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 1.0****************************\n",
      "Mistake constitution tensor(1.) tensor(0.5171)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2170815/3279817322.py:29: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_2170815/3279817322.py:47: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1838\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.1960\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# One column\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "\n",
    "threshold = 0.8\n",
    "log = {}\n",
    "changed_log = {}\n",
    "state = []\n",
    "for threshold in [1.0]:\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    ood_scores_init = []\n",
    "    num_cols = []\n",
    "    log[threshold] = {}\n",
    "    changed = [0, 0, 0]\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    \n",
    "    print(\"Mistake constitution\", (ood_score[ood_labels==0] < threshold).sum()/sum(ood_labels==0), \n",
    "          (ood_score[ood_labels==0] < threshold).sum()/sum(ood_score< threshold))\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        pred_init = logits.argmax().item()\n",
    "        changed_batch = False\n",
    "        ood_score_max = ood_score_init = F.softmax(logits.detach()).max().item()\n",
    "        ood_scores_init.append(ood_score_init)\n",
    "        if 1 in target_col_mask:\n",
    "            if ood_score_max < threshold:\n",
    "                log[threshold][batch_idx] = []\n",
    "                log[threshold][batch_idx].append([ood_score_max])\n",
    "                current_drop = 1\n",
    "                while True:\n",
    "                    improved = False\n",
    "                    log[threshold][batch_idx].append([])\n",
    "                    for col_i in target_col_mask.unique():\n",
    "                        if col_i == 0 or col_i == -1:\n",
    "                            continue\n",
    "                        target_col_mask_temp = target_col_mask.clone()\n",
    "                        target_col_mask_temp[target_col_mask_temp==col_i] = -1\n",
    "                        new_batch_data = batch[\"data\"].T[target_col_mask_temp!=-1].reshape(1, -1)\n",
    "                        cls_indexes = (target_col_mask_temp[target_col_mask_temp!=-1].reshape(1, -1) == 0).nonzero()[0].reshape(1, -1)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n",
    "                        if ood_score_temp > ood_score_max:\n",
    "                            logits = logits_temp.clone()\n",
    "                            temp_best_target_col_mask = target_col_mask_temp.clone()\n",
    "                            # print(f\"{batch_idx} col_{col_i}\", ood_score_max, ood_score_temp)\n",
    "                            ood_score_max = ood_score_temp\n",
    "                            log[threshold][batch_idx][current_drop].append(ood_score_max)\n",
    "                            improved = True\n",
    "                            changed_batch = True\n",
    "                    if not improved:\n",
    "                        break\n",
    "                    else:\n",
    "                        target_col_mask = temp_best_target_col_mask.clone()\n",
    "                        current_drop += 1\n",
    "        if changed_batch:\n",
    "            pred_current = logits.argmax().item()\n",
    "            if pred_current == pred_init:\n",
    "                state.append(1)\n",
    "            elif pred_current == batch[\"label\"].item():\n",
    "                state.append(2)\n",
    "            else:\n",
    "                state.append(3)\n",
    "        else:\n",
    "            state.append(0)\n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    ood_scores_init = torch.tensor(ood_scores_init)\n",
    "    states = torch.tensor(state)\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "    changed_log[threshold] = changed\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ood_scores_init.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "ood_scores_new = logits_test.softmax(1).max(1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbtklEQVR4nO3de5RdZZnn8e+vbkkVBJKQSjrkSkxMm7ZXo6tEvCxHpe0G1hoj08LgmhZU7KhcRKW1vcwsRQcXbSOZ1tWNRmAEl62Ciqa7aZWOKGOmuRSKyMUMZUElFZJUSZELVKUup5754+za2XU5VaeS7HMqye+z1lm1z7vfvfdT76l6n7PffVNEYGZmBlBT7QDMzGzmcFIwM7OUk4KZmaWcFMzMLOWkYGZmqbpqB3AkFixYECtXrqx2GGZmx5SHH3749xHRPNG8YzoprFy5ktbW1mqHYWZ2TJHUUWqeh4/MzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpY7pi9fMzE4UhUKBtra29P3q1aupra096ttxUjAzOwa0tbXRfuGFrGpspL2vD+68k7Vr1x717TgpmJkdI1Y1NrK2qSnXbfiYgpmZpXJLCpJmS3pQ0q8lPS7p2qT865KelvRI8jozKZekL0lqk/SopFfmFZuZmU0sz+GjfuDNEfGCpHrgF5L+LZn30Yj47pj65wFrktergZuSn2ZmViG57SlE0QvJ2/rkFZMssh64PVnufmCupMV5xWdmZuPlekxBUq2kR4Au4J6IeCCZdV0yRLRR0qykbAmwI7N4Z1I2dp0bJLVKau3u7s4zfDOzE06uSSEiChFxJrAUOEvSy4FPAH8IvAqYD/zNNNe5KSJaIqKluXnCBweZmdlhqsjZRxGxF7gXODcidiVDRP3A/wbOSqrtBJZlFlualJmZWYXkefZRs6S5yXQj8BbgtyPHCSQJeBvwWLLIZuCS5Cyks4F9EbErr/jMzGy8PM8+WgzcJqmWYvK5IyL+RdJPJTUDAh4B3p/Uvxs4H2gDeoF35xibmZlNILekEBGPAq+YoPzNJeoHcEVe8ZiZ2dR8RbOZmaWcFMzMLOWkYGZmKScFMzNLOSmYmVnKScHMzFJOCmZmlnJSMDOzlJOCmZmlnBTMzCzlpGBmZqk8b4hnZmZHSaFQYKCvj16gr6+PhkIhl+04KZiZHQM6Ojp4oa2fQl0j7UP9nNzRwbp16476djx8ZGZ2jKipmUVNbRM1NbOmrny428htzWZmdsxxUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0vllhQkzZb0oKRfS3pc0rVJ+RmSHpDUJuk7khqS8lnJ+7Zk/sq8YjMzs4nluafQD7w5Iv4EOBM4V9LZwN8CGyNiNfA8cFlS/zLg+aR8Y1LPzMwqKLekEEUvJG/rk1cAbwa+m5TfBrwtmV6fvCeZf44k5RWfmZmNl+sxBUm1kh4BuoB7gN8BeyNiKKnSCSxJppcAOwCS+fuA0yZY5wZJrZJau7u78wzfzOyEk2tSiIhCRJwJLAXOAv7wKKxzU0S0RERLc3Pzka7OzMwyKnL2UUTsBe4FXgPMlTRyI76lwM5keiewDCCZfyrwXCXiMzOzojzPPmqWNDeZbgTeAjxJMTm8Pal2KfDDZHpz8p5k/k8jIvKKz8zMxsvz1tmLgdsk1VJMPndExL9IegL4tqT/CfwKuCWpfwvwDUltQA9wcY6xmZnZBHJLChHxKPCKCcrbKR5fGFt+ELgwr3jMzGxqvqLZzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmKScFMzNLOSmYmVnKScHMzFJOCmZmlnJSMDOzVG5JQdIySfdKekLS45KuTso/I2mnpEeS1/mZZT4hqU3SNkl/nldsZmY2sboc1z0EXBMRv5Q0B3hY0j3JvI0RcUO2sqR1wMXAHwGnA/8u6aURUcgxRjMzy8htTyEidkXEL5PpA8CTwJJJFlkPfDsi+iPiaaANOCuv+MzMbLyKHFOQtBJ4BfBAUnSlpEcl3SppXlK2BNiRWayTCZKIpA2SWiW1dnd35xm2mdkJJ/ekIOlk4HvAhyJiP3AT8BLgTGAX8MXprC8iNkVES0S0NDc3H+1wzcxOaLkmBUn1FBPCNyPi+wARsSciChExDHyNQ0NEO4FlmcWXJmVmZlYheZ59JOAW4MmIuDFTvjhT7QLgsWR6M3CxpFmSzgDWAA/mFZ+ZmY2X59lHrwPeCfxG0iNJ2SeBd0g6EwjgGeB9ABHxuKQ7gCconrl0hc88MjOrrNySQkT8AtAEs+6eZJnrgOvyisnMzCbnK5rNzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLFVWUpD0unLKzMzs2FbunsKXyywzM7Nj2KTPaJb0GuC1QLOkj2RmnQLU5hmYmZlV3lR7Cg3AyRSTx5zMaz/w9skWlLRM0r2SnpD0uKSrk/L5ku6R9FTyc15SLklfktQm6VFJrzzSX87MzKZn0j2FiPg58HNJX4+Ijmmuewi4JiJ+KWkO8LCke4B3AVsi4npJHwc+DvwNcB6wJnm9Grgp+WlmZhUyaVLImCVpE7Ayu0xEvLnUAhGxC9iVTB+Q9CSwBFgPvDGpdhvwM4pJYT1we0QEcL+kuZIWJ+sxM7MKKDcp3Al8BbgZKEx3I5JWAq8AHgAWZTr63cCiZHoJsCOzWGdSNiopSNoAbABYvnz5dEMxM7NJlJsUhiLipsPZgKSTge8BH4qI/ZLSeRERkmI664uITcAmgJaWlmkta2Zmkyv3lNR/lnS5pMXJgeL5kuZPtZCkeooJ4ZsR8f2keI+kxcn8xUBXUr4TWJZZfGlSZmZmFVJuUrgU+Cjwf4GHk1frZAuouEtwC/BkRNyYmbU5Wd/Ien+YKb8kOQvpbGCfjyeYmVVWWcNHEXHGYaz7dcA7gd9IeiQp+yRwPXCHpMuADuCiZN7dwPlAG9ALvPswtmlmZkegrKQg6ZKJyiPi9lLLRMQvAJWYfc4E9QO4opx4zMwsH+UeaH5VZno2xU79l0DJpGBmZseecoePrsq+lzQX+HYeAZmZWfUc7q2zXwQO5ziDmZnNYOUeU/hnYOSagFrgZcAdeQVlZmbVUe4xhRsy00NAR0R05hCPmZlVUVnDR8mN8X5L8Q6p84CBPIMyM7PqKPfJaxcBDwIXUryu4AFJk94628zMjj3lDh99CnhVRHQBSGoG/h34bl6BmZlZ5ZV79lHNSEJIPDeNZc3M7BhR7p7CjyT9GPhW8v6/UrwthZmZHUemekbzaorPP/iopP8CvD6Z9R/AN/MOzszMKmuqPYX/BXwCILn19fcBJP1xMu8/5xibmZlV2FTHBRZFxG/GFiZlK3OJyMzMqmaqpDB3knmNRzEOMzObAaZKCq2S/mpsoaT3UnzQjpmZHUemOqbwIeAuSf+NQ0mgBWgALsgxLjMzq4JJk0JE7AFeK+lNwMuT4n+NiJ/mHpmZmVVcuc9TuBe4N+dYzMysynxVspmZpZwUzMwslVtSkHSrpC5Jj2XKPiNpp6RHktf5mXmfkNQmaZukP88rLjMzKy3PPYWvA+dOUL4xIs5MXncDSFoHXAz8UbLMP0qqzTE2MzObQG5JISLuA3rKrL4e+HZE9EfE00AbcFZesZmZ2cSqcUzhSkmPJsNL85KyJcCOTJ3OpGwcSRsktUpq7e7uzjtWM7MTSqWTwk3AS4AzgV3AF6e7gojYFBEtEdHS3Nx8lMMzMzuxVTQpRMSeiChExDDwNQ4NEe0ElmWqLk3KzMysgiqaFCQtzry9ABg5M2kzcLGkWZLOANZQfCa0mZlVULlPXps2Sd8C3ggskNQJfBp4o6QzgQCeAd4HEBGPS7oDeAIYAq6IiEJesZmZ2cRySwoR8Y4Jim+ZpP51wHV5xWNmZlPzFc1mZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmKScFMzNLOSmYmVnKScHMzFJOCmZmlnJSMDOzlJOCmZmlcksKkm6V1CXpsUzZfEn3SHoq+TkvKZekL0lqk/SopFfmFZeZmZWW557C14Fzx5R9HNgSEWuALcl7gPOANclrA3BTjnGZmVkJuSWFiLgP6BlTvB64LZm+DXhbpvz2KLofmCtpcV6xmZnZxCp9TGFRROxKpncDi5LpJcCOTL3OpGwcSRsktUpq7e7uzi9SM7MTUNUONEdEAHEYy22KiJaIaGlubs4hMjOzE1elk8KekWGh5GdXUr4TWJaptzQpMzOzCqp0UtgMXJpMXwr8MFN+SXIW0tnAvswwk5mZVUhdXiuW9C3gjcACSZ3Ap4HrgTskXQZ0ABcl1e8GzgfagF7g3XnFZWZmpeWWFCLiHSVmnTNB3QCuyCsWMzMrj69oNjOzlJOCmZmlnBTMzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWSq3W2ebmdmRKRQKtLW1AdDZ2cm86T/BeNqcFMzMZqi2tjbaL7yQVY2N9Hd1EcMNuW/Tw0dmZjPYqsZG1jY1sbQh/4QATgpmZpbhpGBmZiknBTMzS1XlQLOkZ4ADQAEYiogWSfOB7wArgWeAiyLi+WrEZ2Z2oqrmnsKbIuLMiGhJ3n8c2BIRa4AtyXszM6ugmTR8tB64LZm+DXhb9UIxM6u+QqFAX18fvb299A8MVGSb1bpOIYCfSArgqxGxCVgUEbuS+buBRRMtKGkDsAFg+fLllYjVzKwqOjo6eKGtn0JdI7sODnJ6TX3u26xWUnh9ROyUtBC4R9JvszMjIpKEMU6SQDYBtLS05H95n5lZFdXUzKKmtglVICFAlYaPImJn8rMLuAs4C9gjaTFA8rOrGrGZmZ3IKr6nIOkkoCYiDiTTfwZ8FtgMXApcn/z8YaVjMzOrhuw9jgqFAgC1tbUVu99RVjWGjxYBd0ka2f4/RcSPJD0E3CHpMqADuKgKsZmZVVz2Hkdb9+5lAbBu7tyK3e8oq+JJISLagT+ZoPw54JxKx2Nmx56BgQG2bNmSvj/nnHNoqNC9gfIyco+j9r4+FgNrm5p4sqGBwQrH4bukmtkxZ8uWLWy74HKW1c1hx9ABuOsfOe+886od1mErFAoM9PXRC/T39xNVTHBOCmZ2TFpWN4dV9adVO4yjInvqacfAAPMXD9Db1ET/wAA1HOfDR2ZmNt7IqacAzz4bPPkcFbs2YVQcFd2amZlNqUYNFb02YdS2K75FMzObsZwUzMws5aRgZmYpH2g2M6uC7LUWra2tvKzCVy6X4qRgZlYF2Wstegf2MFxzKlT+uPI4TgpmZhUybu+g7mRW1Z9G59CBKkd2iJOCmVmFzNS9gywfaDYzq6CRK7H/oKap2qFMyEnBzMxSHj4ym4bsfe9HrF69mtra2knrTVTHjh/Zz3tgYIDt27dTU1NDoVBgxYoV6R1cq/F8hOlyUjCbhra2Ni68sJ3GxlUA9PW1c+edsHbt2pL1StWx6phuwi5VP1ve3t5OfOxjvKSpiZ93dTH47CDL6+fz8EAPexfCy+fPB+CFnh7mDp+U0292dDgpmE1TY+Mqmpqm7uDLrWf5y3bgbW1tDHz4w5wxezZP9/dTuOsu1q1bV3LZbdu2se2CCzhj1qxR9bPlv9u/n5ZZs1jW1EQzUFN7UnpWUV13UNjfCIAOQtR4T8HMJuAhpsrJPtns6a4uFu6CQsM8hob66ejomDQpdHR0MLQDCnWNo+qPKj/YwzOC+lnj72w6cnM7oCo3uJsuJwXLRSU7vHJ276cbQ6kx4u3bt9PbuySt19fXTqGwYsLl+/o6AHjxxW3ce++ztLe3A4eeElbqG+iRKBV3drvlLAuHP6wysu2j8WS0ctZTiKCzs5Nt27aNiiO7bGdnJ6+fPfvQ08xqoKa2iZrhvrK2NXJb6+HCi7S2tgLJdQY1h+5mWkN91e5sejQ5KVRYHh3YkWy33DrT7Wy2bdvGBRdsZdaspfT3d3LXXQXWrVtX8gHl2e0drW1lx/VffPG3XH31fSxdupTh4WGWL1+ermeitsius6fnR1zT/QOW15/KMwN7eI8aWTlrEQDbh55n69aPjvsdOjo6eG/bh1hWN5eHDu6k94NBb8Np7Bg6QOeXP8kb3vAGtm7dytzMN9CtW7eOW890P8/s79zV9QM++OwtLK8/hY7B/bT+98toaWkZ1Y7Z9Tz11FN85COdzJ69goMHt7NxYxurV68e9Tllp9vb2/nYxwo0Na2ht/cpvvCFdlatKh5rue+++3jxqs9P+mS0bCec/Uyy28iuJ/s7ZG8LsSv6Oe3aa+mbP5+2vj5+9td/zfLly2ltbWXOdbeyrG4OTw/0sHZJPSukkg+uGfs0t5HPqXhwuGh34UVO+txX6G04bcZeZ3CknBQqrJwOrNSBybK+NZXoLEptN2uyGEp1ktl/+Oy2t27dyuU7bmB53Vx2DO2lo2PpuN9z796twALmzl03qlMpdjZBU9NLRnVs2X/U7O/W0dHBFTtuYFnd3KSTPtRpzZ59Bk1Na9m79z5euPI6eutP4ZmB53jfwvcwf/5rOXiwgxtvfIo1a9aMSkCtra1cvuMbLK+bx0MHd7KkpikdI15MPSuT4YCBwee55trtzJ/fTl/f01xzzb2sWLEi/Ra5sraJ7TX1LKaeVfWnMTzcz8c+38/NN0NX126+QDK0MLSXzyfl2bYop0POflYjdUYsrT10xexIZ5Ztx+z4+lM9PVz++3pWNCykdaCLF64EFi7kF88/z6mDg6ydM4df7N9Ps8TaOXPYu38//Q2foqlpDf39O9l3+fX0zZkDFA+oLq05adIno2U74exncuBAKzfWf4N18+bR39XFkpqGcb/DqA55eJC67joK+xvpPNjD8FWfTeusrTk1XbacB9eMXEMwVDiYfq49PY+xcXj4UJvWnDTjrkI+mmZcUpB0LvD3QC1wc0RcX+WQJlXqm2+pb8HldGARhXSoodS3puw/9tjhjRtuGKaxceWoDq9UJz32DIqJYoDRnXy2kxyK4XR3Orvtnp7H2Jh0ngOZXe7t27cTURx+iRgGiv9s/f07+cAHDnLKKXDgQCsnn3w+TU1raWj4A5bWjv9Hzf5unZ2dnF4zq7itTCddXM9CTkpO9hhZT+fQAa7p/gYr9/+EjoEurrrqGhYuXDMqAY10KCOdeim7YzBdV3aPoNS3yOEIQJmSGFee7WCznXCpDjn7dzJ2DyUyF0iNdGajO7yf8MXfB4WGRnQQTq+pzySyQZYBpw4Owi4oPFccOw/VUZjVyJ6DPXxA17Fy9808dHAnJOUw+oBqdnhnbOLN3uYhbceBPZy2WCwDmse0d6kOOftQmqVJAp6szojs329272Ps5xoz9EKzPMyopCCpFvgH4C1AJ/CQpM0R8cTR3M5k37jL+TaerZPtCPfvfxCYyymnvHTUdKlvkNkObP/+B2lo6EUSe/f+n7SDzK6np+cxNk74jz16eOPyZHjj6f49vP/972T+/NpJO+ls/CMx7Nu3lauvnse8ZL955Fvt2E4yuzud3Xb2H2lsnfdk6jSrjpW7M9M9xbi/fPp8mppOZmBgN0zwjzrud0u+yY2tM9F6AJYo+QxUT7aTzn6zLtfIurJ7BKWW3x2DfPDZv2Plc7ePbqMx5SMdbLYTLtUhPzPYw/e+dwkdHR3j9lBKxZDt8FTTNH4sfHhwzDfrpgnHzk/nUBIZKYfRB1R3xwCLPv95uPnmUadrjk2caTuiij2OcrLhoOzneiKZUUkBOAtoi4h2AEnfBtYDRzUpbNmyhZ+uv5RFNY3sGe6j9X9clY61tra2sv9zX55w3ohsnSeG9rFeDZxeP5fHB59jHrWc3jV+uu3K4IW6OTw9tI+Tak5CiEcK+7h4z1c5vec7h+p3Tr6eHZpNTU3NuGVDs9P4IgYZHu6jO/pG1UmXHexm8NqNPF43Z+L4O+eya2gf31q4gdmzi1+zBwd/T2fhADWDYvdwLwXqqB18jt3DvTRn/oxGth0M0jn84pR1ImLcdHf08Red13L6nr8fFffIeib73cbWKbWekfgfKeybsk6paaCsehO1V6k2Gil/NoL64T72MMhwienmpL26CvvTzzP793U4MYytn253kjpTtsvQi8wfhL6+PgYHB0f9Lx3tbR/u51FOPEc6fbh/LxNN7xg6QF4nOyti5pwzK+ntwLkR8d7k/TuBV0fElZk6G4ANydu1wLaKB1pZC4DfVzuIGcDt4DYY4XYoOpJ2WBERY0fngJm3pzCliNgEbKp2HJUiqTUiWqaueXxzO7gNRrgdivJqh5l2Q7ydwLLM+6VJmZmZVcBMSwoPAWsknSGpAbgY2FzlmMzMThgzavgoIoYkXQn8mOIpqbdGxONVDqvaTpihsim4HdwGI9wORbm0w4w60GxmZtU104aPzMysipwUzMws5aQwQ0g6V9I2SW2SPj7B/I9IekLSo5K2SBp/a85j3FRtkKn3F5JC0nF5WmI57SDpouTv4XFJ/1TpGCuhjP+J5ZLulfSr5P/i/GrEmSdJt0rqkvRYifmS9KWkjR6V9Moj3mhE+FXlF8WD6r8DVgENwK+BdWPqvAloSqY/AHyn2nFXug2SenOA+4D7gZZqx12lv4U1wK+Aecn7hdWOu0rtsAn4QDK9Dnim2nHn0A5vAF4JPFZi/vnAv1G8V8vZwANHuk3vKcwM6e09ImIAGLm9Ryoi7o2I3uTt/RSv4TieTNkGic8BfwscrGRwFVROO/wV8A8R8TxARHRVOMZKKKcdAjglmT4VeLaC8VVERNwH9ExSZT1wexTdD8yVtPhItumkMDMsAXZk3ncmZaVcRvHbwfFkyjZIdo2XRcS/VjKwCivnb+GlwEslbZV0f3Jn4eNNOe3wGeAvJXUCdwNXVSa0GWW6fceUZtR1CjY1SX8JtAD/qdqxVJKkGuBG4F1VDmUmqKM4hPRGinuM90n644jYW82gquAdwNcj4ouSXgN8Q9LLo3hPdjtM3lOYGcq6vYekPwU+Bbw1IvorFFulTNUGc4CXAz+T9AzF8dPNx+HB5nL+FjqBzRExGBFPA/+PYpI4npTTDpcBdwBExH8AsyneJO5EctRvDeSkMDNMeXsPSa8AvkoxIRyPY8iTtkFE7IuIBRGxMiJWUjyu8taIaK1OuLkp51YvP6C4l4CkBRSHk9o5vpTTDtuBcwAkvYxiUuiuaJTVtxm4JDkL6WxgX0TsOpIVevhoBogSt/eQ9FmgNSI2A38HnAzcKQlge0S8tWpBH2VltsFxr8x2+DHwZ5KeAArARyPiuepFffSV2Q7XAF+T9GGKB53fFckpOccLSd+i+AVgQXLs5NMkjwGKiK9QPJZyPtAG9ALvPuJtHmdtaGZmR8DDR2ZmlnJSMDOzlJOCmZmlnBTMzCzlpGBmZiknBTMzSzkpmJlZ6v8DM4+YplBIjCUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(ood_scores_init, bins=100, color='blue')\n",
    "sns.histplot(ood_scores_new, bins=100, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ30lEQVR4nO3dfXRcd33n8fd3RpqRHCd+wLIs/BArJ6khS0tg1SwPPd1CSE+WXZKst2RLH+L2uPhgbLYBAk23Pacs3XMKXbYptJyyLqG4eyiQsGbjtils6g3lbAIpUhIIiZNYGUXjccaxHGJZsedBuvPdP+ZKHkkjaSzrzli+n9c5PjP3zr33953fyB/d+c3Vb8zdERGR+Ei0ugAREWkuBb+ISMwo+EVEYkbBLyISMwp+EZGYaWt1AY1Yt26db926tdVliIgsKwMDAyfdvWvm+mUR/Fu3bqW/v7/VZYiILCtmNlxvvYZ6RERiRsEvIhIzCn4RkZhR8IuIxIyCX0QkZhT8IiIxo+AXEYkZBb+ISMxEFvxmts3Mnqj5d9rM7jCztWb2oJkdCW/XRFWDiMhyEwQBQ0NDDA0NEQRBJG1EFvzu/qy7X+fu1wH/EjgLfBO4Czjk7tcAh8JlEREBstksR7dv5+j27WSz2UjaaNZQzw3A8+4+DNwC7A/X7wdubVINIiLLwuZ0ms3pdGTHb1bw/zLw1fB+t7vnw/vHge4m1SAiIjQh+M0sBdwM3DfzMa9+4W/dL/01s11m1m9m/SMjIxFXKSISH8044/83wGPu/lK4/JKZ9QCEtyfq7eTu+9y9z937urpmzSoqIiKL1Izgfx/nhnkADgI7wvs7gPubUIOIiIQiDX4zuwy4EThQs/pTwI1mdgR4V7gsIiJNEukXsbj7GeA1M9a9TPUqHxERaQH95a6ISMwo+EVEYkbBLyISMwp+EZGYUfCLiMSMgl9EJGYU/CIiMaPgFxGJGQW/iEjMKPhFRGJGwS8iEjMKfhGRmFHwi4jEjIJfRCRmFPwiIjGj4BcRiRkFv4hIzCj4RURiRsEvIhIzCn4RkZhR8IuIxEykwW9mq83sG2b2jJkdNrO3mtlaM3vQzI6Et2uirEFERKaL+oz/s8C33P11wBuBw8BdwCF3vwY4FC6LiEiTRBb8ZrYK+HngHgB3L7v7KeAWYH+42X7g1qhqEBGR2aI84+8FRoC/MrPHzeyLZnYZ0O3u+XCb40B3vZ3NbJeZ9ZtZ/8jISIRliojES5TB3wa8GfgLd38TcIYZwzru7oDX29nd97l7n7v3dXV1RVimiEi8RBn8OSDn7o+Gy9+g+ovgJTPrAQhvT0RYg4iIzBBZ8Lv7ceComW0LV90APA0cBHaE63YA90dVg4iIzNYW8fE/BHzFzFJABvhNqr9s7jWzncAwcFvENYiISI1Ig9/dnwD66jx0Q5TtiojI3PSXuyIiMaPgFxGJGQW/iEjMKPhFRGJGwS8iEjMKfhGRmFHwi4jEjIJfRCRmFPwiIjGj4BcRiRkFv4hIzCj4RURiRsEvIhIzCn4RkZhR8IuIxIyCX0QkZhT8IiIxo+AXEYkZBb+ISMwo+EVEYibSL1s3sxeAMSAAJty9z8zWAl8HtgIvALe5+ytR1iEiIuc044z/He5+nbv3hct3AYfc/RrgULgsIiJN0oqhnluA/eH9/cCtLahBROSiEwQBmUyGQrFIoVgkCIJI2ok6+B34P2Y2YGa7wnXd7p4P7x8HuuvtaGa7zKzfzPpHRkYiLlNEpPUGBwfZsyfL4CA891yJ4eHhSNqJdIwf+Dl3P2Zm64EHzeyZ2gfd3c3M6+3o7vuAfQB9fX11txERudR0dGwhkeiItI1Iz/jd/Vh4ewL4JnA98JKZ9QCEtyeirEFERKaLLPjN7DIzu3zyPvCLwI+Bg8COcLMdwP1R1SAiIrNFOdTTDXzTzCbb+Rt3/5aZ/QC418x2AsPAbRHWICIiM0QW/O6eAd5YZ/3LwA1RtSsiIvPTX+6KiMSMgl9EJGYU/CIiMaPgFxGJGQW/iEjMKPhFRGJGwS8iEjMKfhGRmFHwi4jEjIJfRCRmFPwiIjGj4BcRiRkFv4hIzCj4RURiRsEvIhIzCn4RkZhR8IuIxIyCX0QkZhT8IiIx01Dwm9nbG1knIiIXv0bP+P+swXWzmFnSzB43s78Ll3vN7FEzGzSzr5tZqtFiRUTkwrXN96CZvRV4G9BlZh+peegKINlgG78NHA73Afg0cLe7f83MvgDsBP7ivKoWEZFFW+iMPwWspPoL4vKaf6eBX1ro4Ga2Cfi3wBfDZQPeCXwj3GQ/cOsi6hYRkUWa94zf3f8J+Ccz+7K7Dy/i+H8KfJzqLwuA1wCn3H0iXM4BG+vtaGa7gF0AW7ZsWUTTIiJSz7zBXyNtZvuArbX7uPs759rBzP4dcMLdB8zsF863MHffB+wD6Ovr8/PdX0RE6ms0+O8DvkB1yCZocJ+3Azeb2buBDqpj/J8FVptZW3jWvwk4dn4li4jIhWg0+Cfc/bw+gHX33wV+FyA847/T3X/VzO6j+vnA14AdwP3nc1wREbkwjV7O+bdm9kEz6zGztZP/Ftnm7wAfMbNBqmP+9yzyOCIisgiNnvHvCG8/VrPOgasa2dndvwN8J7yfAa5vsF0REVliDQW/u/dGXYiIiDRHQ8FvZrfXW+/uf7205YiISNQaHer52Zr7HcANwGOAgl9EZJlpdKjnQ7XLZraa6lU5IiKyzCx2WuYzgMb9RUSWoUbH+P+W6lU8UJ2c7fXAvVEVJSIi0Wl0jP8zNfcngGF3z0VQj4iIRKyhoZ5wsrZnqE62tgYoR1mUiIhEp9Fv4LoN+GfgvcBtwKNmtuC0zCIicvFpdKjn94CfdfcTAGbWBfwj5+bVFxGRZaLRq3oSk6Efevk89hURkYtIo2f83zKzbwNfDZf/I/BANCWJiEiUFvrO3auBbnf/mJltB34ufOh7wFeiLk5ERJbeQmf8f0o4p767HwAOAJjZT4ePvSfC2kREJAILjdN3u/uTM1eG67ZGUpGIiERqoeBfPc9jnUtYh4iINMlCwd9vZu+fudLMfgsYiKYkERGJ0kJj/HcA3zSzX+Vc0PcBKeDfR1iXiIhEZN7gd/eXgLeZ2TuAN4Sr/97d/2/klYmISCQanY//IeChiGsREZEmiOyvb82sw8z+2cx+aGZPmdl/Cdf3mtmjZjZoZl83s1RUNYiIyGxRTrtQAt7p7m8ErgNuMrO3AJ8G7nb3q4FXgJ0R1iAiIjNEFvxe9Wq42B7+c+CdnJvcbT9wa1Q1iIjIbJFOtGZmSTN7AjgBPAg8D5xy94lwkxywcY59d5lZv5n1j4yMRFmmiEisRBr87h64+3XAJuB64HXnse8+d+9z976urq6oShQRiZ2mTK3s7qeoXhX0VmC1mU1eTbQJONaMGkREpCrKq3q6zGx1eL8TuBE4TPUXwOS3d+0A7o+qBhERma3R+fgXowfYb2ZJqr9g7nX3vzOzp4Gvmdl/BR4H7omwBhERmSGy4Hf3HwFvqrM+Q3W8X0REWkBfnygiEjMKfhGRmFHwi4jEjIJfRCRmFPwiIjGj4BcRiRkFv4hIzCj4RURiRsEvIhIzCn4RkZhR8IuIxIyCX0QkZhT8IiIxo+AXEYkZBb+ISMwo+EVEYkbBLyISMwp+EZGYUfCLiMSMgl9EJGYiC34z22xmD5nZ02b2lJn9drh+rZk9aGZHwts1UdUgIiKzRXnGPwF81N2vBd4C7DGza4G7gEPufg1wKFwWEZEmiSz43T3v7o+F98eAw8BG4BZgf7jZfuDWqGoQEZHZmjLGb2ZbgTcBjwLd7p4PHzoOdM+xzy4z6zez/pGRkWaUKSISC5EHv5mtBP4XcIe7n659zN0d8Hr7ufs+d+9z976urq6oyxQRiY1Ig9/M2qmG/lfc/UC4+iUz6wkf7wFORFmDiMjFLggChoaGyOVylEp55jgfXjJtUR3YzAy4Bzjs7n9S89BBYAfwqfD2/qhqEBFZDrLZLEe3b6fHnduHxqi0XRFpe5EFP/B24NeBJ83siXDdf6Ya+Pea2U5gGLgtwhpERJaFzek03e6sS5Qibyuy4Hf3/wfYHA/fEFW7IiIyP/3lrohIzCj4RURiRsEvIhIzCn4RkZhR8IuIxIyCX0QkZhT8IiIxo+AXEYkZBb+ISMwo+EVEYkbBLyISMwp+EZGYUfCLiMSMgl9EJGYU/CIiMaPgFxGJGQW/iEjMKPhFRGJGwS8iEjMKfhGRmIks+M3sS2Z2wsx+XLNurZk9aGZHwts1UbUvIiL1RXnG/2Xgphnr7gIOufs1wKFwWUREmiiy4Hf37wI/mbH6FmB/eH8/cGtU7YuISH3NHuPvdvd8eP840N3k9kVELipBEJDJZCgUixSKRbwJbbbsw113d5j7OZrZLjPrN7P+kZGRJlYmItI8g4OD7NmTZXAQBo8U8Ur00d/s4H/JzHoAwtsTc23o7vvcvc/d+7q6uppWoIhIs3V0bCGR6MASHU1pr9nBfxDYEd7fAdzf5PZFRGIvyss5vwp8D9hmZjkz2wl8CrjRzI4A7wqXRUSkidqiOrC7v2+Oh26Iqk0RkeUkCAJyuRylUp55PvJccpEFv4iIzC+bzRLs3cvtQ2UqbVc0rV1N2SAi0kKb0mnWJVJNbVNn/CIiTRAEAdlsFoAtW7aQTCZbVouCX0SkCbLZLEe3b68uHDhAb29vy2pR8IuINMnmdLrVJQAKfhGRyARBwODgIAC5XI5ed8ysxVUp+EVEIjM4OMh735uhs/MqRkezHOgo0dnRnL/OnY+CX0QkQp2dV7FixTZKpVyrS5miyzlFRGJGwS8iEjMKfhGRmNEYvyxLtVdLAFx99dVN+YOYVrUr0Yv6tXUPKJZLAOSOHKFQKJDP51lTKjVxlp4qBb8sS7VXSxQKGe67D7Zt23bJtivRi/q1LZWOMfxChWQb3PGBAdrbz9DZeQV/MFTCSTV1/EXBL8vW5NUScWlXohf1a2uJNIlEB6nUBlKpXjo6VmGJNDThW7dqKfhFROZRO8fOxo0bGRoamnqst7eXY8eOTS1v2bIFYGrIKJPJ4N66qRnmouAXEZlH7Rw7mc98hg9/uDw1HHT33Tna77yTzek0R0slOHCAcrk8NWR06lSOdLqHyy5r8ZOYQcEvIrKAyTl2MswcDsqxOZ2mt7Nz2vaT2xQKmeYW2iAF/zIVxRUIF3LMVl3t4h5QKuXI5S68zYvl+dceKwgCUqkUyWRy2lS+59ve5PZBEJDP57nyyiunHpt57MU8t8XW0+j285nZX8DUsSaPO1d7QRDw7LPPks/nyWazFIub6ey8un477uTzeUolgCSFQoaBgSfZUChw1h0HZs7C4x4wPn6MQmEV5fIIzfyWrfko+JepKK5AuJBjtupql2Ixy57n9hLshewDD1zQVLcXy/OfPr/Lo3zO/piOjo5pU/meb3uT2ycSKXY/u4uebZfzZLnMWjPSqdS80wQ30tZi61nq/jp16mFgHatXXzvtuHO1Nzg4yPbtj/LR7B+zrlLig5bir974D3XbebFUYs0nPsEnj6/kKS+zZuIMJ3+/wpG2lZxJwMqrjc4Z+4yPj/Dh4U+z8cXV/LD0Im4rFvUcl5qCfxmL4gqECzlmq6526U6k2bREs91eLM+/dn6XTaTprDOd7/m219l5FWYpepKXszWd5iTQZUZHA1MFN9LWYupZ6v6qDq301D3uXO11dGyhp20V6ysFbIGraza2txMk0pyowGushFk7iUQHyUQKKNXdp9tSbEx2cszaF/PUInFJB/9SvXWf6+1jM2qtvaIAZn9zz3xDHVE8h0a+Rcg9IJMZrvtcap9jbU2N1Fe7/+TVEvPNcFtbaxAElMvlaccHGBoaYnh4eGrbYnHrrLf69dpt1GJ+BoMgoFTKY5aiVMrjqfph5B5QKAxRLA7zyCPHCIKAZDI5daVJ7XM+cuQIxWIHZil8xnBD4M6LR49OtT15Ozw8TDKZJJvNUqlcP9VmJjM8a8hoeHiYUilJZ+fV017/eq8rEH7BOECSYnGY4eFz2wwPD9PT0zNV3+TPRy6Xm1pev349yWSSZDJJJpOhUtlCoTBEuXyc9vb1AFQqZR55pJ9cLkehUMC9Wttkf51bf262zMCdYvEoME4QnKthIY5zpljksYcfZnx8fMEho+qwzzESidO0YvinJcFvZjcBnwWSwBfd/VNRtLNUb93nevvYjFonryiovWqg9i35fEMdUTyHRr5FqFgc5o47iqxezZxvt4FpNTVS3/TnU71aYsWKuc9Wa/suMzrKf/KPs2rV26aOn0qlyLznPZwdhsOVMqv8LB9MrJr1Vr9eu41epbGYn8F8Ps+e5/bS07aKJ8sjjPemoHPmIAKUy3n2Pn8n64Iixz9Q4lde/5dUKuWpK02KxeLUcx4ZeZLfye/nJ4kE7g6cC7sXSyVKu3fD5ZczcPo0a83IF4uczU3w2vY1lMZfZqz3C6xc+S+mXtvOzvy0IaOV5TJ7Xmznnp95YNrrX+91TaVSBHv38smjaZ7yMmsrZY7vdqyjg43t7Qw+O8ZHt+0Lz96r+5ZKeT5weCfdtoKTiQTv33A77e2vC392cphVuPPY7xNMnObuKz8HwNhYP6UP/CGZZJo/2nA7K1duIpHI8sHndrN6dxuZRII/2nA7qdTPTPXFSR9nz3O7gQr5/J9z7bXXNvQ6VyolHhsqY+//BOu9Mu+Q0YiX+Pixz7MhuYrDNoFXxoHmvhtoevCbWRL4PHAjkAN+YGYH3f3pKNpbirfu8719XEpz1VrvqoFa8w11RPEcGvkWoXS6d8G327U1NVrf+V4tMdl3hWKRDrbMOv6mdJpCW5qTlRKvmQCz+l96fSFXaSzmZ7A7kWZjspP8Al/CvcHSrE9UgHY6Oq7CvczklSYF96nnnEptoNtSJC0BPntIYlMqRW9nJ0eLRbrM8EqFs8kUm9vXUKlM3z6drv7hUe2Q0RXuBIn0tG3me103pdMUwiGTdeasSFVruDKVYjAJHR1XUamUp/Y1S7EhuZLXJlbSnkhO/QHUZBvlcoUNlmZixuu3IbmStrYVpFIbzvWtpXht8txxZvd9isWchVeH0TrZQGXBIaNua+O1iTQnSTDXEFGUWnHGfz0w6O4ZADP7GnALEEnwT/5HLRQyZM7j/2wmk6FQqN6vzqNd5OzZzvM+zmLaq20jl8sRjI5SKBbJlUqcePhhMpkM2WyW0VEwaycfjGFjMBo+Nmlym1Ipx6uvPgEMk0i8TLGY5ZFHmHrrXG/7ubbJ5/OsHx0FmKql0bZqtwGmbXe+9U1uPz6+cs7nP1lroVjk+bExTvsAwNTx29vbWTU2RjBRYqRSpuxnSQTO6OjDuI/XrbuR59bI+vkMDAywKhir9nFwhqGzxmV1+vv06RHyE6OMV0q85EzVPTAwQjA6SqlcnnrOZ848zfHgVV6pGBM+wTOn4YVymTEzXnZnzAx354WxMcbMeLFcJgiMxPgrHA9e5cyZx3nllY5Z/f7M6RIvlMtcPjHBiaDM6OjDlEpH531dZ/b7hE+QPOOMj49TaG8nX+c4pVKe48GreKXCyUSCM2eeplwuTbUxPh6QnxglqKl18jknGZ/afnx85azjwLGpfhypjEMwATijAwPTfo6Oh8/TJpj6efmJt0EwQeCVqX0Dr0y9HpM1JMYTjFTOElgboxPjlP0sTLQxwsScxwGn/mDRhbPq277mMbNfAm5y998Kl38d+FfuvnfGdruAXeHiNuDZphYanXXAyVYXcZFQX5yjvjhHfTHdhfTHle7eNXPlRfvhrrvvA/a1uo6lZmb97t7X6jouBuqLc9QX56gvpouiP1oxH/8xYHPN8qZwnYiINEErgv8HwDVm1mvVT9J+GTjYgjpERGKp6UM97j5hZnuBb1O9nPNL7v5Us+tooUtu+OoCqC/OUV+co76Ybsn7o+kf7oqISGvpO3dFRGJGwS8iEjMK/giY2U1m9qyZDZrZXXUe/4iZPW1mPzKzQ2Z2Zb3jXCoW6o+a7f6DmbmZXbKX8jXSF2Z2W/jz8ZSZ/U2za2yWBv6fbDGzh8zs8fD/yrtbUWczmNmXzOyEmf14jsfNzD4X9tWPzOzNF9Sgu+vfEv6j+oH188BVQAr4IXDtjG3eAawI7+8Gvt7qulvZH+F2lwPfBb4P9LW67hb+bFwDPA6sCZfXt7ruFvbFPmB3eP9a4IVW1x1hf/w88Gbgx3M8/m7gH6hO+f8W4NELaU9n/EtvakoKr06cMjklxRR3f8jdz4aL36f6twyXqgX7I/SHwKeBYjOLa7JG+uL9wOfd/RUAdz/R5BqbpZG+cOCK8P4q4MUm1tdU7v5d4CfzbHIL8Nde9X1gtZk1Pn3oDAr+pbcROFqznAvXzWUn1d/kl6oF+yN827rZ3f++mYW1QCM/Gz8F/JSZPWxm3w9nsr0UNdIXnwB+zcxywAPAh5pT2kXpfHNlXhftlA1xYGa/BvQB/7rVtbSKmSWAPwF+o8WlXCzaqA73/ALVd4LfNbOfdvdTrSyqRd4HfNnd/7uZvRX4n2b2BnevtLqw5U5n/EuvoSkpzOxdwO8BN7vXmSf30rFQf1wOvAH4jpm9QHX88uAl+gFvIz8bOeCgu4+7+xDwHNVfBJeaRvpiJ3AvgLt/j+qXCKxrSnUXnyWd6kbBv/QWnJLCzN4E/A+qoX+pjuFOmrc/3H3U3de5+1Z330r1M4+b3b2/NeVGqpHpSv431bN9zGwd1aGfCCYCb7lG+iIL3ABgZq+nGvwjTa3y4nEQuD28uuctwKi75xd7MA31LDGfY0oKM/sk0O/uB4H/BqwE7rPqdwdm3f3mlhUdoQb7IxYa7ItvA79oZk8DAfAxd3+5dVVHo8G++Cjwl2b2Yaof9P6Gh5e4XGrM7KtUf+GvCz/T+APCr+Vy9y9Q/Yzj3cAgcBb4zQtq7xLtRxERmYOGekREYkbBLyISMwp+EZGYUfCLiMSMgl9EJGYU/CIiMaPgFxGJmf8PA9U6ieDxZG4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(ood_scores_init[states==1], bins=100, color='blue')\n",
    "sns.histplot(ood_scores_new[states==1], bins=100, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.8104) tensor(0.9154) tensor(-0.1050)\n"
     ]
    }
   ],
   "source": [
    "print(ood_scores_init[states==1].mean(), ood_scores_new[states==1].mean(), ood_scores_init[states==1].mean()-ood_scores_new[states==1].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQQ0lEQVR4nO3de4zlZX3H8ffX2aXsKjDgjoQKOJDVrcReoCNVMaiYNUBbjJe1kCpixE1atdpaGo1pevujadJYezGaLVIvVYhQaPBeWkGjAeysXMpFzXS61FXbHSxn15UFOtNv/zhn4Owyl9/MOc85Mw/vVzLZc36/33me7zNnzmd/5zm/eSYyE0lSfZ427AIkSWUY8JJUKQNekiplwEtSpQx4SarUhmEX0G3Lli05Pj4+7DIkad3YvXv3g5k5ttC+NRXw4+PjTE5ODrsMSVo3IuKBxfY5RSNJlTLgJalSBrwkVcqAl6RKGfCSVCkDXpIqVTTgI2I0Iq6LiG9HxP0R8eKS/UmSnlD6Ovi/BL6Uma+PiKOAzYX7kyR1FDuDj4jjgHOBjwJk5mOZ2SrVnyStN5nJQw89RKm/y1FyiuY0YAb4u4i4IyKujIinH3lQROyMiMmImJyZmSlYjiStLa1Wiz3bt9NqtYq0XzLgNwBnAR/OzDOBnwDvPfKgzNyVmROZOTE2tuByCpJUreM2lJspLxnwe4G9mXl75/51tANfkjQAxQI+M/8L+F5EbOtseiVwX6n+JEmHK30VzTuBT3WuoJkG3lK4P0lSR9GAz8w7gYmSfUiSFuZvskpSpQx4SaqUAS9JlTLgJalSBrwkVcqAl6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekShnwklQpA16SKmXAS1KlDHhJqpQBL0mVMuAlqVIGvCRVyoCXpEoZ8JJUKQNekiplwEtSpTaUbDwi9gA/BuaA2cycKNmfJOkJRQO+4xWZ+eAA+pEkdXGKRpIqVTrgE/iniNgdETsXOiAidkbEZERMzszMFC5Hkp46Sgf8SzPzLOAC4O0Rce6RB2TmrsycyMyJsbGxwuVI0lNH0YDPzO93/t0H3ACcXbI/SdITigV8RDw9Io6Zvw28CrinVH+SpMOVvIrmROCGiJjv59OZ+aWC/UmSuhQL+MycBn6+VPuSpKV5maQkVcqAl6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekShnwklQpA16SKmXAS1KlDHhJqpQBL0mVMuAlqVIGvCRVyoCXpEoZ8JJUKQNekiplwEtSpQx4SaqUAS9JlTLgJalSBrwkVcqAl6RKFQ/4iBiJiDsi4nOl+5IkPWEQZ/DvAu4fQD+SpC5FAz4iTgZ+GbiyZD+SpCcrfQb/QeD3gP9b7ICI2BkRkxExOTMzU7gcSXrqKBbwEfErwL7M3L3UcZm5KzMnMnNibGysVDmS9JRT8gz+HOCiiNgDXAOcFxF/X7A/SVKXYgGfme/LzJMzcxy4GPhKZr6xVH+SpMN5HbwkVWrDIDrJzFuAWwbRlySpzTN4SaqUAS9JlTLgJalSBrwkVcqAl6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUo0CPiLOabJNkrR2ND2D/+uG2yRJa8SSq0lGxIuBlwBjEfE7XbuOBUZKFiZJ6s1yywUfBTyjc9wxXdsPAK8vVZQkqXdLBnxmfhX4akR8LDMfGFBNkqQ+aPoHP34qInYB492PyczzShQlSepd04C/FvgIcCUwV64cSVK/NA342cz8cNFKJEl91fQyyc9GxG9GxEkRccL8V9HKJEk9aXoG/+bOv1d0bUvg9P6WI0nql0YBn5mnlS5EktRfjQI+Ii5daHtmfqK/5UiS+qXpFM0Lu24fDbwS+BZgwEvSGtV0iuad3fcjYhS4pkRBkqT+WO1ywT8BlpyXj4ijI+KbEXFXRNwbEX+0yr4kSavQdA7+s7SvmoH2ImPPBz6zzMMeBc7LzIMRsRH4ekR8MTNvW3W1kqTGms7B/3nX7Vnggczcu9QDMjOBg527GztfufgjJEn91GiKprPo2Ldpryh5PPBYk8dFxEhE3AnsA27KzNsXOGZnRExGxOTMzEzjwiVJS2v6F53eAHwT2AG8Abg9IpZdLjgz5zLzF4CTgbMj4gULHLMrMycyc2JsbGxFxUuSFtd0iub9wAszcx9ARIwB/wxc1+TBmdmKiJuB84F7VlOoJGllml5F87T5cO/40XKPjYixzuWURMQmYDvtaR5J0gA0PYP/UkR8Gbi6c//XgC8s85iTgI9HxAjt/ww+k5mfW12ZkqSVWu5vsm4FTszMKyLitcBLO7tuBT611GMz827gzL5UKUlaseXO4D8IvA8gM68HrgeIiJ/t7PvVgrVJknqw3Bz8iZn5b0du7GwbL1KRJKkvlgv40SX2bepjHZKkPlsu4Ccj4m1HboyIy4HdZUqSJPXDcnPw7wZuiIhf54lAnwCOAl5TsC5JUo+WDPjM/G/gJRHxCmD+t1A/n5lfKV6ZJKknTdeDvxm4uXAtkqQ+Wu168JKkNc6Al6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekShnwklQpA16SKmXAS1KlDHhJqpQBL0mVMuAlqVIGvCRVyoCXpEoZ8JJUqWIBHxGnRMTNEXFfRNwbEe8q1Zck6cka/dHtVZoF3pOZ34qIY4DdEXFTZt5XsE9JUkexgM/MHwI/7Nz+cUTcDzwbMOB7NDc3x9TU1OP3t27dysjIyLppX1rvMpNWq8Xo6CgR0fNxpQxkDj4ixoEzgdsX2LczIiYjYnJmZmYQ5ax7U1NT7NgxzaWXwo4d04eF8XpoX1rvWq0We7Zvp9Vq9eW4UkpO0QAQEc8A/gF4d2YeOHJ/Zu4CdgFMTExk6XpqsWnT6WzevG3dti+td8dtaBafTY8roegZfERspB3un8rM60v2JUk6XMmraAL4KHB/Zn6gVD+SpIWVPIM/B3gTcF5E3Nn5urBgf5KkLiWvovk6MPiPjSVJgL/JKknVMuAlqVIGvCRVyoCXpEoZ8JJUKQNekiplwEtSpQx4SaqUAS9JlTLgJalSBrwkVcqAl6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekShnwklQpA16SKmXAS1KlDHhJqpQBL0mVKhbwEXFVROyLiHtK9SFJWlzJM/iPAecXbF+StIQNpRrOzK9FxHip9rvNzc0xNTX1+G2AkZERALZu3fr47bWku2bof52LtV+6316VqG/YY85MWq0Wo6OjRMSK9/fLsL8P8/o13l7aafL66CVL5mtrtVrkAv1mJgcOHOCEE06g3DNeMOCbioidwE6AU089dVVtTE1NsWPHNJs2nU6r9Q1gC6OjZ3Do0DTXXgvbtm3rY8X90V1ziToXa790v70qUd+wx9xqtdizfTvjN93E8ccfv+L9/TLs78O8fo23l3aavD56yZJWq0XrggvYPzvLpq7/fObb37jxmfz+/W9m7vQRTti0aUW1r8TQAz4zdwG7ACYmJnKZwxe1adPpbN68jUOHpoGT2Lx57YTWYuZrHnT7pfvtVYn6hj3m4zYs/VJbbn+/DPv7MK9f4+2lneVeH71myWintsc67wS629+48VkcN7KZY0dmV9V2U15FI0mVMuAlqVIlL5O8GrgV2BYReyPiraX6kiQ9WcmraC4p1bYkaXlO0UhSpQx4SaqUAS9JlTLgJalSBrwkVcqAl6RKGfCSVCkDXpIqZcBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekShnwklQpA16SKmXAS1KlDHhJqpQBL0mVMuAlqVIGvCRVyoCXpEoZ8JJUKQNekiplwEtSpQx4SapU0YCPiPMj4jsRMRUR7y3ZlyTpcMUCPiJGgA8BFwBnAJdExBml+pMkHW5DwbbPBqYycxogIq4BXg3cV6KzQ4emAXj00b3AIzz88CYOHZpmerpEb72bnp7m0KH27ZXW2eSxix3Ty2MHoUTfwxwPwP79+5k9eJDp3bs59thjn7T/wIEDbF5if7/s2bOHgwdhdnY/jzyyh7vuatc2aP0aby/tLPa96N7+8MPfAR5k48bHnvT9Wq7vAwcO8NMHD7J/dpbZuTm+1zluvv0NG05g/9zDHD03wsjsLM9a9XdhaZGZZRqOeD1wfmZe3rn/JuCXMvMdRxy3E9jZubsN+E6BcrYADxZod9Acx9pRwxigjnHUMAZY/Tiek5ljC+0oeQbfSGbuAnaV7CMiJjNzomQfg+A41o4axgB1jKOGMUCZcZT8kPX7wCld90/ubJMkDUDJgP9X4LkRcVpEHAVcDNxYsD9JUpdiUzSZORsR7wC+DIwAV2XmvaX6W0bRKaABchxrRw1jgDrGUcMYoMA4in3IKkkaLn+TVZIqZcBLUqWqCvimSyNExOsiIiNiTV5atdw4IuKyiJiJiDs7X5cPo86lNHkuIuINEXFfRNwbEZ8edI1NNHgu/qLrefhuRLSGUOayGozj1Ii4OSLuiIi7I+LCYdS5lAZjeE5E/Eun/lsi4uRh1LmUiLgqIvZFxD2L7I+I+KvOGO+OiLN66jAzq/ii/UHuvwOnA0cBdwFnLHDcMcDXgNuAiWHXvZpxAJcBfzPsWnscw3OBO4DjO/efNey6V/sz1XX8O2lfTDD02lfxfOwCfqNz+wxgz7DrXsUYrgXe3Ll9HvDJYde9wDjOBc4C7llk/4XAF4EAXgTc3kt/NZ3BP740QmY+BswvjXCkPwH+DHhkkMWtQNNxrGVNxvA24EOZ+RBAZu4bcI1NrPS5uAS4eiCVrUyTcSQw/zv3xwE/GGB9TTQZwxnAVzq3b15g/9Bl5teA/1nikFcDn8i224DRiDhptf3VFPDPBr7XdX9vZ9vjOm93TsnMzw+ysBVadhwdr+u8hbsuIk5ZYP8wNRnD84DnRcQ3IuK2iDh/YNU11/S5ICKeA5zGEwGzljQZxx8Cb4yIvcAXaL8bWUuajOEu4LWd268BjomIZw6gtn5q/DPXRE0Bv6SIeBrwAeA9w66lDz4LjGfmzwE3AR8fcj2rsYH2NM3LaZ/5/m1EjA6zoB5dDFyXmXPDLmSVLgE+lpkn054m+GTnNbOe/C7wsoi4A3gZ7d+cX6/PR1+stydwKcstjXAM8ALglojYQ3t+68Y1+EHrsks8ZOaPMvPRzt0rgV8cUG1NNVmmYi9wY2b+b2b+B/Bd2oG/lqxkuY2LWZvTM9BsHG8FPgOQmbcCR9Ne/GqtaPK6+EFmvjYzzwTe39nWGliF/dHXJV5qCvgll0bIzP2ZuSUzxzNznPaHrBdl5uRwyl3Usks8HDEndxFw/wDra6LJMhX/SPvsnYjYQnvKZq0t7txouY2I+BngeODWAdfXVJNx/CfwSoCIeD7tgJ8ZaJVLa/K62NL1ruN9wFUDrrEfbgQu7VxN8yJgf2b+cLWNDX01yX7JRZZGiIg/BiYzc12sg9NwHL8VERcBs7Q/sLlsaAUvoOEYvgy8KiLuo/02+orM/NHwqn6yFfxMXQxck53LINaahuN4D+1pst+m/YHrZWtpPA3H8HLgTyMiaV8p9/ahFbyIiLiadp1bOp93/AGwESAzP0L7848LgSngYeAtPfW3hp5DSVIf1TRFI0nqYsBLUqUMeEmqlAEvSZUy4CWpUga8JFXKgJekSv0/XNFM9HwuKAUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(ood_scores_init[states==2], bins=100, color='blue')\n",
    "sns.histplot(ood_scores_new[states==2], bins=100, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.6469) tensor(0.9412) tensor(-0.2943)\n"
     ]
    }
   ],
   "source": [
    "print(ood_scores_init[states==2].mean(), ood_scores_new[states==2].mean(), ood_scores_init[states==2].mean()-ood_scores_new[states==2].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAX2UlEQVR4nO3dfXRbd33H8fdXciU5zVNZnCYksePWkLOu7ByY2djY2VPZTtdtdA+B0TOeCxnb2q0D2gPjD7Zxds42GLAxDixjXcdgZSNjo7AHYNDQFtoOt6WsKWsT7Mqxk9aBLHFwLMmWvvtDkiu7kiU7uvc6+X1e5+REurr+/b73J/vjmyvpG3N3REQkHKmkCxARkXgp+EVEAqPgFxEJjIJfRCQwCn4RkcD0JF1AJ7Zs2eK7d+9OugwRkfPKAw888G1371u6/bwI/t27dzMyMpJ0GSIi5xUzyzfbrks9IiKBUfCLiARGwS8iEhgFv4hIYBT8IiKBUfCLiAQmsuA3s1vNbMrMHmny2FvMzM1sS1Tzi4hIc1Ge8d8GXL10o5ntAn4GGI9wbhERaSGyD3C5+11mtrvJQ+8DbgE+HdXcIiLnq3K5zNjY2ML9wcFB0ul0V+eI9ZO7ZnYtMOnuD5tZu333AfsA+vv7Y6hORCR5Y2Nj5PfuZSCbJV8swoEDDA0NdXWO2ILfzNYBv0f1Mk9b7r4f2A8wPDys/yZMRIIxkM0ytG5dZOPH+a6ey4FB4GEzewLYCTxoZttirEFEJHixnfG7+/8AW+v3a+E/7O7fjqsGERGJ9u2ctwP3AnvMbMLMro9qLhER6VyU7+q5rs3ju6OaW0REWtMnd0VEAqPgFxEJjIJfRCQwCn4RkcAo+EVEAqPgFxEJjIJfRCQwCn4RkcAo+EVEAqPgFxEJjIJfRCQwCn4RkcAo+EVEAqPgFxEJjIJfRCQwCn4RkcAo+EVEAqPgFxEJjIJfRCQwCn4RkcBEFvxmdquZTZnZIw3b3m1m/2tm3zCzfzGzzVHNLyIizUV5xn8bcPWSbV8ArnT37wceB94e4fwiItJEZMHv7ncBJ5ds+7y7z9fu3gfsjGp+ERFpLslr/K8H/qPVg2a2z8xGzGzkxIkTMZYlInJhSyT4zewdwDzw8Vb7uPt+dx929+G+vr74ihMRucD1xD2hmb0W+HngKnf3uOcXEQldrMFvZlcDtwA/7u5n45xbRESqonw75+3AvcAeM5sws+uBvwQ2AF8ws6+b2Yejml9ERJqL7Izf3a9rsvlvoppPREQ6o0/uiogERsEvIhIYBb+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigVHwi4gEJrLgN7NbzWzKzB5p2PYsM/uCmR2u/X1JVPOLiEhzUZ7x3wZcvWTb24AvuvtzgC/W7ouISIwiC353vws4uWTztcDf1W7/HfCLUc0vIiLNxX2N/1J3P167/SRwaasdzWyfmY2Y2ciJEyfiqU5EJACJvbjr7g74Mo/vd/dhdx/u6+uLsTIRkQtb3MH/lJltB6j9PRXz/CIiwYs7+O8AXlO7/Rrg0zHPLyISvCjfznk7cC+wx8wmzOx64I+Bnzazw8BLavdFRCRGPVEN7O7XtXjoqqjmFBGR9vTJXRGRwCj4RUQCo+AXEQmMgl9EJDAKfhGRwCj4RUQCo+AXEQmMgl9EJDAKfhGRwCj4RUQCo+AXEQmMgl9EJDAKfhGRwCj4RUQCo+AXEQmMgl9EJDAKfhGRwCj4RUQCo+AXEQmMgl9EJDCJBL+Z/a6ZHTKzR8zsdjPLJVGHiEiIOgp+M3txJ9s6HGsH8NvAsLtfCaSBV6xmLBERWblOz/g/0OG2TvUAvWbWA6wDjp3DWCIisgI9yz1oZj8M/AjQZ2ZvbnhoI9Uz9RVz90kzew8wDswCn3f3zzeZex+wD6C/v381U4mISBPtzvgzwHqqvyA2NPyZBvauZkIzuwS4FhgEng1cbGavXLqfu+9392F3H+7r61vNVCIi0sSyZ/zu/mXgy2Z2m7vnuzTnS4Axdz8BYGafovqvio91aXwREVnGssHfIGtm+4HdjV/j7j+1ijnHgReZ2Tqql3quAkZWMY6IiKxCp8H/SeDDwEeA8rlM6O73m9kB4EFgHngI2H8uY4qISOc6Df55d/9QtyZ193cC7+zWeCIi0rlO3875GTP7TTPbbmbPqv+JtDIREYlEp2f8r6n9fXPDNgcu6245IiIStY6C390Hoy5ERETi0VHwm9mrm2139492txwREYlap5d6XthwO0f1LZgPAgp+EZHzTKeXem5svG9mm4FPRFGQiIhEa7VtmWeotlwQEZHzTKfX+D9D9V08UG3O9r3AP0VVlIiIRKfTa/zvabg9D+TdfSKCekREOlIulxkbG1u4Pzg4SDq9qqbBwen0Gv+XzexSnn6R93B0JYmItDc2NkZ+714GslnyxSIcOMDQ0FDSZZ0XOv0fuF4O/DfwMuDlwP1mtqq2zCIi3TKQzTK0bh0D2WzSpZxXOr3U8w7ghe4+BWBmfcB/AQeiKkxERKLR6bt6UvXQr/nOCr5WRETWkE7P+P/TzD4H3F67/6vAv0dTkoiIRKnd/7k7BFzq7jeb2S8DP1p76F7g41EXJyIi3dfujP/9wNsB3P1TwKcAzOx5tcd+IcLaREQkAu2u01/q7v+zdGNt2+5IKhIRkUi1C/7NyzzW28U6REQkJu2Cf8TM3rh0o5m9AXggmpJERCRK7a7x3wT8i5n9Gk8H/TCQAX4pwrpERCQiywa/uz8F/IiZ/SRwZW3zv7n7lyKvTEREItFpr547gTu7NWmtn/9HqP4yceD17n5vt8YXEZHWOv0AV7f9OfCf7r7XzDLAuoTqEBEJTuzBb2abgB8DXgvg7iWgFHcdIiKhSqLfziBwAvhbM3vIzD5iZhcv3cnM9pnZiJmNnDhxIv4qRUQuUEkEfw/wAuBD7v58qv+N49uW7uTu+9192N2H+/r64q5RROSClUTwTwAT7n5/7f4Bqr8IREQkBrEHv7s/CRw1sz21TVcBj8Zdh4hIqJJ6V8+NwMdr7+gZBV6XUB0iIsFJJPjd/etUPwEsIiIx0/+iJSISGAW/iEhgFPwiIoFR8IuIBEbBLyISGAW/iEhgFPwiIoFR8IuIBEbBLyISGAW/iEhgFPwiIoFR8IuIBEbBLyISGAW/iEhgFPwiIoFR8IuIBEbBLyISGAW/iEhgFPwiIoFR8IuIBCax4DeztJk9ZGafTaoGEZEQJXnG/zvANxOcX0QkSIkEv5ntBH4O+EgS84uIhKwnoXnfD9wCbIhyknK5zJEjRxbuDw0NkU6nz5vxpTWtvcStXC4zNja2cH9wcHDF33Otxmjcns/n6e9OyS3FHvxm9vPAlLs/YGY/scx++4B9AP39q1uGI0eO8LKXjdLbexmzs6N88pOwZ8+eVY2VxPjSmtZe4jY2NkZ+714GslnyxSIcOMDQ0FBXxmjcPjk9zdZsFtati+hIkjnjfzHwUjO7BsgBG83sY+7+ysad3H0/sB9geHjYVztZb+9lrFsXXSBEPb60prWXuA1kswydYyC3GqO+PV8onNP4nYj9Gr+7v93dd7r7buAVwJeWhr6IiERH7+MXEQlMUi/uAuDuB4GDSdYgIhIanfGLiARGwS8iEhgFv4hIYBT8IiKBUfCLiARGwS8iEhgFv4hIYBT8IiKBSfQDXOeLtdIJcq3U0cxKa1vLxyLR6EZ3ywtJ2Z2JfB6IpyNnIwV/B9ZKJ8i1UkczK61tLR+LRKMb3S0vJBPFIoUbboCNG2PpyNlIwd+htdIJcq3U0cxKa1vLxyLR6EZ3ywtJf4wdORvpGr+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigVHwi4gERsEvIhIYBb+ISGAU/CIigYk9+M1sl5ndaWaPmtkhM/uduGsQEQlZEr165oG3uPuDZrYBeMDMvuDujyZQi4hIcGIPfnc/Dhyv3T5jZt8EdgCJB39jq+ByuQxAOp1mdHQU90EA3MuMjlZbqTZuj2puaN6yuLGOxv07+dqVztutFsqN47Ra06U1rOR46uPna61ut2/fvrDvxMQEAwMDC90gWx3z4OAg4+PjlMtl8vk8O3fuJJ1ON523sc1wuVymXC53vO6t1qXT57JVi+O4Wx9HOV/j2O3GbWxx3Gz/uOps9fx1Op+7UygWKQJnz57FAetKlYsl2p3TzHYDzwfub/LYPmAfQH9/PJ2qG1sFnzr1FWALmzdfwalTE2Sz27n4YigU8tx0U4HNm1m0Paq5W7UsXlxHY63tv3al83arhfLiuVqt6dM1ACs6nrGxMY5cey2FcZgCfv3SV5HJXEEudzH7Hn8TcwMZ0p/9LOVyueUxv+c9edK33MJWdw4/foabn/th3MtN521sM3zk9Gne7DezefOPrniNWn/ftT72Vi2O4259HOV89bGBtuM2tjhuVkccdQ5ks9w/Pc33AHs2bly4nctmO55vdnaWsSdKFFMZKqkCvZdDb1eqXCyx4Dez9cA/Aze5+/TSx919P7AfYHh42OOqq94qeHZ2FNjecPtp2exg0+1RzL2cxXVsX9HXrnTebrVQXjzX05odC7Di4+nPZin15OgBMpltZLOD5HLr2ZHeQH+2VR2N4+cZyGZ5NvCtNORylwHllvPV2wzPFgrk2LXqNWr9fdf62Fu1OI679XGU8w1ks+13qulvU0fUddbbK281W3S7dwXHAJBOZUilcqRTWSCads2JvKvHzC6iGvofd/dPJVGDiEioknhXjwF/A3zT3d8b9/wiIqFL4oz/xcCrgJ8ys6/X/lyTQB0iIkFK4l099xDNC9UiItIBfXJXRCQwCn4RkcAo+EVEAqPgFxEJjIJfRCQwCn4RkcAo+EVEAqPgFxEJTKLdOePU2PoXVt9auD7W2bNHKBaPAXO4P2/Z/Ttpu1tvU+xeXhh3ZqaXQmGUfL5a5+joKIVCmt7ec+sqWCqVOHjwIABHjx6lUnlR02Osr9fhw4eZnb0ISONeBp5uDdyszXKrY1+6T6t1rG8HKBQmgCJnz66nUikxOjqx7HytOE6hWOL46CjlcrlpHbOzo4yMPMS22VlmzWjXGbBcLlMqFDgLzBQKFJhcGGd0dPFzXLf0eS+XywvPa319bQUfb3R3vjs7y0N3383hw4c5duwYL5yd5SwstPRdaUvipe2td+7cydDQEOPj4wuP1+vP5/O0651bn7/Vvq3aW+fzeQYbjuGxxx6jXC4vtNiemJjo+tx19Zbc9eNceqyNbaDz+Ty7qLZRLhSLFICZmZmF9soXZTJM5vOUy2W++tWv8gMNz09Sggn+xta/59JaGKBUmuS3xq6nr1LgSXc+Wmsh3EonbXfrbYrNxvjt8T9im13MoaPzfI8X2X7zBu4vldhQLPKmJzPc9rzPrqruuoMHD3L42jexI72B2bmTTO/+IOvXf9+ifRrXa2rqYd765EdJp3v50OV/Cgw0Oa7mLaqX26fVOta3b09leah4jC300HNsE+/bcSM33fSsVbXErlRK5J+o8K6bq79MWtXxjd//Lkd61jOLUyGz7Jj5fJ4zh4uUenJ8rTjDjfwhu7Jb+XrpBIXfAvr6Flrzfgee0a63fntDscgfTG3gSYz37biRbHZnZwdFtY3vPY9/F/vNdzHjznhljkt61lNI2UJL35W2JF7a3vrGXTfzgQ9MkL7llme0Hp6cnmZrNgvLdL2szz9ZLDbdt1V761OnjnIgV6A3lyOfz/PWt5YxS7Pv8Tex/bkbmCyVujp3/bj6M5mFltyFwlHea+9mz6ZNi461sQ305PQ0m9x5ZCLLeAXOlmeopC9iHJitlOjdPQ033EApkyH/vyfZlL6YQsoo7ixCLtf+SY5AMMEPT7f+7YbtqSzbcahUOtq/XdvdxjbF2yzDrlSOE5TYWnEuz+WYAja5Y6mVtXhtZUd6AwMXPYtypdhyn/p6ZTLb2GYZeprM3arNcqf7tFrH7aks/el1HLWL2EYPF9XmPpeW2JbKkstdVvtXS/M6jqcqpFI5rMPnNp3KkkqvwyzDNnroT6/jWCrDrgyLWvPm3J/Rrrd+u/q85lbdxySdyrA9lePZOE/OzzZt6bvSlsSN7a1zuV2LxlhafycGslnw1ue4zdpbFwr5Rfv09l4GpNmR3rDwM9HNuevHdXk229CSG/rJNj3W/oavo1gkXXsOrVKqPZ+phWvp/dksl2ezXJq+uOH5af2zFzVd4xcRCYyCX0QkMAp+EZHAKPhFRAKj4BcRCYyCX0QkMAp+EZHAKPhFRAKj4BcRCYyCX0QkMIkEv5ldbWaPmdkRM3tbEjWIiIQq9uA3szTwQeBngSuA68xs+S5nIiLSNUk0aftB4Ii7jwKY2SeAa4FHo5is3syrWJwACpw927vQNnep0dFRZmd5xv5Lb5dKcxybP81cpciTDqdP3417hXvuYaGVbaOjR49y6lS14+WZMw8Cl2A21fR2qdTL8fIMXqnwFPPM+xyHTsNoqcTGuTm+XS5x+vTdtVbGrccxm6JQONq0ppGRETaVzwBwvDzDzMyDnDyZaznOzMwhjpe/S5o5pqdHuOeeo+Tz+ZbH1Tjvcsc+N/fMdSwWjy1s90qBE5VZnDQ98ylmZg5RKhVbHmcqlWLzmTNU5otMwcL+9TWFnqZr11jHicocVp5n3r3tc9u4jgt1zp3kqfIM35qB7MmTjJ45wzTwbWAawH1hW/32xrk5euZZVHO753JycpKtp0/zf8BT5RkqXmLefaH+ciVD6kyJU3ffDcDW06eZLRQYLxaZuvtuRpv9ANRMTk4uWsfp6RFGRmCuNsbS+uu3W41dr/VYqdR03/rjs4UCR86cYdpHoDbvETtDpljk4ZERTp06ilmKyfIZDp0uMtowXqs6VjJ3fYxCJsPkws/ZcY5Y9TluddyNz+FTlRJzlVmYP73w85uaqbZCL2QyC89VuZLhqZkZSqXSovGW7pMrQnfaSi5mvkzXuiiY2V7gand/Q+3+q4Afcvcbluy3D9hXu7sHeCzWQju3herPtSymdWlO69Ka1qa5c1mXAXfvW7pxzbZldvf9wP6k62jHzEbcfTjpOtYarUtzWpfWtDbNRbEuSby4Ownsari/s7ZNRERikETwfw14jpkNmlkGeAVwRwJ1iIgEKfZLPe4+b2Y3AJ+j+p+33uruh+Kuo4vW/OWohGhdmtO6tKa1aa7r6xL7i7siIpIsfXJXRCQwCn4RkcAo+DvQrsWEmb3ZzB41s2+Y2RfNbCCJOpPQafsNM/sVM3MzC+Ltep2si5m9vPZ9c8jM/iHuGpPQwc9Sv5ndaWYP1X6erkmizriZ2a1mNmVmj7R43MzsL2rr9g0ze8E5Teju+rPMH6ovQH8LuAzIAA8DVyzZ5yeBdbXbvwH8Y9J1r5W1qe23AbgLuA8YTrrutbAuwHOAh4BLave3Jl33GlmX/cBv1G5fATyRdN0xrc2PAS8AHmnx+DXAfwAGvAi4/1zm0xl/ewstJty9BNRbTCxw9zvd/Wzt7n1UP5sQgrZrU/Mu4E+AQpzFJaiTdXkj8EF3/z8Ad5+KucYkdLIuDmys3d4EHIuxvsS4+13AyWV2uRb4qFfdB2w2s+2rnU/B394O4GjD/Ynatlaup/qbOQRt16b2T9Jd7v5vcRaWsE6+Z54LPNfMvmJm95nZ1bFVl5xO1uX3gVea2QTw78CN8ZS25q00h5a1Zls2nI/M7JXAMPDjSdeyFphZCngv8NqES1mLeqhe7vkJqv9CvMvMnufup5Isag24DrjN3f/MzH4Y+Hszu9LdK0kXdiHRGX97HbWYMLOXAO8AXuruxZhqS1q7tdkAXAkcNLMnqF6bvCOAF3g7+Z6ZAO5w9zl3HwMep/qL4ELWybpcD/wTgLvfC+SoNikLXVdb3Sj422vbYsLMng/8FdXQD+Fabd2ya+Pup919i7vvdvfdVF//eKl7re/uhauTtiT/SvVsHzPbQvXST+teyReGTtZlHLgKwMy+l2rwn4i1yrXpDuDVtXf3vAg47e7HVzuYLvW04S1aTJjZHwIj7n4H8G5gPfBJMwMYd/eXJlZ0TDpcm+B0uC6fA37GzB4FysDN7v6d5KqOXofr8hbgr83sd6m+0Ptar72t5UJmZrdTPRHYUnt9453ARQDu/mGqr3dcAxwBzgKvO6f5AlhTERFpoEs9IiKBUfCLiARGwS8iEhgFv4hIYBT8IiKBUfCLiARGwS8iEpj/B+ZBNDiynVh8AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(ood_scores_init[states==3], bins=100, color='blue')\n",
    "sns.histplot(ood_scores_new[states==3], bins=100, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.4290) tensor(0.7837) tensor(-0.3547)\n"
     ]
    }
   ],
   "source": [
    "print(ood_scores_init[states==3].mean(), ood_scores_new[states==3].mean(), ood_scores_init[states==3].mean()-ood_scores_new[states==3].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.5078, ts_macro_f1=0.2298\n",
      "ts_micro_f1=0.5303, ts_macro_f1=0.2481\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# Cheat perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        col_idx_set = target_col_mask.unique().tolist()\n",
    "        assert -1 not in col_idx_set\n",
    "        for x in itertools.permutations(col_idx_set):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                logits = logits_temp.clone()\n",
    "                corrected += 1\n",
    "                break\n",
    "            \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 5])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor([2,5,1,0]).unique(sorted=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.tensor([2,5,1,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "new = []\n",
    "for k in t:\n",
    "    if k not in new:\n",
    "        new.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor(2), tensor(5), tensor(1), tensor(0)]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "333"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_permutation(x):\n",
    "    new = []\n",
    "    for k in x.tolist()[0]:\n",
    "        if k not in new:\n",
    "            new.append(k)\n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4068173671.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp.append(F.softmax(logits_temp).max().item())\n",
      "/tmp/ipykernel_4054970/4068173671.py:29: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_entropy.append(compute_entropy(F.softmax(logits_temp)).item())\n",
      "/tmp/ipykernel_4054970/4068173671.py:31: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  if F.softmax(logits_temp).argmax().item() != batch[\"label\"].item() and batch_idx in MSP_corrected_ids:\n",
      "/tmp/ipykernel_4054970/4068173671.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  elif F.softmax(logits_temp).argmax().item() != batch[\"label\"].item() and batch_idx not in MSP_corrected_ids:\n",
      "/tmp/ipykernel_4054970/4068173671.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  elif F.softmax(logits_temp).argmax().item() == batch[\"label\"].item() and batch_idx not in MSP_corrected_ids:\n",
      "/tmp/ipykernel_4054970/4068173671.py:37: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  elif F.softmax(logits_temp).argmax().item() == batch[\"label\"].item() and batch_idx in MSP_corrected_ids:\n"
     ]
    }
   ],
   "source": [
    "# Cheat brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "init_logits = []\n",
    "init_msp = []\n",
    "init_entropy = []\n",
    "msp_log = []\n",
    "entropy_log = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask:\n",
    "        cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "        logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)    \n",
    "        init_msp.append(F.softmax(logits_temp).max().item())\n",
    "        init_entropy.append(compute_entropy(F.softmax(logits_temp)).item())\n",
    "        init_logits.append(logits_temp.clone().detach().cpu())\n",
    "        if F.softmax(logits_temp).argmax().item() != batch[\"label\"].item() and batch_idx in MSP_corrected_ids:\n",
    "            msp_log.append(0)\n",
    "        elif F.softmax(logits_temp).argmax().item() != batch[\"label\"].item() and batch_idx not in MSP_corrected_ids:\n",
    "            msp_log.append(1)\n",
    "        elif F.softmax(logits_temp).argmax().item() == batch[\"label\"].item() and batch_idx not in MSP_corrected_ids:\n",
    "            msp_log.append(2)\n",
    "        elif F.softmax(logits_temp).argmax().item() == batch[\"label\"].item() and batch_idx in MSP_corrected_ids:\n",
    "            msp_log.append(3)\n",
    "        else:\n",
    "            raise ValueError()\n",
    "        # if compute_entropy(F.softmax(logits_temp)).argmax().item() != batch[\"label\"].item() and batch_idx in Entropy_corrected_ids:\n",
    "        #     entropy_log.append(0)\n",
    "        # elif compute_entropy(F.softmax(logits_temp)).argmax().item() != batch[\"label\"].item() and batch_idx not in Entropy_corrected_ids:\n",
    "        #     entropy_log.append(1)\n",
    "        # elif compute_entropy(F.softmax(logits_temp)).argmax().item() == batch[\"label\"].item() and batch_idx not in Entropy_corrected_ids:\n",
    "        #     entropy_log.append(2)\n",
    "        # elif compute_entropy(F.softmax(logits_temp)).argmax().item() == batch[\"label\"].item() and batch_idx in Entropy_corrected_ids:\n",
    "        #     entropy_log.append(3)\n",
    "        # else:\n",
    "        #     raise ValueError()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1085 tensor(577)\n"
     ]
    }
   ],
   "source": [
    "num_cols = torch.tensor(num_cols)\n",
    "print(len(num_cols), (num_cols>0).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_mistakes = 333\n",
    "origin_correct = 244\n",
    "best_corrected = 78"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "244 0.42287694974003465\n"
     ]
    }
   ],
   "source": [
    "print(len(entropy_log) - total_mistakes, (len(entropy_log) - total_mistakes)/len(entropy_log))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "322 0.5580589254766031\n"
     ]
    }
   ],
   "source": [
    "print(len(entropy_log) - total_mistakes +best_corrected, (len(entropy_log) - total_mistakes +best_corrected)/len(entropy_log))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 1, 0,  ..., 3, 3, 0])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_msp = torch.tensor(init_msp)\n",
    "init_entropy = torch.tensor(init_entropy)\n",
    "msp_log = torch.tensor(msp_log)\n",
    "entropy_log = torch.tensor(entropy_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "577 577\n",
      "0 tensor(152) tensor(155)\n",
      "1 tensor(196) tensor(311)\n",
      "2 tensor(215) tensor(99)\n",
      "3 tensor(14) tensor(12)\n"
     ]
    }
   ],
   "source": [
    "# 2 indicates single column能划分正确\n",
    "print(len(msp_log), len(entropy_log))\n",
    "for i in range(4):\n",
    "    print(i, (msp_log==i).sum(), (entropy_log==i).sum())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "577 0\n",
      "0 tensor(164)\n",
      "1 tensor(184)\n",
      "2 tensor(214)\n",
      "3 tensor(15)\n"
     ]
    }
   ],
   "source": [
    "# permuation starting with 0\n",
    "print(len(msp_log), len(entropy_log))\n",
    "for i in range(4):\n",
    "    print(i, (msp_log==i).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.8917)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAS10lEQVR4nO3df5Tld13f8ecrWRNEwQ3smKb7g11lBSO1h3RECj2ICdo1KhuQhuT4Y9HgCgbUQkUoPcVj5RSqRwRLqWuSZvHQ/CDFk6X8sDEk5qhscEJ+B4E1JGQ2ITsIgZ7SCovv/nG/++U6zuzc2Z3v/d7d+3ycc898v5/v9977mjuz89rvj/u9qSokSQI4pe8AkqTJYSlIklqWgiSpZSlIklqWgiSpta7vAMdjw4YNtXXr1r5jSNIJ5bbbbvt8Vc0steyELoWtW7cyNzfXdwxJOqEkeXC5Ze4+kiS1LAVJUstSkCS1LAVJUquzUkhyRZJDSe5ZYtlrk1SSDc18krwjyYEkdyU5p6tckqTldbmlcCWwY/Fgks3ADwOfHRr+EWB7c9sNvKvDXJKkZXRWClV1C/CFJRa9DXgdMHx51p3Au2tgP7A+yVldZZMkLW2sxxSS7AQOVtWdixZtBB4amp9vxpZ6jN1J5pLMLSwsdJRUkqbT2EohyeOBfwv8++N5nKraU1WzVTU7M7PkG/IkScdonFsK3wlsA+5M8gCwCfh4kn8EHAQ2D627qRmTpKmxcfMWkox027h5SycZxnaZi6q6G/j2I/NNMcxW1eeT7ANeleRq4PuBL1XVI+PKJkmT4OH5h3jp7//FSOte8wvP6SRDl6ekXgV8FHhakvkklxxl9Q8C9wMHgD8AfrGrXJKk5XW2pVBVF6+wfOvQdAGXdpVFkjQa39EsSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKklqUgSWp1VgpJrkhyKMk9Q2O/leSvktyV5I+SrB9a9oYkB5J8Msm/7CqXJGl5XW4pXAnsWDR2A/CMqvpe4FPAGwCSnA1cBHxPc5//kuTUDrNJkpbQWSlU1S3AFxaN/a+qOtzM7gc2NdM7gaur6m+r6jPAAeBZXWWTJC2tz2MKPwd8qJneCDw0tGy+GfsHkuxOMpdkbmFhoeOIkjRdeimFJG8EDgPvWe19q2pPVc1W1ezMzMzah5OkKbZu3E+Y5GXAjwHnVVU1wweBzUOrbWrGJEljNNYthSQ7gNcBL6yqrwwt2gdclOT0JNuA7cDHxplNktThlkKSq4DnAxuSzANvYnC20enADUkA9lfVK6rq3iTXAvcx2K10aVV9vatskqSldVYKVXXxEsOXH2X9NwNv7iqPJGllvqNZktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBktTqrBSSXJHkUJJ7hsaelOSGJJ9uvp7RjCfJO5IcSHJXknO6yiVJWl6XWwpXAjsWjb0euLGqtgM3NvMAPwJsb267gXd1mEuStIzOSqGqbgG+sGh4J7C3md4LXDA0/u4a2A+sT3JWV9kkSUsb9zGFM6vqkWb6c8CZzfRG4KGh9eabsX8gye4kc0nmFhYWuksqSVOotwPNVVVAHcP99lTVbFXNzszMdJBMkqbXuEvh0SO7hZqvh5rxg8DmofU2NWOSpDEadynsA3Y107uA64fGf6Y5C+nZwJeGdjNJksZkXVcPnOQq4PnAhiTzwJuAtwDXJrkEeBC4sFn9g8D5wAHgK8DPdpVLkrS8zkqhqi5eZtF5S6xbwKVdZZEkjcZ3NEuSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKnVSykk+ddJ7k1yT5KrkjwuybYktyY5kOSaJKf1kU2SptnYSyHJRuCXgNmqegZwKnAR8FbgbVX1VOCLwCXjziZJ026kUkjy3FHGVmEd8M1J1gGPBx4BzgWua5bvBS44jseXJB2DUbcUfm/EsRVV1UHgt4HPMiiDLwG3AY9V1eFmtXlg47E8viTp2K072sIk/xx4DjCT5DVDi57IYLfPqiU5A9gJbAMeA94L7FjF/XcDuwG2bNlyLBEkSctYaUvhNOBbGZTHE4ZuXwZecozP+QLgM1W1UFVfA94HPBdY3+xOAtgEHFzqzlW1p6pmq2p2ZmbmGCNIkpZy1C2FqvpT4E+TXFlVD67Rc34WeHaSxwP/FzgPmANuYlA0VwO7gOvX6PkkSSM6aikMOT3JHmDr8H2q6tzVPmFV3ZrkOuDjwGHgdmAP8AHg6iS/2YxdvtrHliQdn1FL4b3AfwUuA75+vE9aVW8C3rRo+H7gWcf72JKkYzdqKRyuqnd1mkSS1LtRT0l9f5JfTHJWkicduXWaTJI0dqNuKexqvv7q0FgB37G2cSRJfRqpFKpqW9dBJEn9G6kUkvzMUuNV9e61jSNJ6tOou4++b2j6cQzeW/BxwFKQpJPIqLuPXj08n2Q9gzeZSZJOIsd66ez/w+DaRZKkk8ioxxTez+BsIxhcCO+7gWu7CiVJ6seoxxR+e2j6MPBgVc13kEeS1KORdh81F8b7KwZXSD0D+GqXoSRJ/Rj1k9cuBD4G/CvgQuDWJMd66WxJ0oQadffRG4Hvq6pDAElmgD/hGx+fKUk6CYx69tEpRwqh8TeruK8k6QQx6pbCh5P8MXBVM/9S4IPdRJIk9WWlz2h+KnBmVf1qkhcD/6JZ9FHgPV2HkySN10pbCr8LvAGgqt7H4POUSfJPmmU/3mE2SdKYrXRc4MyqunvxYDO2tZNEkqTerFQK64+y7JvXMIckaQKsVApzSX5+8WCSlwO3dRNJktSXlY4p/ArwR0l+km+UwCxwGvCiDnNJknpw1FKoqkeB5yT5QeAZzfAHquojx/OkzaW3L2ses4CfAz4JXMPgWMUDwIVV9cXjeR5J0uqMeu2jm6rq95rbcRVC4+3Ah6vq6cA/BT4BvB64saq2Azc285KkMRr7u5KTfBvwPOBygKr6alU9BuwE9jar7QUuGHc2SZp2fVyqYhuwAPy3JLcnuSzJtzA4/fWRZp3PAWcudecku5PMJZlbWFgYU2RJmg59lMI64BzgXVX1TAaf4vb3dhVVVfGND/Vh0bI9VTVbVbMzMzOdh5WkadJHKcwD81V1azN/HYOSeDTJWQDN10PL3F+S1JGxl0JVfQ54KMnTmqHzgPuAfcCuZmwXcP24s0nStBv1Kqlr7dXAe5KcBtwP/CyDgro2ySXAgww+zEeSNEa9lEJV3cHgTXCLnTfmKJKkIX5QjiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSpZSlIklqWgiSp1VspJDk1ye1J/mczvy3JrUkOJLkmyWl9ZZOkadXnlsIvA58Ymn8r8LaqeirwReCSXlJJ0hTrpRSSbAJ+FLismQ9wLnBds8pe4II+sknSNOtrS+F3gdcBf9fMPxl4rKoON/PzwMal7phkd5K5JHMLCwudB5WkaTL2UkjyY8ChqrrtWO5fVXuqaraqZmdmZtY4nSRNt3U9POdzgRcmOR94HPBE4O3A+iTrmq2FTcDBHrJJ0lQb+5ZCVb2hqjZV1VbgIuAjVfWTwE3AS5rVdgHXjzubJE27SXqfwq8Br0lygMExhst7ziNJU6eP3UetqroZuLmZvh94Vp95JGnaTdKWgiSpZ5aCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWpaCJKllKUiSWmMvhSSbk9yU5L4k9yb55Wb8SUluSPLp5usZ484mSdOujy2Fw8Brq+ps4NnApUnOBl4P3FhV24Ebm3lJ0hiNvRSq6pGq+ngz/b+BTwAbgZ3A3ma1vcAF484mSdOu12MKSbYCzwRuBc6sqkeaRZ8DzlzmPruTzCWZW1hYGE9QSZoSvZVCkm8F/gfwK1X15eFlVVVALXW/qtpTVbNVNTszMzOGpJI0PXophSTfxKAQ3lNV72uGH01yVrP8LOBQH9kkaZr1cfZRgMuBT1TV7wwt2gfsaqZ3AdePO5skTbs+thSeC/w0cG6SO5rb+cBbgB9K8mngBc28JJ3wNm7eQpIVb5Ng3bifsKr+DFjuuz9vnFkkaRwenn+Il/7+X6y43jW/8JwxpDk639EsSWpZCpKklqUgSWpZCpKklqUgScdg1DOKJuWsolGN/ewjSToZjHpGEUzGWUWjcktBktSyFCRJLUtBktSyFCRJLUtBktSyFCRJLUtBkoacSFc07YLvU5A0UTZu3sLD8w+NtO4/3rSZgw99dk2f/0S6omkXLAVJE+VkfVPYicLdR9IYjbprYuPmLX1H1ZRyS0Eao2nfNaHJ55aCJKllKeik4G4ZaW24+2gNreasiVO/6XS+/rW/XXG9Ls6uWI1Rv6fV5OziMbvYLdNFzi6cbDnVL0thDa32rIkTYd9yF39sT5T96uZcWydKzmk3cbuPkuxI8skkB5K8vu88GtEp607KDxwZSRffe8+P6W626TVRWwpJTgXeCfwQMA/8ZZJ9VXXfWj9X32+Q6VMnm/F/d3h6zy3v4nvv+zFf+byRCmfU3aCdaYpuJb3nPIFMVCkAzwIOVNX9AEmuBnYCa14K0/wGmWn+3jWiEQtk1N2gR9ZdcydKzhNIqqrvDK0kLwF2VNXLm/mfBr6/ql41tM5uYHcz+zTgk8fwVBuAzx9n3K5MajZzrY65Vm9Ss52MuZ5SVTNLLZi0LYUVVdUeYM/xPEaSuaqaXaNIa2pSs5lrdcy1epOabdpyTdqB5oPA5qH5Tc2YJGkMJq0U/hLYnmRbktOAi4B9PWeSpKkxUbuPqupwklcBfwycClxRVfd28FTHtfupY5OazVyrY67Vm9RsU5Vrog40S5L6NWm7jyRJPbIUJEmtk7oUVrpkRpLXJLkvyV1JbkzylAnJ9Yokdye5I8mfJTl7EnINrfcTSSrJ2E7TG+E1e1mSheY1uyPJyychV7POhc3v2b1J/vsk5ErytqHX6lNJHpuQXFuS3JTk9ubf5fnjyDVitqc0fyfuSnJzkk1jyHRFkkNJ7llmeZK8o8l8V5JzjvtJq+qkvDE4UP3XwHcApwF3AmcvWucHgcc3068ErpmQXE8cmn4h8OFJyNWs9wTgFmA/MDtBP8uXAf95An/HtgO3A2c0898+CbkWrf9qBid19J6LwcHTVzbTZwMPTNDP8r3Armb6XOAPx5DrecA5wD3LLD8f+BAQ4NnArcf7nCfzlkJ7yYyq+ipw5JIZraq6qaq+0szuZ/C+iEnI9eWh2W8BxnE2wIq5Gv8BeCvw/8aQabXZxm2UXD8PvLOqvghQVYcmJNewi4GrJiRXAU9spr8NeHgMuUbNdjbwkWb6piWWr7mqugX4wlFW2Qm8uwb2A+uTnHU8z3kyl8JGYPiqb/PN2HIuYdC4XRspV5JLk/w18J+AX5qEXM2m6eaq+sAY8gwb9Wf5E80m9HVJNi+xvI9c3wV8V5I/T7I/yY4JyQUMdokA2/jGH7u+c/068FNJ5oEPMtiKGYdRst0JvLiZfhHwhCRPHkO2o1nt37kVncylMLIkPwXMAr/Vd5YjquqdVfWdwK8B/67vPElOAX4HeG3fWZbxfmBrVX0vcAOwt+c8R6xjsAvp+Qz+R/4HSdb3GWiRi4DrqurrfQdpXAxcWVWbGOwa+cPmd28S/BvgB5LcDvwAg6stTMrrtmYm5cXuwkiXzEjyAuCNwAurahzX1l3tpTyuBi7oMlBjpVxPAJ4B3JzkAQb7L/eN6WDziq9ZVf3N0M/vMuCfTUIuBv9z21dVX6uqzwCfYlASfec64iLGs+sIRst1CXAtQFV9FHgcgwu/9Z6tqh6uqhdX1TMZ/M2gqh4bQ7ajWftLA43jIE4fNwb/Q7ufwabxkQNH37NonWcyOLi0fcJybR+a/nFgbhJyLVr/ZsZ3oHmU1+ysoekXAfsnJNcOYG8zvYHBpv6T+87VrPd04AGaN7FOyOv1IeBlzfR3Mzim0Hm+EbNtAE5ppt8M/MaYXretLH+g+Uf5+weaP3bczzeOb6qvG4PNz081f/jf2Iz9BoOtAoA/AR4F7mhu+yYk19uBe5tMNx3tj/M4cy1ad2ylMOJr9h+b1+zO5jV7+oTkCoPdbvcBdwMXTUKuZv7XgbeM62c44ut1NvDnzc/xDuCHJyjbS4BPN+tcBpw+hkxXAY8AX2Ow1XkJ8ArgFUO/X+9sMt+9Fv8mvcyFJKl1Mh9TkCStkqUgSWpZCpKklqUgSWpZCpKklqUgSWpZCpKk1v8Hpv9CHZDrJq4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(init_msp[(msp_log==2) + (msp_log==3)])\n",
    "init_msp[(msp_log==2) + (msp_log==3)].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8 tensor(251)\n",
      "0.9 tensor(255)\n",
      "0.95 tensor(240)\n",
      "0.99 tensor(196)\n",
      "0.999 tensor(163)\n",
      "1.0 tensor(164)\n"
     ]
    }
   ],
   "source": [
    "# permutation starting with 0\n",
    "for threshold in [0.8, 0.9, 0.95, 0.99, 0.999, 1.0]:\n",
    "    print(threshold, (init_msp[(msp_log==2) + (msp_log==3)] > threshold).sum() + (init_msp[(msp_log==0)] < threshold).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.895 tensor(256)\n",
      "0.8951 tensor(256)\n",
      "0.8952 tensor(256)\n",
      "0.8953 tensor(256)\n",
      "0.8954 tensor(256)\n",
      "0.8955 tensor(256)\n",
      "0.8956 tensor(256)\n",
      "0.8957 tensor(256)\n",
      "0.8958 tensor(257)\n",
      "0.8959 tensor(257)\n",
      "0.896 tensor(257)\n",
      "0.8961 tensor(257)\n",
      "0.8962 tensor(256)\n",
      "0.8963 tensor(256)\n",
      "0.8964 tensor(256)\n",
      "0.8965 tensor(256)\n",
      "0.8966 tensor(256)\n",
      "0.8967 tensor(256)\n",
      "0.8968 tensor(256)\n",
      "0.8969 tensor(256)\n"
     ]
    }
   ],
   "source": [
    "# permutation starting with 0\n",
    "for threshold in range(8950, 8970):\n",
    "    threshold = threshold/10000\n",
    "    print(threshold, (init_msp[(msp_log==2) + (msp_log==3)] > threshold).sum() + (init_msp[(msp_log==0)] < threshold).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8 tensor(242)\n",
      "0.9 tensor(245)\n",
      "0.95 tensor(230)\n",
      "0.99 tensor(185)\n",
      "0.999 tensor(151)\n",
      "1.0 tensor(152)\n"
     ]
    }
   ],
   "source": [
    "for threshold in [0.8, 0.9, 0.95, 0.99, 0.999, 1.0]:\n",
    "    print(threshold, (init_msp[(msp_log==2) + (msp_log==3)] > threshold).sum() + (init_msp[(msp_log==0)] < threshold).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.895 tensor(246)\n",
      "0.8951 tensor(246)\n",
      "0.8952 tensor(246)\n",
      "0.8953 tensor(246)\n",
      "0.8954 tensor(246)\n",
      "0.8955 tensor(246)\n",
      "0.8956 tensor(246)\n",
      "0.8957 tensor(246)\n",
      "0.8958 tensor(247)\n",
      "0.8959 tensor(247)\n",
      "0.896 tensor(247)\n",
      "0.8961 tensor(247)\n",
      "0.8962 tensor(246)\n",
      "0.8963 tensor(246)\n",
      "0.8964 tensor(246)\n",
      "0.8965 tensor(246)\n",
      "0.8966 tensor(246)\n",
      "0.8967 tensor(246)\n",
      "0.8968 tensor(246)\n",
      "0.8969 tensor(246)\n"
     ]
    }
   ],
   "source": [
    "for threshold in range(8950, 8970):\n",
    "    threshold = threshold/10000\n",
    "    print(threshold, (init_msp[(msp_log==2) + (msp_log==3)] > threshold).sum() + (init_msp[(msp_log==0)] < threshold).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.6848)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQaUlEQVR4nO3da7BdZX3H8e9PYopVKiDHTCYQgyOijI5gjxTQsQjiIG2BthZlvEQnGsdWR6u1Yn3R6wudtl7acdRUrLGj3CyUeKmWpiDTKuhBULloQQoSDORIRa1OtcF/X+zF5JCc5OwkrLVP8nw/M3v2Ws/aa68/z5z89uLZaz07VYUkqR2PmHQBkqRhGfyS1BiDX5IaY/BLUmMMfklqzJJJFzCOww47rFatWjXpMiRpn3Ldddd9r6qmtm/fJ4J/1apVzMzMTLoMSdqnJLlzvnaHeiSpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mN6S34kxyd5IY5jx8meVOSQ5NckeTW7vmQvmqQJO2ot+Cvqm9V1bFVdSzwy8BPgMuA84CNVXUUsLFbl6RFZcURK0ky0ceKI1b28t821JQNpwLfrqo7k5wFnNy1rweuAt42UB2SNJbvbrqLF3/oixOt4aLXntTL+w41xv8S4IJueVlVbe6W7wGWDVSDJIkBgj/JUuBM4JLtt9XoB3/n/dHfJGuTzCSZmZ2d7blKSWrHEGf8LwS+WlX3duv3JlkO0D1vmW+nqlpXVdNVNT01tcOsopKkPTRE8J/LtmEegA3A6m55NXD5ADVIkjq9Bn+SRwOnAZfOaX4ncFqSW4Hnd+uSpIH0elVPVf0YeNx2bfcxuspHkjQB3rkrSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1Jhegz/JwUk+meSbSW5JcmKSQ5NckeTW7vmQPmuQJD1U32f87wM+V1VPAZ4B3AKcB2ysqqOAjd26JGkgvQV/kscCzwXOB6iqn1XV/cBZwPruZeuBs/uqQZK0oz7P+I8EZoG/T3J9kg8neTSwrKo2d6+5B1g2385J1iaZSTIzOzvbY5mS1JY+g38J8EzgA1V1HPBjthvWqaoCar6dq2pdVU1X1fTU1FSPZUpSW/oM/k3Apqq6tlv/JKMPgnuTLAfonrf0WIMkaTu9BX9V3QPcleTorulU4GZgA7C6a1sNXN5XDZKkHS3p+f3fAHw8yVLgduBVjD5sLk6yBrgTOKfnGiRJc/Qa/FV1AzA9z6ZT+zyuJGnnvHNXkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mN6fXH1pPcAfwIeADYWlXTSQ4FLgJWAXcA51TV9/usQ5K0zRBn/M+rqmOrarpbPw/YWFVHARu7dUnSQCYx1HMWsL5bXg+cPYEaJKlZfQd/Af+S5Loka7u2ZVW1uVu+B1g2345J1iaZSTIzOzvbc5mS1I5ex/iB51TV3UkeD1yR5JtzN1ZVJan5dqyqdcA6gOnp6XlfI0nafb2e8VfV3d3zFuAy4Hjg3iTLAbrnLX3WIEl6qN6CP8mjkxz04DLwAuBGYAOwunvZauDyvmqQJO2oz6GeZcBlSR48zieq6nNJvgJcnGQNcCdwTo81SJK201vwV9XtwDPmab8POLWv40qSds07dyWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGjBX8SZ49TpskafEb94z/b8ds20GSA5Jcn+TT3fqRSa5NcluSi5IsHbdYSdLeW7KrjUlOBE4CppK8ec6mXwIOGPMYbwRu6fYBeBfwnqq6MMkHgTXAB3araknSHlvojH8p8BhGHxAHzXn8EHjRQm+e5HDg14APd+sBTgE+2b1kPXD2HtQtSdpDuzzjr6ovAF9I8tGqunMP3v+9wB8y+rAAeBxwf1Vt7dY3ASvm2zHJWmAtwMqVK/fg0JKk+ewy+Of4hSTrgFVz96mqU3a2Q5JfB7ZU1XVJTt7dwqpqHbAOYHp6unZ3f0nS/MYN/kuADzIasnlgzH2eDZyZ5AzgQEZj/O8DDk6ypDvrPxy4e/dKliTtjXGDf2tV7dYXsFX1duDtAN0Z/x9U1UuTXMLo+4ELgdXA5bvzvpKkvTPu5ZyfSvK7SZYnOfTBxx4e823Am5PcxmjM//w9fB9J0h4Y94x/dff81jltBTxxnJ2r6irgqm75duD4MY8rSXqYjRX8VXVk34VIkoYxVvAnecV87VX1sYe3HElS38Yd6nnWnOUDgVOBrwIGvyTtY8Yd6nnD3PUkBzO6KkeStI/Z02mZfww47i9J+6Bxx/g/xegqHhhNzvZU4OK+ipIk9WfcMf6/mrO8Fbizqjb1UI8kqWdjDfV0k7V9k9Fka4cAP+uzKElSf8b9Ba5zgC8DvwOcA1ybZMFpmSVJi8+4Qz3vAJ5VVVsAkkwB/8q2efUlSfuIca/qecSDod+5bzf2lSQtIuOe8X8uyeeBC7r1FwOf7ackSVKfFvrN3ScBy6rqrUl+C3hOt+lLwMf7Lk6S9PBb6Iz/vXRz6lfVpcClAEme3m37jR5rkyT1YKFx+mVV9Y3tG7u2Vb1UJEnq1ULBf/Autj3qYaxDkjSQhYJ/Jslrtm9M8mrgun5KkiT1aaEx/jcBlyV5KduCfhpYCvxmj3VJknqyy+CvqnuBk5I8D3ha1/yZqvq33iuTJPVi3Pn4rwSu7LkWSdIAerv7NsmBSb6c5GtJbkryp137kUmuTXJbkouSLO2rBknSjvqcduGnwClV9QzgWOD0JCcA7wLeU1VPAr4PrOmxBknSdnoL/hr5n271kd2jgFPYNrnbeuDsvmqQJO2o14nWkhyQ5AZgC3AF8G3g/qra2r1kE7BiJ/uuTTKTZGZ2drbPMiWpKb0Gf1U9UFXHAocDxwNP2Y1911XVdFVNT01N9VWiJDVnkKmVq+p+RlcFnQgcnOTBq4kOB+4eogZJ0kifV/VMJTm4W34UcBpwC6MPgAd/vWs1cHlfNUiSdjTufPx7YjmwPskBjD5gLq6qTye5GbgwyV8A1wPn91iDJGk7vQV/VX0dOG6e9tsZjfdLkibAn0+UpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1/NWnHESpJM/LHiiJWT7go1ps8fW5cWte9uuosXf+iLky6Di1570qRLUGN6O+NPckSSK5PcnOSmJG/s2g9NckWSW7vnQ/qqQZK0oz6HerYCb6mqY4ATgN9LcgxwHrCxqo4CNnbrkqSB9Bb8VbW5qr7aLf8IuAVYAZwFrO9eth44u68aJEk7GuTL3SSrgOOAa4FlVbW523QPsGwn+6xNMpNkZnZ2dogyJakJvQd/kscA/wi8qap+OHdbVRVQ8+1XVeuqarqqpqempvouU5Ka0WvwJ3kko9D/eFVd2jXfm2R5t305sKXPGiRJD9XnVT0Bzgduqap3z9m0AVjdLa8GLu+rBmmf8IglE7+XYDHdT7BY7q/Yn/V5Hf+zgZcD30hyQ9f2R8A7gYuTrAHuBM7psQZp8fv5Vu8nmMP7K/rXW/BX1b8DO/vYPLWv40qSds0pGySpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEG/0AWy/wji2E+lsXSF1Kr/M3dgTj/yDb2hTRZnvFLUmM845c00k0Prf2fwS9pxOmhm+FQjyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWpMb8Gf5CNJtiS5cU7boUmuSHJr93xIX8eXJM2vzzP+jwKnb9d2HrCxqo4CNnbrGlJ3k45z5Ejt6u0Grqq6Osmq7ZrPAk7ultcDVwFv66sGzWMR3KTjDTrSZA09xr+sqjZ3y/cAywY+viQ1b2Jf7lZVAbWz7UnWJplJMjM7O7vHx3EKYEl6qKHn6rk3yfKq2pxkObBlZy+sqnXAOoDp6emdfkAsxCmAJemhhj7j3wCs7pZXA5cPfHxJal6fl3NeAHwJODrJpiRrgHcCpyW5FXh+ty5JGlCfV/Wcu5NNp/Z1TEnSwrxzV5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxEwn+JKcn+VaS25KcN4kaJKlVgwd/kgOA9wMvBI4Bzk1yzNB1SFKrJnHGfzxwW1XdXlU/Ay4EzppAHZLUpFTVsAdMXgScXlWv7tZfDvxKVb1+u9etBdZ2q0cD3xq00OEcBnxv0kUsAvbDiP2wjX0xsjf98ISqmtq+ccne1dOfqloHrJt0HX1LMlNV05OuY9LshxH7YRv7YqSPfpjEUM/dwBFz1g/v2iRJA5hE8H8FOCrJkUmWAi8BNkygDklq0uBDPVW1Ncnrgc8DBwAfqaqbhq5jEdnvh7PGZD+M2A/b2BcjD3s/DP7lriRpsrxzV5IaY/BLUmMM/oEsNE1FkjcnuTnJ15NsTPKESdTZt3Gn60jy20kqyX55Od84/ZDknO5v4qYknxi6xiGM8e9iZZIrk1zf/ds4YxJ19i3JR5JsSXLjTrYnyd90/fT1JM/cqwNWlY+eH4y+xP428ERgKfA14JjtXvM84Be75dcBF0267kn0Q/e6g4CrgWuA6UnXPaG/h6OA64FDuvXHT7ruCfXDOuB13fIxwB2Trrunvngu8Ezgxp1sPwP4ZyDACcC1e3M8z/iHseA0FVV1ZVX9pFu9htH9Dfubcafr+HPgXcD/DlncgMbph9cA76+q7wNU1ZaBaxzCOP1QwC91y48FvjtgfYOpqquB/97FS84CPlYj1wAHJ1m+p8cz+IexArhrzvqmrm1n1jD6dN/fLNgP3f/CHlFVnxmysIGN8/fwZODJSf4jyTVJTh+suuGM0w9/ArwsySbgs8Abhilt0dndDNmlRTtlQ6uSvAyYBn510rUMLckjgHcDr5xwKYvBEkbDPScz+r+/q5M8varun2RRE3Au8NGq+uskJwL/kORpVfXzSRe2L/OMfxhjTVOR5PnAO4Azq+qnA9U2pIX64SDgacBVSe5gNJa5YT/8gnecv4dNwIaq+r+q+i/gPxl9EOxPxumHNcDFAFX1JeBARpOWteZhnerG4B/GgtNUJDkO+BCj0N8fx3NhgX6oqh9U1WFVtaqqVjH6ruPMqpqZTLm9GWfakn9idLZPksMYDf3cPmCNQxinH74DnAqQ5KmMgn920CoXhw3AK7qre04AflBVm/f0zRzqGUDtZJqKJH8GzFTVBuAvgccAlyQB+E5VnTmxonswZj/s98bsh88DL0hyM/AA8Naqum9yVT/8xuyHtwB/l+T3GX3R+8rqLnPZnyS5gNEH/WHd9xl/DDwSoKo+yOj7jTOA24CfAK/aq+Pth30oSdoFh3okqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWrM/wPkdUtmFQG5+AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(init_msp[msp_log==1])\n",
    "init_msp[(msp_log==1)].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.8144)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAARaUlEQVR4nO3de7BdZX3G8e8DIeIFDcgxE0NicECUoSPaI+Xi2MrFQdoB2lKE8RKcaBxbrYpjpXU69vaHTm3Vdhw1FWvsKIIUSrxUy0SUaRX0cFG5aEHkEgjkiKBWp2rw1z/2opycnORsQtbeSd7vZ2bPXutde+31yzvJs1fevfa7UlVIktqx17gLkCSNlsEvSY0x+CWpMQa/JDXG4JekxiwYdwHDOPDAA2vFihXjLkOSdivXXHPND6pqYnb7bhH8K1asYGpqatxlSNJuJckdc7U71CNJjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CVpDkuXLSfJWB9Lly3v5c+2W0zZIEmjds+Gu3jZh7861houfN2xvbxvr2f8Sd6S5MYkNyS5IMm+SQ5OcnWSW5NcmGRhnzVIkrbUW/AnWQr8MTBZVUcAewNnAe8G3ltVhwAPAKv6qkGStLW+x/gXAI9PsgB4ArAROB64uNu+Fji95xokSTP0FvxVdTfwHuBOBoH/I+Aa4MGq2ty9bAOwdK79k6xOMpVkanp6uq8yJak5fQ717A+cBhwMPB14InDysPtX1ZqqmqyqyYmJre4jIEnaQX0O9ZwIfL+qpqvql8AlwHHAom7oB+Ag4O4ea5AkzdJn8N8JHJ3kCUkCnADcBFwBnNG9ZiVwWY81SJJm6XOM/2oGX+JeC3y7O9Ya4O3AuUluBZ4KnN9XDZKkrfX6A66qeifwzlnNtwFH9XlcSdK2OWWDJDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4Jakxfd5s/bAk1894/DjJm5MckOTyJLd0z/v3VYMkaWt93nrxu1V1ZFUdCfw68DPgUuA8YH1VHQqs79YlSSMyqqGeE4DvVdUdwGnA2q59LXD6iGqQJDG64D8LuKBbXlxVG7vle4HFc+2QZHWSqSRT09PTo6hRkprQe/AnWQicCnx69raqKqDm2q+q1lTVZFVNTkxM9FylJLVjFGf8LwWurar7uvX7kiwB6J43jaAGSVJnFMF/No8M8wCsA1Z2yyuBy0ZQgySp02vwJ3kicBJwyYzmdwEnJbkFOLFblySNyII+37yqfgo8dVbb/Qyu8pEkjYG/3JWkxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNabvO3AtSnJxku8kuTnJMUkOSHJ5klu65/37rEGStKW+z/jfD3yhqp4NPBe4GTgPWF9VhwLru3VJ0oj0FvxJngK8CDgfoKp+UVUPAqcBa7uXrQVO76sGSdLW+jzjPxiYBv45yXVJPtLdfH1xVW3sXnMvsHiunZOsTjKVZGp6errHMiWpLX0G/wLg+cAHq+p5wE+ZNaxTVQXUXDtX1ZqqmqyqyYmJiR7LlKS29Bn8G4ANVXV1t34xgw+C+5IsAeieN/VYgyRplt6Cv6ruBe5KcljXdAJwE7AOWNm1rQQu66sGSdLWFvT8/m8EPpFkIXAb8GoGHzYXJVkF3AGc2XMNkqQZeg3+qroemJxj0wl9HleStG3+cleSGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1Jheb8SS5HbgJ8BDwOaqmkxyAHAhsAK4HTizqh7osw5J0iNGccb/4qo6sqoevhPXecD6qjoUWN+tS5JGZBxDPacBa7vltcDpY6hBkprVd/AX8B9JrkmyumtbXFUbu+V7gcU91yBJmqHXMX7ghVV1d5KnAZcn+c7MjVVVSWquHbsPitUAy5cv77lMSWpHr2f8VXV397wJuBQ4CrgvyRKA7nnTNvZdU1WTVTU5MTHRZ5mS1JTegj/JE5Ps9/Ay8BLgBmAdsLJ72Urgsr5qkCRtrc+hnsXApUkePs4nq+oLSb4BXJRkFXAHcGaPNUiSZukt+KvqNuC5c7TfD5zQ13ElSds31FBPkuOGaZMk7fqGHeP/xyHbJEm7uO0O9SQ5BjgWmEhy7oxNTwb27rMwSVI/5hvjXwg8qXvdfjPafwyc0VdRkqT+bDf4q+orwFeSfKyq7hhRTZKkHg17Vc/jkqxhMKPm/+9TVcf3UZQkqT/DBv+ngQ8BH2EwxbIkaTc1bPBvrqoP9lqJJGkkhr2c8zNJ/jDJkiQHPPzotTJJUi+GPeN/eG6dt81oK+CZO7ccSVLfhgr+qjq470IkSaMxVPAnedVc7VX18Z1bjiSpb8MO9bxgxvK+DCZZuxYw+CVpNzPsUM8bZ64nWQR8qo+CJEn92tEbsfwUcNxfknZDw47xf4bBVTwwmJztOcBFfRUlSerPsGP875mxvBm4o6o29FCPJKlnQw31dJO1fYfBDJ37A78Y9gBJ9k5yXZLPdusHJ7k6ya1JLkyycEcKlyTtmGHvwHUm8HXgDxjcI/fqJMNOy/wm4OYZ6+8G3ltVhwAPAKuGL1eS9FgN++XuO4AXVNXKqnoVcBTw5/PtlOQg4LcZTO5GBndePx64uHvJWuD0R1mzJOkxGDb496qqTTPW7x9y3/cBfwL8qlt/KvBgVW3u1jcAS+faMcnqJFNJpqanp4csU5I0n2GD/wtJvpjknCTnAJ8DPr+9HZL8DrCpqq7ZkcKqak1VTVbV5MTExI68hSRpDvPdc/cQYHFVvS3J7wEv7DZ9DfjEPO99HHBqklMY/Nr3ycD7gUVJFnRn/QcBdz+WP4Ak6dGZ74z/fQzur0tVXVJV51bVucCl3bZtqqo/raqDqmoFcBbwpap6OXAFj9yvdyVw2Q5XL0l61OYL/sVV9e3ZjV3bih085tuBc5PcymDM//wdfB9J0g6Y7wdci7az7fHDHqSqvgx8uVu+jcFVQZKkMZjvjH8qyWtnNyZ5DbBDX9pKksZrvjP+NwOXJnk5jwT9JLAQ+N0e65Ik9WS7wV9V9wHHJnkxcETX/Lmq+lLvlUmSejHsfPxXMLgaR5K0m9vR+fglSbspg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWpMb8GfZN8kX0/yzSQ3JvnLrv3gJFcnuTXJhUkW9lWDJGlrfZ7x/xw4vqqeCxwJnJzkaODdwHur6hDgAWBVjzVIkmbpLfhr4H+61X26RwHHAxd37WuB0/uqQZK0tV7H+JPsneR6YBNwOfA94MGq2ty9ZAOwdBv7rk4ylWRqenq6zzIlqSm9Bn9VPVRVRwIHMbjB+rMfxb5rqmqyqiYnJib6KlGSmjOSq3qq6kEGd/A6BliU5OE7fx0E3D2KGiRJA31e1TORZFG3/HjgJOBmBh8AZ3QvWwlc1lcNkqStDXXP3R20BFibZG8GHzAXVdVnk9wEfCrJ3wDXAef3WIMkaZbegr+qvgU8b4722xiM90uSxsBf7kpSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGtPnrReXJbkiyU1Jbkzypq79gCSXJ7mle96/rxokSVvr84x/M/DWqjocOBr4oySHA+cB66vqUGB9ty5JGpHegr+qNlbVtd3yTxjcaH0pcBqwtnvZWuD0vmqQJG1tJGP8SVYwuP/u1cDiqtrYbboXWLyNfVYnmUoyNT09PYoyJakJvQd/kicB/wq8uap+PHNbVRVQc+1XVWuqarKqJicmJvouU5Ka0WvwJ9mHQeh/oqou6ZrvS7Kk274E2NRnDZKkLfV5VU+A84Gbq+rvZ2xaB6zsllcCl/VVgyRpawt6fO/jgFcC305yfdf2Z8C7gIuSrALuAM7ssQZJ0iy9BX9V/SeQbWw+oa/jSpK2z1/uSlJjDH5JaozBL0mNMfglqTF9XtUjSY/a0mXLuWfDXeMuY49m8Evapdyz4S5e9uGvjrsMLnzdseMuoTcO9UhSYwx+acyWLltOkrE/li5bPu6u0Ig41CONmUMbGjXP+CWpMQa/JDXG4Jekxhj8ktQYg1+SGuNVPZIG9lrA4P5J2tMZ/JIGfrXZy0ob0eetFz+aZFOSG2a0HZDk8iS3dM/793V8SdLc+hzj/xhw8qy284D1VXUosL5blySNUG/BX1VXAj+c1XwasLZbXguc3tfxJUlzG/VVPYuramO3fC+weMTHl6Tmje1yzqoqoLa1PcnqJFNJpqanp0dYmfrmpGTSeI36qp77kiypqo1JlgCbtvXCqloDrAGYnJzc5geEdj9OSiaN16jP+NcBK7vllcBlIz6+JDWvz8s5LwC+BhyWZEOSVcC7gJOS3AKc2K1Lkkaot6Geqjp7G5tO6OuYkqT5OVePJDXG4JekxjhXj9rlpGRqlMGvdjkpmRrlUI8kNcbgl6TGGPyS1Jg9fox/6bLl3LPhrnGXwd77PI6HfvnzcZfB0w9axt133TnuMiSN0R4f/LvSvDC7Sh2S2uZQjyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxYwn+JCcn+W6SW5OcN44aJKlVIw/+JHsDHwBeChwOnJ3k8FHXIUmtGscZ/1HArVV1W1X9AvgUcNoY6pCkJqWqRnvA5Azg5Kp6Tbf+SuA3quoNs163GljdrR4GfHekhe5cBwI/GHcRuxD7Y0v2x5bsjy09lv54RlVNzG7cZSdpq6o1wJpx17EzJJmqqslx17GrsD+2ZH9syf7YUh/9MY6hnruBZTPWD+raJEkjMI7g/wZwaJKDkywEzgLWjaEOSWrSyId6qmpzkjcAXwT2Bj5aVTeOuo4R2yOGrHYi+2NL9seW7I8t7fT+GPmXu5Kk8fKXu5LUGINfkhpj8O8k801DkeTcJDcl+VaS9UmeMY46R2XYaTmS/H6SSrJHX743TH8kObP7O3Jjkk+OusZRGuLfy/IkVyS5rvs3c8o46hyVJB9NsinJDdvYniT/0PXXt5I8/zEdsKp8PMYHgy+pvwc8E1gIfBM4fNZrXgw8oVt+PXDhuOseZ390r9sPuBK4Cpgcd91j/vtxKHAdsH+3/rRx1z3m/lgDvL5bPhy4fdx199wnLwKeD9ywje2nAP8OBDgauPqxHM8z/p1j3mkoquqKqvpZt3oVg98v7KmGnZbjr4F3A/87yuLGYJj+eC3wgap6AKCqNo24xlEapj8KeHK3/BTgnhHWN3JVdSXww+285DTg4zVwFbAoyZIdPZ7Bv3MsBe6asb6ha9uWVQw+vfdU8/ZH91/VZVX1uVEWNibD/P14FvCsJP+V5KokJ4+sutEbpj/+AnhFkg3A54E3jqa0XdajzZjt2mWnbNhTJXkFMAn85rhrGZckewF/D5wz5lJ2JQsYDPf8FoP/DV6Z5Neq6sFxFjVGZwMfq6q/S3IM8C9JjqiqX427sD2BZ/w7x1DTUCQ5EXgHcGpV/XxEtY3DfP2xH3AE8OUktzMYs1y3B3/BO8zfjw3Auqr6ZVV9H/hvBh8Ee6Jh+mMVcBFAVX0N2JfBZGWt2qlT3Rj8O8e801AkeR7wYQahvyeP38I8/VFVP6qqA6tqRVWtYPCdx6lVNTWecns3zDQl/8bgbJ8kBzIY+rlthDWO0jD9cSdwAkCS5zAI/umRVrlrWQe8qru652jgR1W1cUffzKGenaC2MQ1Fkr8CpqpqHfC3wJOATycBuLOqTh1b0T0asj+aMWR/fBF4SZKbgIeAt1XV/eOruj9D9sdbgX9K8hYGX/SeU93lLXuiJBcw+OA/sPte453APgBV9SEG33OcAtwK/Ax49WM63h7cl5KkOTjUI0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSY/4PBEWxNBlxCp8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(init_msp[msp_log==0])\n",
    "init_msp[(msp_log==0)].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3969) tensor(0.1924)\n"
     ]
    }
   ],
   "source": [
    "print(((msp_log==2).sum()+(msp_log==3).sum())/len(msp_log), ((entropy_log==2).sum()+ (entropy_log==3).sum())/len(msp_log))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(MSP_permutation, \"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/MSP_full_permutation.pt\")\n",
    "torch.save(MSP_logits, \"/data/zhihao/TU/Watchog/outputs/gt-semtab22-dbpedia-all0/MSP_full_logits.pt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3122311/130419269.py:42: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  max_msp = F.softmax(logits_init).detach().cpu().max().item()\n",
      "/tmp/ipykernel_3122311/130419269.py:72: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  temp_msp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.7272, ts_macro_f1=0.4835\n",
      "ts_micro_f1=0.7269, ts_macro_f1=0.4835\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "# Cheat brute force perumate\n",
    "\n",
    "import itertools\n",
    "import torch.nn.functional as F\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "MSP_permutation = defaultdict(list)\n",
    "MSP_logits = defaultdict(list)\n",
    "MSP_corrected = 0\n",
    "Entropy_permutation = defaultdict(list)\n",
    "Entropy_logits = defaultdict(list)\n",
    "Entropy_corrected = 0\n",
    "\n",
    "max_col_length = 3\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "        total_mistakes += 1\n",
    "        col_idx_set = target_col_mask.unique().tolist()\n",
    "        successs = False\n",
    "        init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "        init_logits[batch_idx].append(logits_init.detach().cpu())        \n",
    "        max_msp = F.softmax(logits_init).detach().cpu().max().item()\n",
    "        msp_logits = logits_init.clone().detach().cpu()\n",
    "        msp_perm = get_permutation(target_col_mask)[0]\n",
    "        assert -1 not in col_idx_set\n",
    "        for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "            for subset in itertools.combinations(col_idx_set, r):\n",
    "                if 0 not in subset:\n",
    "                    continue\n",
    "                for x in itertools.permutations(subset):\n",
    "                    # if 0 not in x:\n",
    "                    #     continue\n",
    "                    # if x[0] != 0:\n",
    "                    #     continue\n",
    "                    new_batch_data = []\n",
    "                    for col_i in x:\n",
    "                        if col_i == 0:\n",
    "                            if len(new_batch_data) == 0:\n",
    "                                cls_indexes_value = 0\n",
    "                            else:\n",
    "                                cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                        new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                    new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                    cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                    logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                    if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                        logits = logits_temp.clone()\n",
    "                        successs = True\n",
    "                        corrected_permutation[batch_idx].append(x)\n",
    "                        corrected_logits[batch_idx].append(logits_temp.clone().detach().cpu())\n",
    "                    \n",
    "                    temp_msp = F.softmax(logits_temp).max().item()\n",
    "                    if max_msp < temp_msp:\n",
    "                        max_msp = temp_msp\n",
    "                        msp_logits = logits_temp.clone().detach().cpu()\n",
    "                        msp_perm = x\n",
    "        MSP_logits[batch_idx].append(msp_logits)\n",
    "        MSP_permutation[batch_idx].append(msp_perm)\n",
    "        if msp_logits.argmax().item() == batch[\"label\"].item():\n",
    "            MSP_corrected += 1\n",
    "            # print(f\"MSP correct {batch_idx}\")\n",
    "        if successs:\n",
    "            corrected += 1\n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 4, 6, 8, 12, 14, 15, 16, 18, 19]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(init_permutation.keys())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_sublist(A, B):\n",
    "    it = iter(B)\n",
    "    return all(x in it for x in A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 2, 3, 4, 5, 6, 0, 7]]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_permutation[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 8, 12, 15, 16, 18, 19, 23, 27, 33]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(corrected_permutation.keys())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 0)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " corrected_permutation[4][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_sublist = 0\n",
    "for idx in corrected_permutation:\n",
    "    for sublist in corrected_permutation[idx]:\n",
    "        if is_sublist(sublist, init_permutation[idx][0]):\n",
    "            num_sublist += 1\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "251"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corrected_permutation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "194"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_sublist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7729083665338645"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_sublist/len(corrected_permutation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Max Column Length: 2****************************\n",
      "ts_micro_f1=0.6783, ts_macro_f1=0.3947\n",
      "ts_micro_f1=0.6780, ts_macro_f1=0.3947\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "Time: 53.744505167007446\n",
      "*********************Max Column Length: 3****************************\n",
      "ts_micro_f1=0.7475, ts_macro_f1=0.4875\n",
      "ts_micro_f1=0.7472, ts_macro_f1=0.4875\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "Time: 315.4276714324951\n",
      "*********************Max Column Length: 4****************************\n",
      "ts_micro_f1=0.7880, ts_macro_f1=0.5643\n",
      "ts_micro_f1=0.7878, ts_macro_f1=0.5643\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "Time: 1577.7969288825989\n"
     ]
    }
   ],
   "source": [
    "# Cheat brute force perumate\n",
    "\n",
    "import itertools\n",
    "import torch.nn.functional as F\n",
    "import time\n",
    "for max_col_length in [2,3,4]:\n",
    "    start = time.time()\n",
    "    print(f\"*********************Max Column Length: {max_col_length}****************************\")\n",
    "    model.load_state_dict(best_state_dict, strict=False)\n",
    "    model.eval()\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    threshold = 0.8\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "    init_permutation = defaultdict(list)\n",
    "    corrected_permutation = defaultdict(list)\n",
    "    init_logits = defaultdict(list)\n",
    "    corrected_logits = defaultdict(list)\n",
    "    MSP_permutation = defaultdict(list)\n",
    "    MSP_logits = defaultdict(list)\n",
    "    MSP_corrected = 0\n",
    "    Entropy_permutation = defaultdict(list)\n",
    "    Entropy_logits = defaultdict(list)\n",
    "    Entropy_corrected = 0\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        if 1 in target_col_mask and logits.argmax().item() != batch[\"label\"].item():\n",
    "            total_mistakes += 1\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            successs = False\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    if 0 not in subset:\n",
    "                        continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                            logits = logits_temp.clone()\n",
    "                            successs = True\n",
    "                            corrected_permutation[batch_idx].append(x)\n",
    "                            corrected_logits[batch_idx].append(logits_temp.clone().detach().cpu())\n",
    "                            break\n",
    "                if successs:\n",
    "                    break          \n",
    "            if successs:\n",
    "                corrected += 1\n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    print(f\"Time: {time.time()-start}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "333"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "MSP_corrected_ids = []\n",
    "MSP_failed_ids = []\n",
    "for k, v in MSP_logits.items():\n",
    "    msp_logits = v[0]\n",
    "    if msp_logits.argmax().item() == batch[\"label\"].item():\n",
    "        MSP_corrected_ids.append(k)\n",
    "    else:\n",
    "        MSP_failed_ids.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "Entropy_corrected_ids = []\n",
    "Entropy_failed_ids = []\n",
    "for k, v in Entropy_logits.items():\n",
    "    Entropy_logits = v[0]\n",
    "    if Entropy_logits.argmax().item() == batch[\"label\"].item():\n",
    "        Entropy_corrected_ids.append(k)\n",
    "    else:\n",
    "        Entropy_failed_ids.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "        MSP_logits[batch_idx].append(msp_logits)\n",
    "        MSP_permutation[batch_idx].append(msp_perm)\n",
    "        if msp_logits.argmax().item() == batch[\"label\"].item():\n",
    "            MSP_corrected += 1\n",
    "            # print(f\"MSP correct {batch_idx}\")\n",
    "        Entropy_logits[batch_idx].append(entropy_logits)\n",
    "        Entropy_permutation[batch_idx].append(entropy_perm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Entropy_corrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MSP_corrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "314"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_mistakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0,) tensor(0.2383, device='cuda:1', grad_fn=<MaxBackward1>) tensor(37, device='cuda:1')\n",
      "(1,) tensor(0.5257, device='cuda:1', grad_fn=<MaxBackward1>) tensor(10, device='cuda:1')\n",
      "(0, 1) tensor(0.2742, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(1, 0) tensor(0.4955, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3294044442.py:50: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# Cheat brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 18:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())        \n",
    "assert -1 not in col_idx_set\n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 0, 2] 1\n",
      "********************************************************\n",
      "(0,) tensor(0.2444, device='cuda:1', grad_fn=<MaxBackward1>) tensor(67, device='cuda:1')\n",
      "(1,) tensor(0.5232, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(2,) tensor(0.7384, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(0, 1) tensor(0.2680, device='cuda:1', grad_fn=<MaxBackward1>) tensor(56, device='cuda:1')\n",
      "(1, 0) tensor(0.8958, device='cuda:1', grad_fn=<MaxBackward1>) tensor(67, device='cuda:1')\n",
      "(0, 2) tensor(0.5103, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0) tensor(0.2122, device='cuda:1', grad_fn=<MaxBackward1>) tensor(67, device='cuda:1')\n",
      "(1, 2) tensor(0.9971, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(2, 1) tensor(0.9902, device='cuda:1', grad_fn=<MaxBackward1>) tensor(10, device='cuda:1')\n",
      "(0, 1, 2) tensor(0.2955, device='cuda:1', grad_fn=<MaxBackward1>) tensor(37, device='cuda:1')\n",
      "(0, 2, 1) tensor(0.5985, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0, 2) tensor(0.4347, device='cuda:1', grad_fn=<MaxBackward1>) tensor(59, device='cuda:1')\n",
      "(1, 2, 0) tensor(0.9305, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0, 1) tensor(0.4064, device='cuda:1', grad_fn=<MaxBackward1>) tensor(59, device='cuda:1')\n",
      "(2, 1, 0) tensor(0.9894, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "********************************************************\n",
      "(2, 1, 0) 0.9894070029258728 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/952217777.py:54: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_3359783/952217777.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_3359783/952217777.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 154:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "\n",
    "print(get_permutation(target_col_mask), batch[\"label\"].item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, msp_temp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1] 0.9975814819335938 4\n",
      "********************************************************\n",
      "(0,) tensor(0.9927, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1,) tensor(0.7488, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 1) tensor(0.9976, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0) tensor(0.9895, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1')\n",
      "********************************************************\n",
      "(0, 1) 0.9975814819335938 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/687120700.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_3359783/687120700.py:54: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_3359783/687120700.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_3359783/687120700.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 247:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(get_permutation(target_col_mask), init_msp, batch[\"label\"].item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 0, 2] 0.4322320818901062 24\n",
      "********************************************************\n",
      "(0,) tensor(0.3846, device='cuda:1', grad_fn=<MaxBackward1>) tensor(13, device='cuda:1')\n",
      "(1,) tensor(0.6272, device='cuda:1', grad_fn=<MaxBackward1>) tensor(26, device='cuda:1')\n",
      "(2,) tensor(0.8308, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(0, 1) tensor(0.4106, device='cuda:1', grad_fn=<MaxBackward1>) tensor(34, device='cuda:1')\n",
      "(1, 0) tensor(0.4616, device='cuda:1', grad_fn=<MaxBackward1>) tensor(34, device='cuda:1')\n",
      "(0, 2) tensor(0.6058, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(2, 0) tensor(0.8639, device='cuda:1', grad_fn=<MaxBackward1>) tensor(24, device='cuda:1')\n",
      "(1, 2) tensor(0.8671, device='cuda:1', grad_fn=<MaxBackward1>) tensor(7, device='cuda:1')\n",
      "(2, 1) tensor(0.9611, device='cuda:1', grad_fn=<MaxBackward1>) tensor(26, device='cuda:1')\n",
      "(0, 1, 2) tensor(0.9930, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(0, 2, 1) tensor(0.9970, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(1, 0, 2) tensor(0.4322, device='cuda:1', grad_fn=<MaxBackward1>) tensor(34, device='cuda:1')\n",
      "(1, 2, 0) tensor(0.8516, device='cuda:1', grad_fn=<MaxBackward1>) tensor(10, device='cuda:1')\n",
      "(2, 0, 1) tensor(0.4788, device='cuda:1', grad_fn=<MaxBackward1>) tensor(24, device='cuda:1')\n",
      "(2, 1, 0) tensor(0.6521, device='cuda:1', grad_fn=<MaxBackward1>) tensor(24, device='cuda:1')\n",
      "********************************************************\n",
      "(0, 2, 1) 0.9970195889472961 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/1050205004.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_3359783/1050205004.py:54: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_3359783/1050205004.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_3359783/1050205004.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 355:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(get_permutation(target_col_mask), init_msp, batch[\"label\"].item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 0] 0.9007524251937866 1\n",
      "********************************************************\n",
      "(0,) tensor(0.8422, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1,) tensor(0.8806, device='cuda:1', grad_fn=<MaxBackward1>) tensor(8, device='cuda:1')\n",
      "(0, 1) tensor(0.1787, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0) tensor(0.9008, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "********************************************************\n",
      "(1, 0) 0.9007524251937866 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3796749687.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_3359783/3796749687.py:54: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_3359783/3796749687.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_3359783/3796749687.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 392:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(get_permutation(target_col_mask), init_msp, batch[\"label\"].item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/413784755.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_3359783/413784755.py:54: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_3359783/413784755.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_3359783/413784755.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 0, 2, 3] 0.9957796335220337 4\n",
      "********************************************************\n",
      "(0,) tensor(0.9934, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1,) tensor(0.4030, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(2,) tensor(0.9896, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3,) tensor(0.9720, device='cuda:1', grad_fn=<MaxBackward1>) tensor(8, device='cuda:1')\n",
      "(0, 1) tensor(0.9948, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0) tensor(0.9617, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(0, 2) tensor(0.9964, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0) tensor(0.8613, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 3) tensor(0.8364, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 0) tensor(0.8831, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(1, 2) tensor(0.3134, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 1) tensor(0.5857, device='cuda:1', grad_fn=<MaxBackward1>) tensor(11, device='cuda:1')\n",
      "(1, 3) tensor(0.9951, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 1) tensor(0.7448, device='cuda:1', grad_fn=<MaxBackward1>) tensor(39, device='cuda:1')\n",
      "(2, 3) tensor(0.8681, device='cuda:1', grad_fn=<MaxBackward1>) tensor(8, device='cuda:1')\n",
      "(3, 2) tensor(0.7427, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(0, 1, 2) tensor(0.9975, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 2, 1) tensor(0.9970, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0, 2) tensor(0.9856, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 2, 0) tensor(0.7049, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(2, 0, 1) tensor(0.9741, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 1, 0) tensor(0.9785, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(0, 1, 3) tensor(0.9801, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 3, 1) tensor(0.7826, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0, 3) tensor(0.7970, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 3, 0) tensor(0.7732, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1')\n",
      "(3, 0, 1) tensor(0.3869, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1')\n",
      "(3, 1, 0) tensor(0.8394, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(0, 2, 3) tensor(0.9966, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 3, 2) tensor(0.9924, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0, 3) tensor(0.9941, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 3, 0) tensor(0.8248, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 0, 2) tensor(0.8973, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 2, 0) tensor(0.9966, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 2, 3) tensor(0.9962, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 3, 2) tensor(0.3363, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(2, 1, 3) tensor(0.9941, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 3, 1) tensor(0.9378, device='cuda:1', grad_fn=<MaxBackward1>) tensor(11, device='cuda:1')\n",
      "(3, 1, 2) tensor(0.2855, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(3, 2, 1) tensor(0.9814, device='cuda:1', grad_fn=<MaxBackward1>) tensor(11, device='cuda:1')\n",
      "(0, 1, 2, 3) tensor(0.9954, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 1, 3, 2) tensor(0.9968, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 2, 1, 3) tensor(0.9954, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 2, 3, 1) tensor(0.9955, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 3, 1, 2) tensor(0.9925, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(0, 3, 2, 1) tensor(0.9864, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0, 2, 3) tensor(0.9958, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 0, 3, 2) tensor(0.9931, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 2, 0, 3) tensor(0.7715, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(1, 2, 3, 0) tensor(0.5923, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(1, 3, 0, 2) tensor(0.5012, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1')\n",
      "(1, 3, 2, 0) tensor(0.3819, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0, 1, 3) tensor(0.9955, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 0, 3, 1) tensor(0.9945, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 1, 0, 3) tensor(0.4121, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 1, 3, 0) tensor(0.5059, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(2, 3, 0, 1) tensor(0.6854, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(2, 3, 1, 0) tensor(0.7568, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(3, 0, 1, 2) tensor(0.9186, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 0, 2, 1) tensor(0.9150, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 1, 0, 2) tensor(0.7509, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1')\n",
      "(3, 1, 2, 0) tensor(0.5618, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 2, 0, 1) tensor(0.9953, device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1')\n",
      "(3, 2, 1, 0) tensor(0.6157, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "********************************************************\n",
      "(0, 1, 2) 0.9975000023841858 1\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 433:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(get_permutation(target_col_mask), init_msp, batch[\"label\"].item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([1, 2, 4])"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permutation_logits_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 tensor(1., device='cuda:1', grad_fn=<MaxBackward1>) tensor(1, device='cuda:1') tensor(2.6728e-11, device='cuda:1', grad_fn=<NegBackward0>)\n",
      "2 tensor(0.9007, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1') tensor(0.7803, device='cuda:1', grad_fn=<NegBackward0>)\n",
      "4 tensor(0.0985, device='cuda:1', grad_fn=<MaxBackward1>) tensor(4, device='cuda:1') tensor(4.4671, device='cuda:1', grad_fn=<NegBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/2545117251.py:5: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  temp += F.softmax(res)\n",
      "/tmp/ipykernel_3359783/2545117251.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  temp = F.softmax(temp)\n"
     ]
    }
   ],
   "source": [
    "res_all = {}\n",
    "for k, v in permutation_logits_dict.items():\n",
    "    temp = 0 \n",
    "    for res in v:\n",
    "        temp += F.softmax(res)\n",
    "    temp = F.softmax(temp)\n",
    "    print(k, temp.max(), temp.argmax(), compute_entropy(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_choices = []\n",
    "num_cols = []\n",
    "for k, v in corrected_permutation.items():\n",
    "    num_choices.append(len(v))\n",
    "    num_cols.append(len(init_permutation[k][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([14, 18, 81, 107, 125, 154, 172, 247, 279, 283, 297, 302, 345, 355, 374, 392, 400, 433, 434, 437, 470, 473, 482, 522, 545, 561, 570, 587, 597, 624, 627, 630, 652, 694, 698, 711, 723, 727, 729, 730, 747, 827, 833, 835, 880, 886, 894, 914, 949, 950, 952, 975, 978, 980, 981, 993, 1014, 1081, 1083, 0, 8])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected_permutation.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0, 1]]\n",
      "tensor(0.2742) tensor(3)\n",
      "************************Corrected*************************\n",
      "(1, 0) tensor(0.4955) tensor(0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/960455000.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/960455000.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "target = 18\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([14, 18, 81, 107, 125, 154, 172, 247, 279, 283, 297, 302, 345, 355, 374, 392, 400, 433, 434, 437, 470, 473, 482, 522, 545, 561, 570, 587, 597, 624, 627, 630, 652, 694, 698, 711, 723, 727, 729, 730, 747, 827, 833, 835, 880, 886, 894, 914, 949, 950, 952, 975, 978, 980, 981, 993, 1014, 1081, 1083])"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected_permutation.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 0, 2]]\n",
      "tensor(0.4347) tensor(59)\n",
      "************************Corrected*************************\n",
      "(0, 2) tensor(0.5103) tensor(1)\n",
      "(0, 2, 1) tensor(0.5985) tensor(1)\n",
      "(1, 2, 0) tensor(0.9305) tensor(1)\n",
      "(2, 1, 0) tensor(0.9894) tensor(1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/1376675092.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/1376675092.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "# TODO: 0 only results\n",
    "target = 154\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0, 1]]\n",
      "tensor(0.9976) tensor(1)\n",
      "************************Corrected*************************\n",
      "(1, 0) tensor(0.9895) tensor(4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3230554120.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/3230554120.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "# TODO: 0 only results\n",
    "target = 247\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 0]]\n",
      "tensor(0.5605) tensor(57)\n",
      "************************Corrected*************************\n",
      "(0,) tensor(0.9603) tensor(33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/2999824171.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/2999824171.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "# TODO: (0,1)\n",
    "target = 302\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 0, 2]]\n",
      "tensor(0.4322) tensor(34)\n",
      "************************Corrected*************************\n",
      "(2, 0) tensor(0.8639) tensor(24)\n",
      "(2, 0, 1) tensor(0.4788) tensor(24)\n",
      "(2, 1, 0) tensor(0.6521) tensor(24)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/2962954506.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/2962954506.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "# TODO: ????\n",
    "target = 355\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 0]]\n",
      "tensor(0.9008) tensor(2)\n",
      "************************Corrected*************************\n",
      "(0,) tensor(0.8422) tensor(1)\n",
      "(0, 1) tensor(0.1787) tensor(1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/917667902.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/917667902.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "# TODO: 1\n",
    "target = 392\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0, 1]]\n",
      "tensor(0.5274) tensor(24)\n",
      "************************Corrected*************************\n",
      "(0,) tensor(0.9763) tensor(2)\n",
      "(1, 0) tensor(0.9977) tensor(2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/1852677904.py:3: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
      "/tmp/ipykernel_3359783/1852677904.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())\n"
     ]
    }
   ],
   "source": [
    "target = 400\n",
    "print(init_permutation[target])\n",
    "print(F.softmax(init_logits[target][0]).max(), F.softmax(init_logits[target][0]).argmax())\n",
    "print(\"************************Corrected*************************\")\n",
    "for i, v in enumerate(corrected_permutation[target]):\n",
    "    print(v, F.softmax(corrected_logits[target][i]).max(), F.softmax(corrected_logits[target][i]).argmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# MAH \n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "idx_list = []\n",
    "uncertain_init_mask = []\n",
    "uncertain_target_mask = []\n",
    "correct_init_mask = []\n",
    "correct_target_mask = []\n",
    "correct_permutation_mask = []\n",
    "correct_permutation_confident_mask = []\n",
    "correct_mah_mask = []\n",
    "init_target_mask = []\n",
    "ood_score_init_list = []\n",
    "ood_score_target_list = []\n",
    "ood_score_final_list = []\n",
    "\n",
    "mah_predicts = []\n",
    "max_col_length = 3\n",
    "\n",
    "\n",
    "for threshold in [0.8]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    preds_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        if 1 not in target_col_mask:\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "            labels_test.append(batch[\"label\"].cpu())\n",
    "            preds_test.append(logits.argmax().cpu())\n",
    "            mah_predicts.append(logits.argmax().cpu())\n",
    "        else:\n",
    "            labels_test.append(batch[\"label\"].cpu())\n",
    "            idx_list.append(batch_idx)\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            logits, embs = model(batch[\"data\"].T, cls_indexes=cls_indexes, get_enc=True)\n",
    "            num_cols.append(batch[\"target_col_mask\"].max().item())         \n",
    "            ood_score_init = mah_dist(embs.detach().cpu().reshape(1,-1).numpy())\n",
    "            ood_score_init_list.append(ood_score_init)\n",
    "            predict_init = logits.argmax().item()\n",
    "            correct_init_mask.append(predict_init == batch[\"label\"].item())\n",
    "            \n",
    "            cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "            logits_temp, embs_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes, get_enc=True)  \n",
    "            ood_score_target = mah_dist(embs_temp.detach().cpu().reshape(1,-1).numpy())\n",
    "            ood_score_target_list.append(ood_score_target)\n",
    "            predict_target = logits_temp.argmax().item()\n",
    "            correct_target_mask.append(predict_target == batch[\"label\"].item())\n",
    "            \n",
    "\n",
    "            init_target_mask.append(predict_target == predict_init)\n",
    "            \n",
    "            correct_permutation = False\n",
    "            correct_permutation_ood_score = float(\"inf\")\n",
    "            correct_permutation_confident = False\n",
    "            ood_score_min = min_mah = float(\"inf\")\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    # if 0 not in subset:\n",
    "                    #     continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        if 0 not in x:\n",
    "                            cls_indexes_value = 0\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp, embs_temp = model(new_batch_data, cls_indexes=cls_indexes, get_enc=True)\n",
    "                        mah_temp = mah_dist(embs_temp.detach().cpu().reshape(1,-1).numpy())\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        # if 0 not in x and msp_temp > debias_threshold and (predict_temp!=predict_target or predict_temp!=predict_init):\n",
    "                        #     debias_class.append(predict_temp)\n",
    "                        #     continue\n",
    "                        # print(x, msp_temp, predict_temp)\n",
    "                        # if msp_temp > max_msp and 0 in x and predict_temp not in debias_class:\n",
    "                        if mah_temp < min_mah and 0 in x:\n",
    "                            min_mah = mah_temp\n",
    "                            mah_predict = predict_temp\n",
    "                        if predict_temp == batch[\"label\"].item():\n",
    "                            correct_permutation = True\n",
    "                            if mah_temp < correct_permutation_ood_score:\n",
    "                                correct_permutation_ood_score = mah_temp\n",
    "            mah_predicts.append(mah_predict)\n",
    "            ood_score_final_list.append(min_mah)\n",
    "            correct_permutation_mask.append(correct_permutation)\n",
    "            correct_permutation_confident_mask.append(correct_permutation_ood_score)\n",
    "            correct_mah_mask.append(mah_predict == batch[\"label\"].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_permutation_confident_scores = []\n",
    "for v in correct_permutation_confident_mask:\n",
    "    if v != float(\"inf\"):\n",
    "        correct_permutation_confident_scores.append(v[0])\n",
    "    else:\n",
    "        correct_permutation_confident_scores.append(float(\"inf\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/4168744881.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  idx_list = torch.tensor(idx_list).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  uncertain_init_mask = torch.tensor(uncertain_init_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  uncertain_target_mask = torch.tensor(uncertain_target_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_init_mask = torch.tensor(correct_init_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:5: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_target_mask = torch.tensor(correct_target_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_permutation_mask = torch.tensor(correct_permutation_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_permutation_confident_scores = torch.tensor(correct_permutation_confident_scores).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_mah_mask = torch.tensor(correct_mah_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  init_target_mask = torch.tensor(init_target_mask).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:10: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_init_list = torch.tensor(ood_score_init_list).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:11: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_target_list = torch.tensor(ood_score_target_list).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:12: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_final_list = torch.tensor(ood_score_final_list).reshape(-1)\n",
      "/tmp/ipykernel_3443360/4168744881.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mah_predicts = torch.tensor(mah_predicts).reshape(-1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1084 1084 1084 1084 1084\n"
     ]
    }
   ],
   "source": [
    "\n",
    "idx_list = torch.tensor(idx_list).reshape(-1)\n",
    "uncertain_init_mask = torch.tensor(uncertain_init_mask).reshape(-1)\n",
    "uncertain_target_mask = torch.tensor(uncertain_target_mask).reshape(-1)\n",
    "correct_init_mask = torch.tensor(correct_init_mask).reshape(-1)\n",
    "correct_target_mask = torch.tensor(correct_target_mask).reshape(-1)\n",
    "correct_permutation_mask = torch.tensor(correct_permutation_mask).reshape(-1)\n",
    "correct_permutation_confident_scores = torch.tensor(correct_permutation_confident_scores).reshape(-1)\n",
    "correct_mah_mask = torch.tensor(correct_mah_mask).reshape(-1)\n",
    "init_target_mask = torch.tensor(init_target_mask).reshape(-1)\n",
    "ood_score_init_list = torch.tensor(ood_score_init_list).reshape(-1)\n",
    "ood_score_target_list = torch.tensor(ood_score_target_list).reshape(-1)\n",
    "ood_score_final_list = torch.tensor(ood_score_final_list).reshape(-1)\n",
    "mah_predicts = torch.tensor(mah_predicts).reshape(-1)\n",
    "print(len(idx_list), len(init_target_mask), len(correct_init_mask), len(correct_target_mask), len(correct_permutation_mask))\n",
    "\n",
    "\n",
    "torch.save({\"idx_list\": idx_list, \"uncertain_init_mask\": uncertain_init_mask, \"uncertain_target_mask\": uncertain_target_mask, \"correct_init_mask\": correct_init_mask, \"correct_target_mask\": correct_target_mask, \"correct_permutation_mask\": correct_permutation_mask, \"correct_mah_mask\": correct_mah_mask, \"init_target_mask\": init_target_mask, \"ood_score_init_list\": ood_score_init_list, \"ood_score_target_list\": ood_score_target_list, \"ood_score_final_list\": ood_score_final_list}, f\"./results/brute_force_permutation_MAH_lens{max_col_length}.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1084, 1])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ood_score_init_list.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(230549.6875)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ood_score_init_list.median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(541)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ood_score_init_list<ood_score_init_list.median()).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "314360"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(condition_mask&correct_init_mask).sum().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1084, 1])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "condition_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1084])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_init_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "437 542 0.8062730627306273\n",
      "143 542 0.26383763837638374\n",
      "MSP\n",
      "408 542 0.7527675276752768\n",
      "117 542 0.2158671586715867\n",
      "MSP & Init\n",
      "399\n",
      "446 542 0.8228782287822878\n",
      "195 542 0.35977859778597787\n",
      "Permutation\n",
      "489 542 0.9022140221402214\n",
      "350 542 0.6457564575645757\n"
     ]
    }
   ],
   "source": [
    "# weighted, only uncertain init\n",
    "\n",
    "for threshold in [0.5]:\n",
    "    threshold_score = ood_score_init_list.median()\n",
    "    uncertain_init_mask = ood_score_init_list > threshold_score\n",
    "    uncertain_target_mask = ood_score_target_list > threshold_score\n",
    "    condition_mask = ~uncertain_init_mask\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((condition_mask&correct_init_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_init_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # init prediction\n",
    "    print((~condition_mask&correct_init_mask).sum().item(), \n",
    "        (~condition_mask).sum().item(), \n",
    "       (~condition_mask&correct_init_mask).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((condition_mask&correct_mah_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_mah_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # msp prediction\n",
    "    print((~condition_mask&correct_mah_mask).sum().item(), \n",
    "        (~condition_mask).sum().item(), \n",
    "        (~condition_mask&correct_mah_mask).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"MSP & Init\")\n",
    "    print(((condition_mask&correct_init_mask)&(condition_mask&correct_mah_mask)).sum().item())\n",
    "    # print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_mah_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item())\n",
    "    print(((condition_mask&correct_init_mask)|(condition_mask&correct_mah_mask)).sum().item(),\n",
    "          (condition_mask).sum().item(), \n",
    "          ((condition_mask&correct_init_mask)|(condition_mask&correct_mah_mask)).sum().item()/(condition_mask).sum().item())\n",
    "    print(((~condition_mask&correct_init_mask)|(~condition_mask&correct_mah_mask)).sum().item(),\n",
    "          (~condition_mask).sum().item(), \n",
    "          ((~condition_mask&correct_init_mask)|(~condition_mask&correct_mah_mask)).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"Permutation\")\n",
    "    print((condition_mask&correct_permutation_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_permutation_mask).sum().item()/(condition_mask).sum().item())\n",
    "    print((~condition_mask&correct_permutation_mask).sum().item(), \n",
    "         (~condition_mask).sum().item(), \n",
    "        (~condition_mask&correct_permutation_mask).sum().item()/(~condition_mask).sum().item())\n",
    "    # print(\"Permutation Confident\")\n",
    "    # print((condition_mask&correct_permutation_confident_mask).sum().item(), \n",
    "    #     (condition_mask).sum().item(), \n",
    "    #     (condition_mask&correct_permutation_confident_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # print((~condition_mask&correct_permutation_confident_mask).sum().item(),\n",
    "    #         (~condition_mask).sum().item(), \n",
    "    #         (~condition_mask&correct_permutation_confident_mask).sum().item()/(~condition_mask).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_choices = []\n",
    "num_cols = []\n",
    "min_choices = []\n",
    "for k, v in corrected_permutation.items():\n",
    "    num_choices.append(len(v))\n",
    "    if len(init_permutation[k][0])>=4:\n",
    "        num_cols.append(len(init_permutation[k][0]))\n",
    "        len_choices_i = []\n",
    "        for choice in v:\n",
    "            len_choices_i.append(len(choice))\n",
    "        min_choices.append(min(len_choices_i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 1, 3, 2, 1, 1, 1, 1, 3, 1, 4, 4, 5, 4, 5, 6, 1, 1, 2, 2, 3]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 5, 7, 7, 7, 7, 4, 4, 4]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_choices = []\n",
    "num_cols = []\n",
    "min_choices = []\n",
    "for k, v in corrected_permutation.items():\n",
    "    num_choices.append(len(v))\n",
    "    num_cols.append(len(init_permutation[k][0]))\n",
    "    len_choices_i = []\n",
    "    for choice in v:\n",
    "        len_choices_i.append(len(choice))\n",
    "    min_choices.append(min(len_choices_i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PearsonRResult(statistic=0.5184125822601837, pvalue=2.600736799525952e-05)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import pearsonr\n",
    "pearsonr(num_cols, min_choices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD5CAYAAAAgGF4oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOuElEQVR4nO3dbYxmZX3H8e8PFh8KWLBMNttl6VBraIhNwYy0gjEVqsHWVmyolLSUNOiSVAxUY6P0hTbpC9P4lLQNurLomiIP5SGiUpTiRkq02FmkPK1GS6AsruxYYwBflCz++2IO7bA7Ozs7O+cc9r6+n+TOfe7rPFz/8+Y3Z677OudOVSFJasdhYxcgSRqWwS9JjTH4JakxBr8kNcbgl6TGGPyS1Jg1fR04yUuAO4EXd/3cUFUfTHIicC3wC8A24IKqemapYx133HE1PT3dV6mSNJG2bdv2o6qa2rO9t+AH/gc4s6qeTnIEcFeSfwbeA3y8qq5N8kngIuCKpQ40PT3N7Oxsj6VK0uRJ8uhi7b0N9dS8p7uPR3SvAs4EbujatwDn9FWDJGlvvY7xJzk8yb3ALuB24D+Bn1TV7m6THcD6PmuQJD1fr8FfVc9W1SnA8cBpwK8ud98kG5PMJpmdm5vrq0RJas4gs3qq6ifAVuC1wDFJnvtu4Xjg8X3ss6mqZqpqZmpqr+8mJEkr1FvwJ5lKcky3/FLgjcB25v8AnNttdiHwhb5qkCTtrc9ZPeuALUkOZ/4PzPVV9aUkDwHXJvkb4NvA5h5rkCTtobfgr6r7gFMXaX+Y+fF+SdIIvHNXkhpj8EtSYyY++NdvOIEko7zWbzhh7NOXpL30+eXuC8IPdjzGeZ/6xih9X3fx6aP0K0lLmfgrfknS8xn8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMb0Ff5INSbYmeSjJg0ku7do/lOTxJPd2r9/pqwZJ0t7W9Hjs3cB7q+qeJEcD25Lc3q37eFV9pMe+JUn70FvwV9VOYGe3/FSS7cD6vvqTJC3PIGP8SaaBU4G7u6ZLktyX5Kokx+5jn41JZpPMzs3NDVGmJDWh9+BPchRwI3BZVT0JXAG8AjiF+f8IPrrYflW1qapmqmpmamqq7zIlqRm9Bn+SI5gP/aur6iaAqnqiqp6tqp8BnwZO67MGSdLz9TmrJ8BmYHtVfWxB+7oFm70NeKCvGiRJe+tzVs8ZwAXA/Unu7douB85PcgpQwCPAxT3WIEnaQ5+zeu4CssiqW/vqU5K0f965K0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1prfgT7IhydYkDyV5MMmlXfvLk9ye5Hvd+7F91SBJ2lufV/y7gfdW1cnAbwLvSnIy8H7gjqp6JXBH91mSNJDegr+qdlbVPd3yU8B2YD3wVmBLt9kW4Jy+apAk7W2QMf4k08CpwN3A2qra2a36IbB2H/tsTDKbZHZubm6IMifG+g0nkGTw1/oNJ4x96pKWYU3fHSQ5CrgRuKyqnkzyf+uqqpLUYvtV1SZgE8DMzMyi22hxP9jxGOd96huD93vdxacP3qekA9frFX+SI5gP/aur6qau+Ykk67r164BdfdYgSXq+Pmf1BNgMbK+qjy1YdQtwYbd8IfCFvmqQJO2tz6GeM4ALgPuT3Nu1XQ58GLg+yUXAo8Dbe6xBkrSH3oK/qu4Cso/VZ/XVryRpad65K0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmOWFfxJzlhOmyTphW+5V/x/t8w2SdIL3JK/wJXktcDpwFSS9yxY9TLg8D4LkyT1Y38/vfgi4Khuu6MXtD8JnNtXUZKk/iwZ/FX1deDrST5bVY8OVJMkqUfL/bH1FyfZBEwv3KeqzuyjKElSf5Yb/P8EfBK4Eni2v3IkSX1bbvDvrqoreq1EkjSI5U7n/GKSP0+yLsnLn3v1WpkkqRfLveK/sHt/34K2An55dcuRJPVtWcFfVSf2XYgkaRjLCv4kf7pYe1V9bol9rgLeAuyqqld1bR8C3gnMdZtdXlW3HkjBkqSDs9yhntcsWH4JcBZwD7DP4Ac+C/z9Itt8vKo+stwCJUmra7lDPe9e+DnJMcC1+9nnziTTK65MktSLlT6W+afASsf9L0lyX5Krkhy7wmNIklZouY9l/mKSW7rXl4HvAjevoL8rgFcApwA7gY8u0efGJLNJZufm5va1mSTpAC13jH/hmPxu4NGq2nGgnVXVE88tJ/k08KUltt0EbAKYmZmpA+1LkrS4ZV3xdw9r+w7zT+g8FnhmJZ0lWbfg49uAB1ZyHEnSyi13qOftwLeAPwTeDtydZMnHMie5BvgmcFKSHUkuAv42yf1J7gPeAPzFQVUvSTpgyx3q+SvgNVW1CyDJFPAvwA372qGqzl+kefMBVyhJWlXLndVz2HOh3/nvA9hXkvQCstwr/tuSfAW4pvt8HuAdt5J0CNrfb+7+CrC2qt6X5A+A13Wrvglc3XdxkqTVt78r/k8AHwCoqpuAmwCS/Fq37vd6rE2S1IP9jdOvrar792zs2qZ7qUiS1Kv9Bf8xS6x76SrWIUkayP6CfzbJO/dsTPIOYFs/JUmS+rS/Mf7LgJuT/DH/H/QzwIuYv/NWknSIWTL4u2frnJ7kDcCruuYvV9XXeq9MktSL5T6PfyuwtedaJEkD8O5bSWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JakxvwZ/kqiS7kjywoO3lSW5P8r3u/di++pckLa7PK/7PAmfv0fZ+4I6qeiVwR/dZkjSg3oK/qu4EfrxH81uBLd3yFuCcvvqXJC1u6DH+tVW1s1v+IbB2Xxsm2ZhkNsns3NzcMNVJUgNG+3K3qgqoJdZvqqqZqpqZmpoasDJJmmxDB/8TSdYBdO+7Bu5fkpo3dPDfAlzYLV8IfGHg/iWpeX1O57wG+CZwUpIdSS4CPgy8Mcn3gN/uPkuSBrSmrwNX1fn7WHVWX31KkvbPO3clqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMasGaPTJI8ATwHPAruramaMOiSpRaMEf+cNVfWjEfuXpCY51CNJjRkr+Av4apJtSTYutkGSjUlmk8zOzc0NXJ4kTa6xgv91VfVq4M3Au5K8fs8NqmpTVc1U1czU1NTwFUrShBol+Kvq8e59F3AzcNoYdUhSiwYP/iRHJjn6uWXgTcADQ9chSa0aY1bPWuDmJM/1//mqum2EOiSpSYMHf1U9DPz60P1KkuY5nVOSGmPwayKs33ACSQZ/rd9wwtinLh2wMe/clVbND3Y8xnmf+sbg/V538emD9ykdLK/4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLhyinsGqlnM4pHaKcwqqV8opfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl3TI8N6F1eE8fkmHDO9dWB1e8UtSYwx+SWqMwS9JjTH4JakxBr8kNcZZPZK0P4etIckoXf/i8Rt4/LH/WtVjGvyStD8/2z3KNFLoZyqpQz2S1JhRgj/J2Um+m+T7Sd4/Rg2S1KrBgz/J4cA/AG8GTgbOT3Ly0HVIUqvGuOI/Dfh+VT1cVc8A1wJvHaEOSWrSGMG/HnhswecdXZskaQCpqmE7TM4Fzq6qd3SfLwB+o6ou2WO7jcDG7uNJwHdX2OVxwI9WuO+hynNug+fchoM551+qqqk9G8eYzvk4sGHB5+O7tuepqk3ApoPtLMlsVc0c7HEOJZ5zGzznNvRxzmMM9fw78MokJyZ5EfBHwC0j1CFJTRr8ir+qdie5BPgKcDhwVVU9OHQdktSqUe7crapbgVsH6u6gh4sOQZ5zGzznNqz6OQ/+5a4kaVw+skGSGjOxwZ/kqiS7kjwwdi1DSbIhydYkDyV5MMmlY9fUtyQvSfKtJP/RnfNfj13TEJIcnuTbSb40di1DSPJIkvuT3Jtkdux6hpDkmCQ3JPlOku1JXrtqx57UoZ4krweeBj5XVa8au54hJFkHrKuqe5IcDWwDzqmqh0YurTeZf1bukVX1dJIjgLuAS6vq30YurVdJ3gPMAC+rqreMXU/fkjwCzFRVM3P4k2wB/rWqruxmQP5cVf1kNY49sVf8VXUn8OOx6xhSVe2sqnu65aeA7Uz4XdE17+nu4xHdazKvZjpJjgd+F7hy7FrUjyQ/D7we2AxQVc+sVujDBAd/65JMA6cCd49cSu+6YY97gV3A7VU16ef8CeAvgZ+NXMeQCvhqkm3dXf2T7kRgDvhMN6R3ZZIjV+vgBv8ESnIUcCNwWVU9OXY9fauqZ6vqFObvAj8tycQO7SV5C7CrqraNXcvAXldVr2b+qb7v6oZyJ9ka4NXAFVV1KvBTYNUeYW/wT5hunPtG4OqqumnseobU/Su8FTh75FL6dAbw+92Y97XAmUn+cdyS+ldVj3fvu4CbmX/K7yTbAexY8N/rDcz/IVgVBv8E6b7o3Axsr6qPjV3PEJJMJTmmW34p8EbgO6MW1aOq+kBVHV9V08w/7uRrVfUnI5fVqyRHdpMV6IY73gRM9Gy9qvoh8FiSk7qms4BVm6Qxsb+5m+Qa4LeA45LsAD5YVZvHrap3ZwAXAPd3Y94Al3d3Sk+qdcCW7gd+DgOur6ompjg2ZC1wc/dj52uAz1fVbeOWNIh3A1d3M3oeBv5stQ48sdM5JUmLc6hHkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1Jj/BbcNjBLyxontAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(min_choices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_choices = []\n",
    "num_cols = []\n",
    "min_choices = []\n",
    "start_with_target = 0\n",
    "for k, v in corrected_permutation.items():\n",
    "    num_choices.append(len(v))\n",
    "    num_cols.append(len(init_permutation[k][0]))\n",
    "    len_choices_i = []\n",
    "    for choice in v:\n",
    "        if choice[0] == 0:\n",
    "            start_with_target += 1\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7457627118644068"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_with_target/len(corrected_permutation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/4127766715.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/4127766715.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4083, ts_macro_f1=0.1855\n",
      "ts_micro_f1=0.4489, ts_macro_f1=0.1968\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/4127766715.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/4127766715.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4129, ts_macro_f1=0.1904\n",
      "ts_micro_f1=0.4575, ts_macro_f1=0.2042\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/4127766715.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/4127766715.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4138, ts_macro_f1=0.1938\n",
      "ts_micro_f1=0.4593, ts_macro_f1=0.2125\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 2.0****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/4127766715.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/4127766715.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4111, ts_macro_f1=0.1947\n",
      "ts_micro_f1=0.4541, ts_macro_f1=0.2084\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.5, 1.0, 1.5, 2.0]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
    "        predict_init = logits.argmax().item()\n",
    "        if 1 in target_col_mask and ood_score_min > threshold:\n",
    "            total_mistakes += 1\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        if ood_score_temp < ood_score_min and predict_init != predict_temp:\n",
    "                            ood_score_min = ood_score_temp \n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/1658014317.py:25: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_4054970/1658014317.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4442, ts_macro_f1=0.1641\n",
      "ts_micro_f1=0.4107, ts_macro_f1=0.1462\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate, target column\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.9]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "\n",
    "        cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "        logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)    \n",
    "        init_msp = F.softmax(logits_temp).max().item()\n",
    "        logits = logits_temp.clone().detach().cpu()      \n",
    "\n",
    "        if 1 in target_col_mask and init_msp < threshold:\n",
    "            max_msp = init_msp\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        msp_temp = F.softmax(logits_temp).max().item()\n",
    "                        # predict_temp = logits_temp.argmax().item()\n",
    "                        if msp_temp > max_msp:\n",
    "                            max_msp = msp_temp\n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3892576816.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/3892576816.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4498, ts_macro_f1=0.1765\n",
      "ts_micro_f1=0.4211, ts_macro_f1=0.1604\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.6****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3892576816.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/3892576816.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4479, ts_macro_f1=0.1734\n",
      "ts_micro_f1=0.4177, ts_macro_f1=0.1560\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.7****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3892576816.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/3892576816.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4507, ts_macro_f1=0.1720\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1554\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.8****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3892576816.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/3892576816.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4535, ts_macro_f1=0.1741\n",
      "ts_micro_f1=0.4281, ts_macro_f1=0.1598\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate, original answer\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.5, 0.6, 0.7, 0.8]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        init_msp = F.softmax(logits).max().item()\n",
    "\n",
    "        if 1 in target_col_mask and init_msp < threshold:\n",
    "            max_msp = init_msp\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        msp_temp = F.softmax(logits_temp).max().item()\n",
    "                        # predict_temp = logits_temp.argmax().item()\n",
    "                        if msp_temp > max_msp:\n",
    "                            max_msp = msp_temp\n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4188673808.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/4188673808.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4498, ts_macro_f1=0.1714\n",
      "ts_micro_f1=0.4211, ts_macro_f1=0.1564\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.85****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4188673808.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/4188673808.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4479, ts_macro_f1=0.1660\n",
      "ts_micro_f1=0.4177, ts_macro_f1=0.1492\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4188673808.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/4188673808.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4461, ts_macro_f1=0.1661\n",
      "ts_micro_f1=0.4142, ts_macro_f1=0.1494\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.95****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4188673808.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/4188673808.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4442, ts_macro_f1=0.1613\n",
      "ts_micro_f1=0.4107, ts_macro_f1=0.1427\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.99****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/4188673808.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/4188673808.py:48: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4387, ts_macro_f1=0.1594\n",
      "ts_micro_f1=0.4003, ts_macro_f1=0.1393\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate, original answer, skip permuation not starting with 0\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.8, 0.85, 0.9, 0.95, 0.99]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        init_msp = F.softmax(logits).max().item()\n",
    "\n",
    "        if 1 in target_col_mask and init_msp < threshold:\n",
    "            max_msp = init_msp\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        if x[0] != 0:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        msp_temp = F.softmax(logits_temp).max().item()\n",
    "                        # predict_temp = logits_temp.argmax().item()\n",
    "                        if msp_temp > max_msp:\n",
    "                            max_msp = msp_temp\n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3393821501.py:25: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_4054970/3393821501.py:50: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4442, ts_macro_f1=0.1663\n",
      "ts_micro_f1=0.4107, ts_macro_f1=0.1525\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************Threshold: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/3393821501.py:25: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_4054970/3393821501.py:50: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4415, ts_macro_f1=0.1657\n",
      "ts_micro_f1=0.4055, ts_macro_f1=0.1513\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.8, 0.9]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "\n",
    "        cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "        logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)    \n",
    "        init_msp = F.softmax(logits_temp).max().item()\n",
    "        logits = logits_temp.clone().detach().cpu()      \n",
    "\n",
    "        if 1 in target_col_mask and init_msp < threshold:\n",
    "            max_msp = init_msp\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        # if x[0] != 0:\n",
    "                        #     continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        msp_temp = F.softmax(logits_temp).max().item()\n",
    "                        # predict_temp = logits_temp.argmax().item()\n",
    "                        if msp_temp > max_msp:\n",
    "                            max_msp = msp_temp\n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3432584509.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/3432584509.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4138, ts_macro_f1=0.1897\n",
      "ts_micro_f1=0.4593, ts_macro_f1=0.2045\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.0****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3432584509.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/3432584509.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4184, ts_macro_f1=0.1936\n",
      "ts_micro_f1=0.4679, ts_macro_f1=0.2097\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 1.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3432584509.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/3432584509.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4166, ts_macro_f1=0.1959\n",
      "ts_micro_f1=0.4645, ts_macro_f1=0.2159\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n",
      "*********************Threshold: 2.0****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3359783/3432584509.py:24: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
      "/tmp/ipykernel_3359783/3432584509.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4120, ts_macro_f1=0.1950\n",
      "ts_micro_f1=0.4558, ts_macro_f1=0.2090\n",
      "ts_micro_f1=0.3622, ts_macro_f1=0.1474\n"
     ]
    }
   ],
   "source": [
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "for threshold in [0.5, 1.0, 1.5, 2.0]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        ood_score_min = compute_entropy(F.softmax(logits.detach())).item()\n",
    "        predict_init = logits.argmax().item()\n",
    "        if 1 in target_col_mask and ood_score_min > threshold:\n",
    "            total_mistakes += 1\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = compute_entropy(F.softmax(logits_temp.detach())).item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        if ood_score_temp < ood_score_min:\n",
    "                            ood_score_min = ood_score_temp \n",
    "                            logits = logits_temp.clone()\n",
    "                \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_dataloader_iter' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [59]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtrain_dataloader_iter\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_dataloader_iter' is not defined"
     ]
    }
   ],
   "source": [
    "train_dataloader_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 1.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/145533155.py:30: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  score_init[batch_idx] = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_955267/145533155.py:58: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [12]\u001b[0m, in \u001b[0;36m<cell line: 12>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m new_batch_data \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat(new_batch_data, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     56\u001b[0m cls_indexes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor([\u001b[38;5;241m0\u001b[39m, cls_indexes_value])\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 57\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_batch_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcls_indexes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     58\u001b[0m ood_score_temp \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39msoftmax(logits_temp\u001b[38;5;241m.\u001b[39mdetach())\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m     59\u001b[0m score_permutation[batch_idx]\u001b[38;5;241m.\u001b[39mappend(ood_score_temp)\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:575\u001b[0m, in \u001b[0;36mBertForMultiOutputClassification.forward\u001b[0;34m(self, input_ids, get_enc, cls_indexes, token_type_ids)\u001b[0m\n\u001b[1;32m    573\u001b[0m table_length \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mlen\u001b[39m(input_ids[i]\u001b[38;5;241m.\u001b[39mnonzero()) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(input_ids))]\n\u001b[1;32m    574\u001b[0m \u001b[38;5;66;03m# Note: returned tensor contains pooled_output of all tokens (to make the tensor size consistent)\u001b[39;00m\n\u001b[0;32m--> 575\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpooler\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbert_output\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpooler_output\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbert_output\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcls_indexes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtable_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtable_length\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    576\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(pooled_output)\n\u001b[1;32m    577\u001b[0m logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclassifier(pooled_output)\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:460\u001b[0m, in \u001b[0;36mBertMultiPooler.forward\u001b[0;34m(self, hidden_states, pooler_output, cls_indexes, table_length)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, hidden_states, pooler_output\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, cls_indexes\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, table_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m    453\u001b[0m     \u001b[38;5;66;03m# We \"pool\" the model by simply taking the hidden state corresponding\u001b[39;00m\n\u001b[1;32m    454\u001b[0m     \u001b[38;5;66;03m# to the first token.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    457\u001b[0m     \u001b[38;5;66;03m# Apply\u001b[39;00m\n\u001b[1;32m    458\u001b[0m     \u001b[38;5;66;03m#pooled_outputs = self.dense(token_tensor)\u001b[39;00m\n\u001b[1;32m    459\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mversion \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mv0\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 460\u001b[0m         pooled_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mextract_cls_tokens\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    461\u001b[0m         pooled_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdense(pooled_outputs)\n\u001b[1;32m    462\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mversion \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mv0.1\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:438\u001b[0m, in \u001b[0;36mextract_cls_tokens\u001b[0;34m(hidden_states, cls_indexes, head)\u001b[0m\n\u001b[1;32m    436\u001b[0m cls_embeddings \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    437\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, j \u001b[38;5;129;01min\u001b[39;00m cls_indexes:\n\u001b[0;32m--> 438\u001b[0m     sub_sentence_cls_embeddings \u001b[38;5;241m=\u001b[39m hidden_states[i, \u001b[38;5;241m0\u001b[39m, :] \u001b[38;5;28;01mif\u001b[39;00m head \u001b[38;5;28;01melse\u001b[39;00m \u001b[43mhidden_states\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m    439\u001b[0m     cls_embeddings\u001b[38;5;241m.\u001b[39mappend(sub_sentence_cls_embeddings)\n\u001b[1;32m    440\u001b[0m cls_embeddings \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mstack(cls_embeddings)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# brute force perumate in training TODO: restrict mamimus length of permutation\n",
    "import itertools\n",
    "import torch.nn.functional as F\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "for threshold in [1.5]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "    num_permutations = {}\n",
    "    init_correctness = {}\n",
    "    score_init = {}\n",
    "    score_permutation = defaultdict(list)\n",
    "    permutation_correctness = defaultdict(list)\n",
    "    for batch_idx, batch in enumerate(train_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        score_init[batch_idx] = F.softmax(logits.detach()).max().item()\n",
    "        label_i = batch[\"label\"].item()\n",
    "        predict_init = logits.argmax().item()\n",
    "        if predict_init == label_i:\n",
    "            init_correctness[batch_idx] = True\n",
    "        else:\n",
    "            init_correctness[batch_idx] = False\n",
    "        num_permutations[batch_idx] = 0\n",
    "        if 1 in target_col_mask:\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, min(len(col_idx_set) + 1, 3)):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    if 0 not in subset:\n",
    "                        continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "\n",
    "                        num_permutations[batch_idx] += 1\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp = F.softmax(logits_temp.detach()).max().item()\n",
    "                        score_permutation[batch_idx].append(ood_score_temp)\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        permutation_correctness[batch_idx].append(predict_temp == label_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "484"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************alpha: 0.25****************************\n",
      "165\n",
      "*********************alpha: 0.5****************************\n",
      "166\n",
      "*********************alpha: 1.0****************************\n",
      "169\n"
     ]
    }
   ],
   "source": [
    "for alpha in [0.25, 0.5, 1.0]:\n",
    "    class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "    # Normalize the weights\n",
    "    class_weights /= class_weights.sum()\n",
    "    \n",
    "    \n",
    "    print(f\"*********************alpha: {alpha}****************************\")\n",
    "    msp_predict_mask_weighted = []\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        if batch_idx in idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))]:\n",
    "            target_col_mask = batch[\"target_col_mask\"].T\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            max_msp = 0\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    if 0 not in subset:\n",
    "                        continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                        msp_temp = logits_temp.max().item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        # print(x, msp_temp, predict_temp)\n",
    "                        if msp_temp > max_msp and 0 in x:\n",
    "                            max_msp = msp_temp\n",
    "                            best_msp_perm = x\n",
    "                            msp_predict = predict_temp\n",
    "                            logits = logits_temp.clone()\n",
    "            msp_predict_mask_weighted.append(msp_predict==batch[\"label\"].item())\n",
    "    print(sum(msp_predict_mask_weighted))\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "168"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))& correct_msp_mask])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "rest_idx = []\n",
    "for idx in idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))& correct_msp_mask]:\n",
    "    if idx not in idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))][msp_predict_mask_weighted]:\n",
    "        rest_idx.append(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rest_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "169"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idx_list[(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))][msp_predict_mask_weighted])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "msp_predict_mask_weighted = torch.tensor(msp_predict_mask_weighted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "169"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(msp_predict_mask_weighted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "428"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idx_list[~condition_mask])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9"
      ]
     },
     "execution_count": 417,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************debias: 0.9; msp: 0.8****************************\n",
      "ts_micro_f1=0.4525, ts_macro_f1=0.1804\n",
      "ts_micro_f1=0.4263, ts_macro_f1=0.1747\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************debias: 0.9; msp: 0.95****************************\n",
      "ts_micro_f1=0.4571, ts_macro_f1=0.1903\n",
      "ts_micro_f1=0.4350, ts_macro_f1=0.1862\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************debias: 0.9; msp: 0.99****************************\n",
      "ts_micro_f1=0.4525, ts_macro_f1=0.1865\n",
      "ts_micro_f1=0.4263, ts_macro_f1=0.1784\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "alpha = 0.5\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "for msp_threshold in [0.8,  0.95, 0.99]:\n",
    "    for debias_threshold in [0.9]:\n",
    "        print(f\"*********************debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "        ft_embs_test = []\n",
    "        labels_test = []\n",
    "        logits_test = []\n",
    "        log = defaultdict(list)\n",
    "        num_cols = []\n",
    "        corrected = 0\n",
    "        total_mistakes = 0\n",
    "\n",
    "        for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            target_col_mask = batch[\"target_col_mask\"].T\n",
    "            logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "            num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "            predict_init = logits.argmax().item()\n",
    "            max_msp = 0\n",
    "            if batch_idx in idx_list[~condition_mask]:\n",
    "                col_idx_set = target_col_mask.unique().tolist()\n",
    "                debias_classes = []\n",
    "                assert -1 not in col_idx_set\n",
    "                for r in range(1, len(col_idx_set) + 1):\n",
    "                    for subset in itertools.combinations(col_idx_set, r):\n",
    "                        # if 0 not in subset:\n",
    "                        #     continue\n",
    "                        for x in itertools.permutations(subset):\n",
    "                            new_batch_data = []\n",
    "                            for col_i in x:\n",
    "                                if col_i == 0:\n",
    "                                    if len(new_batch_data) == 0:\n",
    "                                        cls_indexes_value = 0\n",
    "                                    else:\n",
    "                                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                            if 0 not in x:\n",
    "                                cls_indexes_value = 0\n",
    "                            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                            logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                            msp_temp = logits_temp.max().item()\n",
    "                            predict_temp = logits_temp.argmax().item()\n",
    "                            if len(x) == 1 and 0 in x:\n",
    "                                predict_target = predict_temp\n",
    "                                msp_target = msp_temp\n",
    "                            # print(x, msp_temp, predict_temp)\n",
    "                            if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                debias_classes.append(predict_temp)\n",
    "                                continue\n",
    "                            if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                max_msp = msp_temp\n",
    "                                best_msp_perm = x\n",
    "                                msp_predict = predict_temp\n",
    "                                logits_msp = logits_temp.clone()\n",
    "            if max_msp > msp_threshold:\n",
    "                logits = logits_msp.clone()\n",
    "            labels_test.append(batch[\"label\"].cpu())\n",
    "            logits_test.append(logits.detach().cpu())\n",
    "        labels_test = torch.cat(labels_test, dim=0)\n",
    "        logits_test = torch.stack(logits_test, dim=0)\n",
    "        preds_test = torch.argmax(logits_test, dim=1)\n",
    "        num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        from sklearn.metrics import confusion_matrix, f1_score\n",
    "        mask = num_cols > 0\n",
    "        ts_pred_list = logits_test.argmax(\n",
    "                                    1).cpu().detach().numpy().tolist()\n",
    "        ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"micro\")\n",
    "        ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"macro\")\n",
    "        print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "        ts_pred_list = logits_test.argmax(\n",
    "                                    1).cpu().detach()[mask].numpy().tolist()\n",
    "        ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"micro\")\n",
    "        ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"macro\")\n",
    "        print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "        ts_pred_list = logits_test.argmax(\n",
    "                                    1).cpu().detach()[~mask].numpy().tolist()\n",
    "        ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"micro\")\n",
    "        ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                            ts_pred_list,\n",
    "                            average=\"macro\")\n",
    "        print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************uncertain: 0.9; debias: 0.8; msp: 0.8****************************\n",
      "ts_micro_f1=0.5336, ts_macro_f1=0.2490\n",
      "ts_micro_f1=0.5332, ts_macro_f1=0.2490\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************uncertain: 0.9; debias: 0.8; msp: 0.9****************************\n",
      "ts_micro_f1=0.5346, ts_macro_f1=0.2527\n",
      "ts_micro_f1=0.5341, ts_macro_f1=0.2527\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "alpha = 0.25\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "max_col_length = 3\n",
    "# for global_threshold in [ 0.7, 0.8, 0.9, 0.99]:\n",
    "for uncertainty_threshold in [0.9]:\n",
    "    for msp_threshold in [0.8, 0.9]:\n",
    "        for debias_threshold in [0.8]:\n",
    "            print(f\"*********************uncertain: {uncertainty_threshold}; debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "            ft_embs_test = []\n",
    "            labels_test = []\n",
    "            logits_test = []\n",
    "            log = defaultdict(list)\n",
    "            num_cols = []\n",
    "            corrected = 0\n",
    "            total_mistakes = 0\n",
    "\n",
    "            for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                target_col_mask = batch[\"target_col_mask\"].T\n",
    "                logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "                logits = reweight_logits(logits.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "                msp_init = logits.max().item()\n",
    "                num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                predict_init = logits.argmax().item()\n",
    "                max_msp = 0\n",
    "                if 1 in target_col_mask and msp_init < uncertainty_threshold:\n",
    "                    col_idx_set = target_col_mask.unique().tolist()\n",
    "                    debias_classes = []\n",
    "                    assert -1 not in col_idx_set\n",
    "                    for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                        for subset in itertools.combinations(col_idx_set, r):\n",
    "                            # if 0 not in subset:\n",
    "                            #     continue\n",
    "                            for x in itertools.permutations(subset):\n",
    "                                new_batch_data = []\n",
    "                                for col_i in x:\n",
    "                                    if col_i == 0:\n",
    "                                        if len(new_batch_data) == 0:\n",
    "                                            cls_indexes_value = 0\n",
    "                                        else:\n",
    "                                            cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                    new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                                if 0 not in x:\n",
    "                                    cls_indexes_value = 0\n",
    "                                new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                msp_temp = logits_temp.max().item()\n",
    "                                predict_temp = logits_temp.argmax().item()\n",
    "                                if len(x) == 1 and 0 in x:\n",
    "                                    predict_target = predict_temp\n",
    "                                    msp_target = msp_temp\n",
    "                                # print(x, msp_temp, predict_temp)\n",
    "                                if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                    debias_classes.append(predict_temp)\n",
    "                                    continue\n",
    "                                if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                    max_msp = msp_temp\n",
    "                                    best_msp_perm = x\n",
    "                                    msp_predict = predict_temp\n",
    "                                    logits_msp = logits_temp.clone()\n",
    "                if max_msp > msp_threshold:\n",
    "                    logits = logits_msp.clone()\n",
    "                labels_test.append(batch[\"label\"].cpu())\n",
    "                logits_test.append(logits.detach().cpu())\n",
    "            labels_test = torch.cat(labels_test, dim=0)\n",
    "            logits_test = torch.stack(logits_test, dim=0)\n",
    "            preds_test = torch.argmax(logits_test, dim=1)\n",
    "            num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "            from sklearn.metrics import confusion_matrix, f1_score\n",
    "            mask = num_cols > 0\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach().numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach()[mask].numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach()[~mask].numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************uncertain: 0.9; debias: 1.0; msp: 0.95****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4525, ts_macro_f1=0.1810\n",
      "ts_micro_f1=0.4263, ts_macro_f1=0.1742\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.9; debias: 1.0; msp: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4535, ts_macro_f1=0.1840\n",
      "ts_micro_f1=0.4281, ts_macro_f1=0.1790\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.9; debias: 1.0; msp: 0.99****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4553, ts_macro_f1=0.1895\n",
      "ts_micro_f1=0.4315, ts_macro_f1=0.1826\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.8; debias: 1.0; msp: 0.95****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4553, ts_macro_f1=0.1863\n",
      "ts_micro_f1=0.4315, ts_macro_f1=0.1813\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.8; debias: 1.0; msp: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4571, ts_macro_f1=0.1894\n",
      "ts_micro_f1=0.4350, ts_macro_f1=0.1861\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.8; debias: 1.0; msp: 0.99****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4562, ts_macro_f1=0.1914\n",
      "ts_micro_f1=0.4333, ts_macro_f1=0.1849\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.95; debias: 1.0; msp: 0.95****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4507, ts_macro_f1=0.1762\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1675\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.95; debias: 1.0; msp: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4525, ts_macro_f1=0.1801\n",
      "ts_micro_f1=0.4263, ts_macro_f1=0.1738\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.95; debias: 1.0; msp: 0.99****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4544, ts_macro_f1=0.1868\n",
      "ts_micro_f1=0.4298, ts_macro_f1=0.1788\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.99; debias: 1.0; msp: 0.95****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4488, ts_macro_f1=0.1748\n",
      "ts_micro_f1=0.4194, ts_macro_f1=0.1650\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.99; debias: 1.0; msp: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4507, ts_macro_f1=0.1786\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1710\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
      "*********************uncertain: 0.99; debias: 1.0; msp: 0.99****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4535, ts_macro_f1=0.1856\n",
      "ts_micro_f1=0.4281, ts_macro_f1=0.1765\n",
      "ts_micro_f1=0.4823, ts_macro_f1=0.1573\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate weighted\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "alpha = 0.5\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "for uncertainty_threshold in [0.9, 0.8, 0.95, 0.99]:\n",
    "    for msp_threshold in [ 0.95, 0.9, 0.99]:\n",
    "        for debias_threshold in [1.0]:\n",
    "            print(f\"*********************uncertain: {uncertainty_threshold}; debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "            ft_embs_test = []\n",
    "            labels_test = []\n",
    "            logits_test = []\n",
    "            log = defaultdict(list)\n",
    "            num_cols = []\n",
    "            corrected = 0\n",
    "            total_mistakes = 0\n",
    "\n",
    "            for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                target_col_mask = batch[\"target_col_mask\"].T\n",
    "                logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "                msp_init = F.softmax(logits).max().item()\n",
    "                # logits = reweight_logits(logits.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "                num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                predict_init = logits.argmax().item()\n",
    "                max_msp = 0\n",
    "                if 1 in target_col_mask and msp_init < uncertainty_threshold:\n",
    "                    col_idx_set = target_col_mask.unique().tolist()\n",
    "                    debias_classes = []\n",
    "                    assert -1 not in col_idx_set\n",
    "                    for r in range(1, len(col_idx_set) + 1):\n",
    "                        for subset in itertools.combinations(col_idx_set, r):\n",
    "                            # if 0 not in subset:\n",
    "                            #     continue\n",
    "                            for x in itertools.permutations(subset):\n",
    "                                new_batch_data = []\n",
    "                                for col_i in x:\n",
    "                                    if col_i == 0:\n",
    "                                        if len(new_batch_data) == 0:\n",
    "                                            cls_indexes_value = 0\n",
    "                                        else:\n",
    "                                            cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                    new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                                if 0 not in x:\n",
    "                                    cls_indexes_value = 0\n",
    "                                new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                # logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                logits_temp = F.softmax(logits_temp)\n",
    "                                msp_temp = logits_temp.max().item()\n",
    "                                predict_temp = logits_temp.argmax().item()\n",
    "                                if len(x) == 1 and 0 in x:\n",
    "                                    predict_target = predict_temp\n",
    "                                    msp_target = msp_temp\n",
    "                                # print(x, msp_temp, predict_temp)\n",
    "                                if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                    debias_classes.append(predict_temp)\n",
    "                                    continue\n",
    "                                if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                    max_msp = msp_temp\n",
    "                                    best_msp_perm = x\n",
    "                                    msp_predict = predict_temp\n",
    "                                    logits_msp = logits_temp.clone()\n",
    "                if max_msp > msp_threshold:\n",
    "                    logits = logits_msp.clone()\n",
    "                labels_test.append(batch[\"label\"].cpu())\n",
    "                logits_test.append(logits.detach().cpu())\n",
    "            labels_test = torch.cat(labels_test, dim=0)\n",
    "            logits_test = torch.stack(logits_test, dim=0)\n",
    "            preds_test = torch.argmax(logits_test, dim=1)\n",
    "            num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "            from sklearn.metrics import confusion_matrix, f1_score\n",
    "            mask = num_cols > 0\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach().numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach()[mask].numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "            ts_pred_list = logits_test.argmax(\n",
    "                                        1).cpu().detach()[~mask].numpy().tolist()\n",
    "            ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"micro\")\n",
    "            ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                ts_pred_list,\n",
    "                                average=\"macro\")\n",
    "            print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************alpha: 0.0; uncertain: 0.7; debias: 0.7; msp: 0.7****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3122311/845942549.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits = F.softmax(logits)\n",
      "/tmp/ipykernel_3122311/845942549.py:62: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.5134, ts_macro_f1=0.2335\n",
      "ts_micro_f1=0.5129, ts_macro_f1=0.2335\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "# class_weights = (1.0 / class_freq) ** alpha\n",
    "# msp_threshold = 0.9\n",
    "# # Normalize the weights\n",
    "# class_weights /= class_weights.sum()\n",
    "max_col_length = 3\n",
    "for alpha in [0.0]:\n",
    "    for uncertainty_threshold in [0.7]:\n",
    "        for msp_threshold in [0.7]:\n",
    "            for debias_threshold in [0.7]:\n",
    "                print(f\"*********************alpha: {alpha}; uncertain: {uncertainty_threshold}; debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "                ft_embs_test = []\n",
    "                labels_test = []\n",
    "                logits_test = []\n",
    "                log = defaultdict(list)\n",
    "                num_cols = []\n",
    "                corrected = 0\n",
    "                total_mistakes = 0\n",
    "\n",
    "                for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                    target_col_mask = batch[\"target_col_mask\"].T\n",
    "                    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "                    # logits = reweight_logits(logits.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "                    logits = F.softmax(logits)\n",
    "                    msp_init = logits.max().item()\n",
    "                    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                    predict_init = logits.argmax().item()\n",
    "                    max_msp = 0\n",
    "                    if 1 in target_col_mask and msp_init < uncertainty_threshold:\n",
    "                        col_idx_set = target_col_mask.unique().tolist()\n",
    "                        debias_classes = []\n",
    "                        assert -1 not in col_idx_set\n",
    "                        for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                            for subset in itertools.combinations(col_idx_set, r):\n",
    "                                # if 0 not in subset:\n",
    "                                #     continue\n",
    "                                for x in itertools.permutations(subset):\n",
    "                                    new_batch_data = []\n",
    "                                    for col_i in x:\n",
    "                                        if col_i == 0:\n",
    "                                            if len(new_batch_data) == 0:\n",
    "                                                cls_indexes_value = 0\n",
    "                                            else:\n",
    "                                                cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                        new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                                    if 0 not in x:\n",
    "                                        cls_indexes_value = 0\n",
    "                                    new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                    cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                    logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                    logits_temp = F.softmax(logits_temp)\n",
    "                                    # logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                    msp_temp = logits_temp.max().item()\n",
    "                                    predict_temp = logits_temp.argmax().item()\n",
    "                                    if len(x) == 1 and 0 in x:\n",
    "                                        predict_target = predict_temp\n",
    "                                        msp_target = msp_temp\n",
    "                                    # print(x, msp_temp, predict_temp)\n",
    "                                    if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                        debias_classes.append(predict_temp)\n",
    "                                        continue\n",
    "                                    if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                        max_msp = msp_temp\n",
    "                                        best_msp_perm = x\n",
    "                                        msp_predict = predict_temp\n",
    "                                        logits_msp = logits_temp.clone()\n",
    "                    if max_msp > msp_threshold:\n",
    "                        logits = logits_msp.clone()\n",
    "                    labels_test.append(batch[\"label\"].cpu())\n",
    "                    logits_test.append(logits.detach().cpu())\n",
    "                labels_test = torch.cat(labels_test, dim=0)\n",
    "                logits_test = torch.stack(logits_test, dim=0)\n",
    "                preds_test = torch.argmax(logits_test, dim=1)\n",
    "                num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "                from sklearn.metrics import confusion_matrix, f1_score\n",
    "                mask = num_cols > 0\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach().numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0060, 0.0065, 0.0071, 0.0073, 0.0079, 0.0081, 0.0083, 0.0082, 0.0082,\n",
       "        0.0086, 0.0083, 0.0084, 0.0085, 0.0086, 0.0086, 0.0086, 0.0090, 0.0088,\n",
       "        0.0087, 0.0086, 0.0087, 0.0087, 0.0089, 0.0090, 0.0090, 0.0091, 0.0092,\n",
       "        0.0093, 0.0094, 0.0095, 0.0096, 0.0092, 0.0100, 0.0094, 0.0095, 0.0096,\n",
       "        0.0098, 0.0094, 0.0098, 0.0097, 0.0096, 0.0097, 0.0101, 0.0101, 0.0103,\n",
       "        0.0098, 0.0100, 0.0100, 0.0107, 0.0100, 0.0105, 0.0104, 0.0105, 0.0103,\n",
       "        0.0104, 0.0105, 0.0105, 0.0103, 0.0100, 0.0109, 0.0101, 0.0103, 0.0111,\n",
       "        0.0103, 0.0111, 0.0104, 0.0109, 0.0109, 0.0103, 0.0101, 0.0107, 0.0104,\n",
       "        0.0105, 0.0109, 0.0105, 0.0104, 0.0107, 0.0111, 0.0105, 0.0109, 0.0105,\n",
       "        0.0105, 0.0111, 0.0116, 0.0109, 0.0107, 0.0109, 0.0116, 0.0105, 0.0116,\n",
       "        0.0113, 0.0107, 0.0113, 0.0111, 0.0109, 0.0116, 0.0121, 0.0121, 0.0113,\n",
       "        0.0111, 0.0113])"
      ]
     },
     "execution_count": 430,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha = 0.125\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0001, 0.0002, 0.0005, 0.0006, 0.0012, 0.0015, 0.0017, 0.0015, 0.0017,\n",
       "        0.0023, 0.0017, 0.0020, 0.0022, 0.0023, 0.0024, 0.0023, 0.0035, 0.0029,\n",
       "        0.0025, 0.0025, 0.0026, 0.0027, 0.0031, 0.0035, 0.0033, 0.0037, 0.0039,\n",
       "        0.0042, 0.0046, 0.0053, 0.0056, 0.0041, 0.0076, 0.0046, 0.0050, 0.0059,\n",
       "        0.0066, 0.0048, 0.0066, 0.0062, 0.0056, 0.0062, 0.0088, 0.0088, 0.0096,\n",
       "        0.0066, 0.0082, 0.0082, 0.0133, 0.0082, 0.0118, 0.0106, 0.0118, 0.0096,\n",
       "        0.0106, 0.0118, 0.0118, 0.0096, 0.0082, 0.0151, 0.0088, 0.0096, 0.0177,\n",
       "        0.0096, 0.0177, 0.0106, 0.0151, 0.0151, 0.0096, 0.0088, 0.0133, 0.0106,\n",
       "        0.0118, 0.0151, 0.0118, 0.0106, 0.0133, 0.0177, 0.0118, 0.0151, 0.0118,\n",
       "        0.0118, 0.0177, 0.0265, 0.0151, 0.0133, 0.0151, 0.0265, 0.0118, 0.0265,\n",
       "        0.0212, 0.0133, 0.0212, 0.0177, 0.0151, 0.0265, 0.0353, 0.0353, 0.0212,\n",
       "        0.0177, 0.0212])"
      ]
     },
     "execution_count": 432,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha = 1.0\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0036, 0.0039, 0.0045, 0.0047, 0.0054, 0.0056, 0.0058, 0.0057, 0.0058,\n",
       "        0.0063, 0.0058, 0.0060, 0.0062, 0.0063, 0.0063, 0.0063, 0.0071, 0.0067,\n",
       "        0.0064, 0.0064, 0.0065, 0.0065, 0.0068, 0.0071, 0.0069, 0.0071, 0.0073,\n",
       "        0.0075, 0.0077, 0.0080, 0.0081, 0.0074, 0.0091, 0.0077, 0.0079, 0.0083,\n",
       "        0.0087, 0.0078, 0.0087, 0.0085, 0.0081, 0.0085, 0.0097, 0.0097, 0.0100,\n",
       "        0.0087, 0.0094, 0.0094, 0.0115, 0.0094, 0.0109, 0.0104, 0.0109, 0.0100,\n",
       "        0.0104, 0.0109, 0.0109, 0.0100, 0.0094, 0.0123, 0.0097, 0.0100, 0.0134,\n",
       "        0.0100, 0.0134, 0.0104, 0.0123, 0.0123, 0.0100, 0.0097, 0.0115, 0.0104,\n",
       "        0.0109, 0.0123, 0.0109, 0.0104, 0.0115, 0.0134, 0.0109, 0.0123, 0.0109,\n",
       "        0.0109, 0.0134, 0.0173, 0.0123, 0.0115, 0.0123, 0.0173, 0.0109, 0.0173,\n",
       "        0.0149, 0.0115, 0.0149, 0.0134, 0.0123, 0.0173, 0.0218, 0.0218, 0.0149,\n",
       "        0.0134, 0.0149])"
      ]
     },
     "execution_count": 433,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epsilon = 1e-6  # Small constant to prevent division by zero\n",
    "class_weights = (1.0 / torch.log(class_freq))\n",
    "class_weights /= class_weights.sum()\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 520,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************alpha: 0.125; uncertain: 0.9; debias: 1.0; msp: 0.8****************************\n",
      "ts_micro_f1=0.4903, ts_macro_f1=0.2054\n",
      "ts_micro_f1=0.4899, ts_macro_f1=0.2054\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.9; msp: 0.8****************************\n",
      "ts_micro_f1=0.4995, ts_macro_f1=0.2089\n",
      "ts_micro_f1=0.4991, ts_macro_f1=0.2089\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.8; msp: 0.8****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2091\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2091\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 1.0; msp: 0.9****************************\n",
      "ts_micro_f1=0.4903, ts_macro_f1=0.2032\n",
      "ts_micro_f1=0.4899, ts_macro_f1=0.2032\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.9; msp: 0.9****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2146\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2146\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.8; msp: 0.9****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2148\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2147\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 1.0; msp: 0.99****************************\n",
      "ts_micro_f1=0.4949, ts_macro_f1=0.2052\n",
      "ts_micro_f1=0.4945, ts_macro_f1=0.2052\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.9; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2130\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2130\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.9; debias: 0.8; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2129\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2129\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 1.0; msp: 0.8****************************\n",
      "ts_micro_f1=0.4940, ts_macro_f1=0.2204\n",
      "ts_micro_f1=0.4935, ts_macro_f1=0.2204\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.9; msp: 0.8****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2197\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2197\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.8; msp: 0.8****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2199\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2199\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 1.0; msp: 0.9****************************\n",
      "ts_micro_f1=0.4940, ts_macro_f1=0.2183\n",
      "ts_micro_f1=0.4935, ts_macro_f1=0.2183\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.9; msp: 0.9****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2251\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2251\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.8; msp: 0.9****************************\n",
      "ts_micro_f1=0.5023, ts_macro_f1=0.2253\n",
      "ts_micro_f1=0.5018, ts_macro_f1=0.2253\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 1.0; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2131\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2131\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.9; msp: 0.99****************************\n",
      "ts_micro_f1=0.4986, ts_macro_f1=0.2202\n",
      "ts_micro_f1=0.4982, ts_macro_f1=0.2202\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.125; uncertain: 0.8; debias: 0.8; msp: 0.99****************************\n",
      "ts_micro_f1=0.4986, ts_macro_f1=0.2202\n",
      "ts_micro_f1=0.4982, ts_macro_f1=0.2201\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 1.0; msp: 0.8****************************\n",
      "ts_micro_f1=0.4903, ts_macro_f1=0.2054\n",
      "ts_micro_f1=0.4899, ts_macro_f1=0.2054\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.9; msp: 0.8****************************\n",
      "ts_micro_f1=0.4995, ts_macro_f1=0.2089\n",
      "ts_micro_f1=0.4991, ts_macro_f1=0.2089\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.8; msp: 0.8****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2091\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2091\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 1.0; msp: 0.9****************************\n",
      "ts_micro_f1=0.4903, ts_macro_f1=0.2032\n",
      "ts_micro_f1=0.4899, ts_macro_f1=0.2032\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.9; msp: 0.9****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2146\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2146\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.8; msp: 0.9****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2148\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2147\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 1.0; msp: 0.99****************************\n",
      "ts_micro_f1=0.4949, ts_macro_f1=0.2052\n",
      "ts_micro_f1=0.4945, ts_macro_f1=0.2052\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.9; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2130\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2130\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.9; debias: 0.8; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2129\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2129\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 1.0; msp: 0.8****************************\n",
      "ts_micro_f1=0.4940, ts_macro_f1=0.2204\n",
      "ts_micro_f1=0.4935, ts_macro_f1=0.2204\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.9; msp: 0.8****************************\n",
      "ts_micro_f1=0.5005, ts_macro_f1=0.2197\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.2197\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.8; msp: 0.8****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2199\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2199\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 1.0; msp: 0.9****************************\n",
      "ts_micro_f1=0.4940, ts_macro_f1=0.2183\n",
      "ts_micro_f1=0.4935, ts_macro_f1=0.2183\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.9; msp: 0.9****************************\n",
      "ts_micro_f1=0.5014, ts_macro_f1=0.2251\n",
      "ts_micro_f1=0.5009, ts_macro_f1=0.2251\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.8; msp: 0.9****************************\n",
      "ts_micro_f1=0.5023, ts_macro_f1=0.2253\n",
      "ts_micro_f1=0.5018, ts_macro_f1=0.2253\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 1.0; msp: 0.99****************************\n",
      "ts_micro_f1=0.4968, ts_macro_f1=0.2131\n",
      "ts_micro_f1=0.4963, ts_macro_f1=0.2131\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.9; msp: 0.99****************************\n",
      "ts_micro_f1=0.4986, ts_macro_f1=0.2202\n",
      "ts_micro_f1=0.4982, ts_macro_f1=0.2202\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n",
      "*********************alpha: 0.5; uncertain: 0.8; debias: 0.8; msp: 0.99****************************\n",
      "ts_micro_f1=0.4986, ts_macro_f1=0.2202\n",
      "ts_micro_f1=0.4982, ts_macro_f1=0.2201\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "alpha = 0.125\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "max_col_length = 3\n",
    "for alpha in [0.125, 0.5]:\n",
    "    for uncertainty_threshold in [0.9, 0.8]:\n",
    "        for msp_threshold in [ 0.8, 0.9, 0.99]:\n",
    "            for debias_threshold in [1.0, 0.9, 0.8]:\n",
    "                print(f\"*********************alpha: {alpha}; uncertain: {uncertainty_threshold}; debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "                ft_embs_test = []\n",
    "                labels_test = []\n",
    "                logits_test = []\n",
    "                log = defaultdict(list)\n",
    "                num_cols = []\n",
    "                corrected = 0\n",
    "                total_mistakes = 0\n",
    "\n",
    "                for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                    target_col_mask = batch[\"target_col_mask\"].T\n",
    "                    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "                    logits = reweight_logits(logits.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "                    msp_init = logits.max().item()\n",
    "                    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                    predict_init = logits.argmax().item()\n",
    "                    max_msp = 0\n",
    "                    if 1 in target_col_mask and msp_init < uncertainty_threshold:\n",
    "                        col_idx_set = target_col_mask.unique().tolist()\n",
    "                        debias_classes = []\n",
    "                        assert -1 not in col_idx_set\n",
    "                        for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                            for subset in itertools.combinations(col_idx_set, r):\n",
    "                                # if 0 not in subset:\n",
    "                                #     continue\n",
    "                                for x in itertools.permutations(subset):\n",
    "                                    new_batch_data = []\n",
    "                                    for col_i in x:\n",
    "                                        if col_i == 0:\n",
    "                                            if len(new_batch_data) == 0:\n",
    "                                                cls_indexes_value = 0\n",
    "                                            else:\n",
    "                                                cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                        new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                                    if 0 not in x:\n",
    "                                        cls_indexes_value = 0\n",
    "                                    new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                    cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                    logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                    logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                    msp_temp = logits_temp.max().item()\n",
    "                                    predict_temp = logits_temp.argmax().item()\n",
    "                                    if len(x) == 1 and 0 in x:\n",
    "                                        predict_target = predict_temp\n",
    "                                        msp_target = msp_temp\n",
    "                                    # print(x, msp_temp, predict_temp)\n",
    "                                    if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                        debias_classes.append(predict_temp)\n",
    "                                        continue\n",
    "                                    if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                        max_msp = msp_temp\n",
    "                                        best_msp_perm = x\n",
    "                                        msp_predict = predict_temp\n",
    "                                        logits_msp = logits_temp.clone()\n",
    "                    if max_msp > msp_threshold:\n",
    "                        logits = logits_msp.clone()\n",
    "                    labels_test.append(batch[\"label\"].cpu())\n",
    "                    logits_test.append(logits.detach().cpu())\n",
    "                labels_test = torch.cat(labels_test, dim=0)\n",
    "                logits_test = torch.stack(logits_test, dim=0)\n",
    "                preds_test = torch.argmax(logits_test, dim=1)\n",
    "                num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "                from sklearn.metrics import confusion_matrix, f1_score\n",
    "                mask = num_cols > 0\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach().numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *********************alpha: 0.125; uncertain: 0.8; debias: 0.8; msp: 0.9****************************\n",
    "# ts_micro_f1=0.5023, ts_macro_f1=0.2253\n",
    "# ts_micro_f1=0.5018, ts_macro_f1=0.2253\n",
    "# ts_micro_f1=1.0000, ts_macro_f1=1.0000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************alpha: 0.125; uncertain: 0.8; debias: 1.0; msp: 0.9****************************\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "alpha = 0.125\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "msp_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "\n",
    "print(f\"*********************alpha: {alpha}; uncertain: {uncertainty_threshold}; debias: {debias_threshold}; msp: {msp_threshold}****************************\")\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test_init_origin = torch.stack(logits_test, dim=0)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "    logits = reweight_logits(logits_test_init_origin.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "    ood_score = logits.max(1)[0]\n",
    "    preds_test = torch.argmax(logits, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 443,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1085])"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "for alpha in [0.125, 0.25, 0.5, 1.0, 1.1, 0.00001]:\n",
    "    class_weights = (1.0 / class_freq) ** alpha\n",
    "    # Normalize the weights\n",
    "    class_weights /= class_weights.sum()\n",
    "    logits = reweight_logits(logits_test_init_origin.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "    ood_score = logits.max(1)[0]\n",
    "    preds_test = torch.argmax(logits, dim=1)\n",
    "    ood_labels = torch.tensor(preds_test == labels_test).float()\n",
    "    from sklearn.metrics import roc_auc_score\n",
    "\n",
    "    print(alpha, roc_auc_score(ood_labels, ood_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/324254938.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n",
      "/tmp/ipykernel_955267/324254938.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n",
      "/tmp/ipykernel_955267/324254938.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n",
      "/tmp/ipykernel_955267/324254938.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.125 0.8552294444670989\n",
      "0.25 0.8552643204741319\n",
      "0.5 0.8713294825216694\n",
      "1.0 0.8697046905697448\n",
      "1.1 0.8617813386815106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/324254938.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_labels = torch.tensor(preds_test == labels_test).float()\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "for alpha in [0.125, 0.25, 0.5, 1.0, 1.1]:\n",
    "    class_weights = (1.0 / class_freq) ** alpha\n",
    "    # Normalize the weights\n",
    "    class_weights /= class_weights.sum()\n",
    "    logits = reweight_logits(logits_test_init_origin.detach().cpu(), class_weights) # Here !!!!!!!!!!\n",
    "    ood_score = logits.max(1)[0]\n",
    "    preds_test = torch.argmax(logits, dim=1)\n",
    "    ood_labels = torch.tensor(preds_test == labels_test).float()\n",
    "    from sklearn.metrics import roc_auc_score\n",
    "\n",
    "    print(alpha, roc_auc_score(ood_labels, ood_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *********************uncertain: 0.8; debias: 0.8; msp: 0.99****************************\n",
    "# /tmp/ipykernel_955267/1017438977.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
    "#   msp_init = F.softmax(logits).max().item()\n",
    "# /tmp/ipykernel_955267/1017438977.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
    "#   logits_temp = F.softmax(logits_temp)\n",
    "# ts_micro_f1=0.4571, ts_macro_f1=0.1923\n",
    "# ts_micro_f1=0.4350, ts_macro_f1=0.1870\n",
    "# ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
    "\n",
    "# *********************uncertain: 0.8; debias: 1.0; msp: 0.99****************************\n",
    "# /tmp/ipykernel_955267/2474143246.py:33: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
    "#   msp_init = F.softmax(logits).max().item()\n",
    "# /tmp/ipykernel_955267/2474143246.py:61: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
    "#   logits_temp = F.softmax(logits_temp)\n",
    "# ts_micro_f1=0.4562, ts_macro_f1=0.1914\n",
    "# ts_micro_f1=0.4333, ts_macro_f1=0.1849\n",
    "# ts_micro_f1=0.4823, ts_macro_f1=0.1573\n",
    "\n",
    "\n",
    "# *********************alpha: 0.125; uncertain: 0.8; debias: 0.9; msp: 0.99****************************\n",
    "# ts_micro_f1=0.4590, ts_macro_f1=0.1915\n",
    "# ts_micro_f1=0.4350, ts_macro_f1=0.1807\n",
    "# ts_micro_f1=0.4862, ts_macro_f1=0.1626"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncertain init (0.9)\n",
    "# *********************debias: 0.9; msp: 0.95****************************\n",
    "# ts_micro_f1=0.4571, ts_macro_f1=0.1903\n",
    "# ts_micro_f1=0.4350, ts_macro_f1=0.1862\n",
    "# ts_micro_f1=0.4823, ts_macro_f1=0.1573"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/2547658102.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_init = F.softmax(logits.detach()).max().item()\n",
      "/tmp/ipykernel_955267/2547658102.py:45: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_target = F.softmax(logits_temp.detach()).max().item()\n",
      "/tmp/ipykernel_955267/2547658102.py:70: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  ood_score_temp =F.softmax(logits_temp.detach()).max().item()\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate, find uncertain init prediction\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "idx_list = []\n",
    "uncertain_init_mask = []\n",
    "uncertain_target_mask = []\n",
    "correct_init_mask = []\n",
    "correct_target_mask = []\n",
    "correct_permutation_mask = []\n",
    "correct_msp_mask = []\n",
    "init_target_mask = []\n",
    "ood_score_init_list = []\n",
    "ood_score_target_list = []\n",
    "\n",
    "\n",
    "alpha = 1.0\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "for threshold in [0.5]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        if 1 in target_col_mask:\n",
    "            idx_list.append(batch_idx)\n",
    "\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "            num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "            ood_score_init = F.softmax(logits.detach()).max().item()\n",
    "            ood_score_init_list.append(ood_score_init)\n",
    "            uncertain_init_mask.append(ood_score_init < threshold)\n",
    "            predict_init = logits.argmax().item()\n",
    "            correct_init_mask.append(predict_init == batch[\"label\"].item())\n",
    "            \n",
    "            cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)    \n",
    "            ood_score_target = F.softmax(logits_temp.detach()).max().item()\n",
    "            ood_score_target_list.append(ood_score_target)\n",
    "            uncertain_target_mask.append(ood_score_target < threshold)\n",
    "            predict_target = logits_temp.argmax().item()\n",
    "            correct_target_mask.append(predict_target == batch[\"label\"].item())\n",
    "            \n",
    "            init_target_mask.append(predict_target == predict_init)\n",
    "            \n",
    "            correct_permutation = False\n",
    "            ood_score_max = 0\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        if 0 not in x:\n",
    "                            continue\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        ood_score_temp =F.softmax(logits_temp.detach()).max().item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        if ood_score_temp > ood_score_max:\n",
    "                            ood_score_max = ood_score_temp \n",
    "                            predict_ood = logits_temp.argmax().item()\n",
    "                        if predict_temp == batch[\"label\"].item():\n",
    "                            correct_permutation = True\n",
    "            correct_permutation_mask.append(correct_permutation)\n",
    "            correct_msp_mask.append(predict_ood == batch[\"label\"].item())\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate, find uncertain init prediction, weighted msp\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "idx_list = []\n",
    "uncertain_init_mask = []\n",
    "uncertain_target_mask = []\n",
    "correct_init_mask = []\n",
    "correct_target_mask = []\n",
    "correct_permutation_mask = []\n",
    "correct_msp_mask = []\n",
    "init_target_mask = []\n",
    "ood_score_init_list = []\n",
    "ood_score_target_list = []\n",
    "ood_score_final_list = []\n",
    "\n",
    "alpha = 1.0\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "debias_threshold = 0.9\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "for threshold in [0.5]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        if 1 in target_col_mask:\n",
    "            idx_list.append(batch_idx)\n",
    "\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "            logits = reweight_logits(logits.detach().cpu(), class_weights)\n",
    "            num_cols.append(batch[\"target_col_mask\"].max().item())         \n",
    "            ood_score_init = logits.max().item()\n",
    "            ood_score_init_list.append(ood_score_init)\n",
    "            uncertain_init_mask.append(ood_score_init < threshold)\n",
    "            predict_init = logits.argmax().item()\n",
    "            correct_init_mask.append(predict_init == batch[\"label\"].item())\n",
    "            \n",
    "            cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)  \n",
    "            logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "            ood_score_target = logits_temp.max().item()\n",
    "            ood_score_target_list.append(ood_score_target)\n",
    "            uncertain_target_mask.append(ood_score_target < threshold)\n",
    "            predict_target = logits_temp.argmax().item()\n",
    "            correct_target_mask.append(predict_target == batch[\"label\"].item())\n",
    "            \n",
    "\n",
    "            init_target_mask.append(predict_target == predict_init)\n",
    "            \n",
    "            correct_permutation = False\n",
    "            ood_score_max = max_msp = 0\n",
    "            max_msp_debiased = 0\n",
    "            debias_class = []\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, len(col_idx_set) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    # if 0 not in subset:\n",
    "                    #     continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                        msp_temp = logits_temp.max().item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        if 0 not in x and msp_temp > debias_threshold and (predict_temp!=predict_target or predict_temp!=predict_init):\n",
    "                            debias_class.append(predict_temp)\n",
    "                            continue\n",
    "                        # print(x, msp_temp, predict_temp)\n",
    "                        if msp_temp > max_msp and 0 in x and predict_temp not in debias_class:\n",
    "                            max_msp = msp_temp\n",
    "                            msp_predict = predict_temp\n",
    "                        if predict_temp == batch[\"label\"].item():\n",
    "                            correct_permutation = True\n",
    "            ood_score_final_list.append(max_msp)\n",
    "            correct_permutation_mask.append(correct_permutation)\n",
    "            correct_msp_mask.append(msp_predict == batch[\"label\"].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "577 577 577 577 577 577 577 577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/1309743969.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  idx_list = torch.tensor(idx_list)\n",
      "/tmp/ipykernel_955267/1309743969.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  uncertain_init_mask = torch.tensor(uncertain_init_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  uncertain_target_mask = torch.tensor(uncertain_target_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_init_mask = torch.tensor(correct_init_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:5: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_target_mask = torch.tensor(correct_target_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_permutation_mask = torch.tensor(correct_permutation_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  correct_msp_mask = torch.tensor(correct_msp_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  init_target_mask = torch.tensor(init_target_mask)\n",
      "/tmp/ipykernel_955267/1309743969.py:9: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_init_list = torch.tensor(ood_score_init_list)\n",
      "/tmp/ipykernel_955267/1309743969.py:10: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_target_list = torch.tensor(ood_score_target_list)\n",
      "/tmp/ipykernel_955267/1309743969.py:11: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  ood_score_final_list = torch.tensor(ood_score_final_list)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "idx_list = torch.tensor(idx_list)\n",
    "uncertain_init_mask = torch.tensor(uncertain_init_mask)\n",
    "uncertain_target_mask = torch.tensor(uncertain_target_mask)\n",
    "correct_init_mask = torch.tensor(correct_init_mask)\n",
    "correct_target_mask = torch.tensor(correct_target_mask)\n",
    "correct_permutation_mask = torch.tensor(correct_permutation_mask)\n",
    "correct_msp_mask = torch.tensor(correct_msp_mask)\n",
    "init_target_mask = torch.tensor(init_target_mask)\n",
    "ood_score_init_list = torch.tensor(ood_score_init_list)\n",
    "ood_score_target_list = torch.tensor(ood_score_target_list)\n",
    "ood_score_final_list = torch.tensor(ood_score_final_list)\n",
    "print(len(idx_list), len(uncertain_init_mask), len(uncertain_target_mask), len(init_target_mask), len(correct_init_mask), len(correct_target_mask), len(correct_permutation_mask), len(correct_msp_mask))\n",
    "\n",
    "\n",
    "# torch.save({\"idx_list\": idx_list, \"uncertain_init_mask\": uncertain_init_mask, \"uncertain_target_mask\": uncertain_target_mask, \"correct_init_mask\": correct_init_mask, \"correct_target_mask\": correct_target_mask, \"correct_permutation_mask\": correct_permutation_mask, \"correct_msp_mask\": correct_msp_mask, \"init_target_mask\": init_target_mask, \"ood_score_init_list\": ood_score_init_list, \"ood_score_target_list\": ood_score_target_list}, \"./results/brute_force_permutation.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.8****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3443360/3556820158.py:46: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits = F.softmax(logits)\n",
      "/tmp/ipykernel_3443360/3556820158.py:57: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n",
      "/tmp/ipykernel_3443360/3556820158.py:94: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  logits_temp = F.softmax(logits_temp)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [58]\u001b[0m, in \u001b[0;36m<cell line: 28>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     90\u001b[0m new_batch_data \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat(new_batch_data, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     91\u001b[0m cls_indexes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor([\u001b[38;5;241m0\u001b[39m, cls_indexes_value])\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 92\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_batch_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcls_indexes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     93\u001b[0m \u001b[38;5;66;03m# logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\u001b[39;00m\n\u001b[1;32m     94\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39msoftmax(logits_temp)\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:571\u001b[0m, in \u001b[0;36mBertForMultiOutputClassification.forward\u001b[0;34m(self, input_ids, get_enc, cls_indexes, token_type_ids)\u001b[0m\n\u001b[1;32m    569\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    570\u001b[0m \u001b[38;5;66;03m# BertModelMultiOutput\u001b[39;00m\n\u001b[0;32m--> 571\u001b[0m bert_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbert\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m    572\u001b[0m \u001b[43m                        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    573\u001b[0m table_length \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mlen\u001b[39m(input_ids[i]\u001b[38;5;241m.\u001b[39mnonzero()) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(input_ids))]\n\u001b[1;32m    574\u001b[0m \u001b[38;5;66;03m# Note: returned tensor contains pooled_output of all tokens (to make the tensor size consistent)\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:1141\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1134\u001b[0m \u001b[38;5;66;03m# Prepare head mask if needed\u001b[39;00m\n\u001b[1;32m   1135\u001b[0m \u001b[38;5;66;03m# 1.0 in head_mask indicate we keep the head\u001b[39;00m\n\u001b[1;32m   1136\u001b[0m \u001b[38;5;66;03m# attention_probs has shape bsz x n_heads x N x N\u001b[39;00m\n\u001b[1;32m   1137\u001b[0m \u001b[38;5;66;03m# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]\u001b[39;00m\n\u001b[1;32m   1138\u001b[0m \u001b[38;5;66;03m# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]\u001b[39;00m\n\u001b[1;32m   1139\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m-> 1141\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1142\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1143\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1144\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1145\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1146\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1147\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1148\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1149\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1150\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1151\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1152\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1153\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1154\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:694\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    683\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    684\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    685\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    691\u001b[0m         output_attentions,\n\u001b[1;32m    692\u001b[0m     )\n\u001b[1;32m    693\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 694\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    695\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    696\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    697\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    698\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    699\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    700\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    701\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    702\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    704\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    705\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:626\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    623\u001b[0m     cross_attn_present_key_value \u001b[38;5;241m=\u001b[39m cross_attention_outputs[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m    624\u001b[0m     present_key_value \u001b[38;5;241m=\u001b[39m present_key_value \u001b[38;5;241m+\u001b[39m cross_attn_present_key_value\n\u001b[0;32m--> 626\u001b[0m layer_output \u001b[38;5;241m=\u001b[39m \u001b[43mapply_chunking_to_forward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    627\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeed_forward_chunk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchunk_size_feed_forward\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseq_len_dim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_output\u001b[49m\n\u001b[1;32m    628\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    629\u001b[0m outputs \u001b[38;5;241m=\u001b[39m (layer_output,) \u001b[38;5;241m+\u001b[39m outputs\n\u001b[1;32m    631\u001b[0m \u001b[38;5;66;03m# if decoder, return the attn key/values as the last output\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/pytorch_utils.py:238\u001b[0m, in \u001b[0;36mapply_chunking_to_forward\u001b[0;34m(forward_fn, chunk_size, chunk_dim, *input_tensors)\u001b[0m\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;66;03m# concatenate output at same dimension\u001b[39;00m\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat(output_chunks, dim\u001b[38;5;241m=\u001b[39mchunk_dim)\n\u001b[0;32m--> 238\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minput_tensors\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:639\u001b[0m, in \u001b[0;36mBertLayer.feed_forward_chunk\u001b[0;34m(self, attention_output)\u001b[0m\n\u001b[1;32m    637\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfeed_forward_chunk\u001b[39m(\u001b[38;5;28mself\u001b[39m, attention_output):\n\u001b[1;32m    638\u001b[0m     intermediate_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mintermediate(attention_output)\n\u001b[0;32m--> 639\u001b[0m     layer_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput\u001b[49m\u001b[43m(\u001b[49m\u001b[43mintermediate_output\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_output\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    640\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m layer_output\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:553\u001b[0m, in \u001b[0;36mBertOutput.forward\u001b[0;34m(self, hidden_states, input_tensor)\u001b[0m\n\u001b[1;32m    551\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdense(hidden_states)\n\u001b[1;32m    552\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(hidden_states)\n\u001b[0;32m--> 553\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLayerNorm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43minput_tensor\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    554\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m hidden_states\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/normalization.py:202\u001b[0m, in \u001b[0;36mLayerNorm.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    201\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 202\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    203\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meps\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/functional.py:2576\u001b[0m, in \u001b[0;36mlayer_norm\u001b[0;34m(input, normalized_shape, weight, bias, eps)\u001b[0m\n\u001b[1;32m   2572\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_variadic(\u001b[38;5;28minput\u001b[39m, weight, bias):\n\u001b[1;32m   2573\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m   2574\u001b[0m         layer_norm, (\u001b[38;5;28minput\u001b[39m, weight, bias), \u001b[38;5;28minput\u001b[39m, normalized_shape, weight\u001b[38;5;241m=\u001b[39mweight, bias\u001b[38;5;241m=\u001b[39mbias, eps\u001b[38;5;241m=\u001b[39meps\n\u001b[1;32m   2575\u001b[0m     )\n\u001b[0;32m-> 2576\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackends\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcudnn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menabled\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "# brute force perumate, all columns\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "change_log = []\n",
    "score_init = []\n",
    "score_best = [] \n",
    "idx_list = []\n",
    "uncertain_init_mask = []\n",
    "uncertain_target_mask = []\n",
    "correct_init_mask = []\n",
    "correct_target_mask = []\n",
    "correct_permutation_mask = []\n",
    "correct_permutation_confident_mask = []\n",
    "correct_msp_mask = []\n",
    "init_target_mask = []\n",
    "ood_score_init_list = []\n",
    "ood_score_target_list = []\n",
    "ood_score_final_list = []\n",
    "\n",
    "max_col_length = 3\n",
    "\n",
    "\n",
    "for threshold in [0.8]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    corrected = 0\n",
    "    total_mistakes = 0\n",
    "\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter_all):\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        if 1 in target_col_mask:\n",
    "            idx_list.append(batch_idx)\n",
    "\n",
    "            cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "            logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "            # logits = reweight_logits(logits.detach().cpu(), class_weights)\n",
    "            logits = F.softmax(logits)\n",
    "            num_cols.append(batch[\"target_col_mask\"].max().item())         \n",
    "            ood_score_init = logits.max().item()\n",
    "            ood_score_init_list.append(ood_score_init)\n",
    "            uncertain_init_mask.append(ood_score_init < threshold)\n",
    "            predict_init = logits.argmax().item()\n",
    "            correct_init_mask.append(predict_init == batch[\"label\"].item())\n",
    "            \n",
    "            cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes,)  \n",
    "            # logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "            logits_temp = F.softmax(logits_temp)\n",
    "            ood_score_target = logits_temp.max().item()\n",
    "            ood_score_target_list.append(ood_score_target)\n",
    "            uncertain_target_mask.append(ood_score_target < threshold)\n",
    "            predict_target = logits_temp.argmax().item()\n",
    "            correct_target_mask.append(predict_target == batch[\"label\"].item())\n",
    "            \n",
    "\n",
    "            init_target_mask.append(predict_target == predict_init)\n",
    "            \n",
    "            correct_permutation = False\n",
    "            correct_permutation_confident = False\n",
    "            ood_score_max = max_msp = 0\n",
    "            max_msp_debiased = 0\n",
    "            debias_class = []\n",
    "            col_idx_set = target_col_mask.unique().tolist()\n",
    "            max_permutation_confident_score = 0\n",
    "            assert -1 not in col_idx_set\n",
    "            for r in range(1, min(len(col_idx_set), max_col_length) + 1):\n",
    "                for subset in itertools.combinations(col_idx_set, r):\n",
    "                    # if 0 not in subset:\n",
    "                    #     continue\n",
    "                    for x in itertools.permutations(subset):\n",
    "                        new_batch_data = []\n",
    "                        for col_i in x:\n",
    "                            if col_i == 0:\n",
    "                                if len(new_batch_data) == 0:\n",
    "                                    cls_indexes_value = 0\n",
    "                                else:\n",
    "                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                            new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                        if 0 not in x:\n",
    "                            cls_indexes_value = 0\n",
    "                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        # logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                        logits_temp = F.softmax(logits_temp)\n",
    "                        msp_temp = logits_temp.max().item()\n",
    "                        predict_temp = logits_temp.argmax().item()\n",
    "                        # if 0 not in x and msp_temp > debias_threshold and (predict_temp!=predict_target or predict_temp!=predict_init):\n",
    "                        #     debias_class.append(predict_temp)\n",
    "                        #     continue\n",
    "                        # print(x, msp_temp, predict_temp)\n",
    "                        # if msp_temp > max_msp and 0 in x and predict_temp not in debias_class:\n",
    "                        if msp_temp > max_msp and 0 in x:\n",
    "                            max_msp = msp_temp\n",
    "                            msp_predict = predict_temp\n",
    "                        if predict_temp == batch[\"label\"].item():\n",
    "                            correct_permutation = True\n",
    "                            if msp_temp > max_permutation_confident_score:\n",
    "                                max_permutation_confident_score = msp_temp\n",
    "            ood_score_final_list.append(max_msp)\n",
    "            correct_permutation_mask.append(correct_permutation)\n",
    "            correct_permutation_confident_mask.append(max_permutation_confident_score)\n",
    "            correct_msp_mask.append(msp_predict == batch[\"label\"].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "577"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ood_score_final_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_mask = (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(484) tensor(0.3326)\n"
     ]
    }
   ],
   "source": [
    "print(sum(prefix_mask), correct_msp_mask[prefix_mask].sum()/sum(prefix_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(76) tensor(31) tensor(0.4079)\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.95\n",
    "print(sum(prefix_mask&(ood_score_final_list>threshold)), correct_msp_mask[prefix_mask&(ood_score_final_list>threshold)].sum(), correct_msp_mask[prefix_mask&(ood_score_final_list>threshold)].sum()/sum(prefix_mask&(ood_score_final_list>threshold)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(76) tensor(34) tensor(0.4474)\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.95\n",
    "print(sum(prefix_mask&(ood_score_final_list>threshold)), correct_init_mask[prefix_mask&(ood_score_final_list>threshold)].sum(), correct_init_mask[prefix_mask&(ood_score_final_list>threshold)].sum()/sum(prefix_mask&(ood_score_final_list>threshold)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(88)"
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(correct_init_mask[prefix_mask&(ood_score_final_list>threshold)]&correct_init_mask[prefix_mask&(ood_score_final_list>threshold)]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(68)"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(correct_init_mask[prefix_mask&(ood_score_final_list>threshold)]&correct_msp_mask[prefix_mask&(ood_score_final_list>threshold)]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(108)"
      ]
     },
     "execution_count": 358,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(correct_init_mask[prefix_mask&(ood_score_final_list>threshold)]|correct_msp_mask[prefix_mask&(ood_score_final_list>threshold)]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "msp_new_idx = []\n",
    "for i in idx_list[correct_msp_mask&prefix_mask&(ood_score_final_list>threshold)]:\n",
    "    if i not in idx_list[correct_init_mask&prefix_mask&(ood_score_final_list>threshold)]:\n",
    "        msp_new_idx.append(i)\n",
    "msp_new_idx = torch.tensor(msp_new_idx)\n",
    "init_new_idx = []\n",
    "for i in idx_list[correct_init_mask&prefix_mask&(ood_score_final_list>threshold)]:\n",
    "    if i not in idx_list[correct_msp_mask&prefix_mask&(ood_score_final_list>threshold)]:\n",
    "        init_new_idx.append(i)\n",
    "init_new_idx = torch.tensor(init_new_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_target_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    }
   ],
   "source": [
    "target_id_list_pos = idx_list[(uncertain_init_mask & uncertain_target_mask) & correct_permutation_mask & correct_msp_mask ]\n",
    "print(len(target_id_list_pos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    }
   ],
   "source": [
    "target_id_list_neg = idx_list[(uncertain_init_mask & uncertain_target_mask) & correct_permutation_mask & ~correct_msp_mask ]\n",
    "print(len(target_id_list_neg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "577 203 322\n"
     ]
    }
   ],
   "source": [
    "print(len(uncertain_init_mask), len(idx_list[(uncertain_init_mask | uncertain_target_mask)]), correct_permutation_mask.sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "239 0.41421143847487 244 0.42287694974003465 229 0.3968804159445407\n"
     ]
    }
   ],
   "source": [
    "print(correct_msp_mask.sum().item(), correct_msp_mask.sum().item()/len(correct_msp_mask), \n",
    "      correct_init_mask.sum().item(), correct_init_mask.sum().item()/len(correct_init_mask), \n",
    "      correct_target_mask.sum().item(), correct_target_mask.sum().item()/len(correct_target_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.41421143847487 0.42287694974003465 0.3968804159445407\n"
     ]
    }
   ],
   "source": [
    "print(correct_msp_mask.sum().item()/len(correct_msp_mask), correct_init_mask.sum().item()/len(correct_msp_mask), correct_target_mask.sum().item()/len(correct_msp_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193 338 0.5710059171597633\n"
     ]
    }
   ],
   "source": [
    "print((init_target_mask&correct_init_mask).sum().item(), init_target_mask.sum().item(), (init_target_mask&correct_init_mask).sum().item()/init_target_mask.sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init\n",
      "175 282 0.6205673758865248\n",
      "69 295 0.23389830508474577\n",
      "MSP\n",
      "180 282 0.6382978723404256\n",
      "59 295 0.2\n"
     ]
    }
   ],
   "source": [
    "print(\"Init\")\n",
    "# init prediction\n",
    "print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask)).sum().item(), \n",
    "      (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "      (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "# init prediction\n",
    "print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "      (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "print(\"MSP\")\n",
    "# msp prediction\n",
    "print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask)).sum().item(), \n",
    "      (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "      (init_target_mask&correct_msp_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "# msp prediction\n",
    "print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "      (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "577 577 577 319\n"
     ]
    }
   ],
   "source": [
    "print(len(uncertain_init_mask), len(uncertain_target_mask), len(correct_init_mask), len(init_target_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(93)"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "140 209 0.6698564593301436\n",
      "94 368 0.2554347826086957\n",
      "MSP\n",
      "136 209 0.6507177033492823\n",
      "100 368 0.2717391304347826\n",
      "*********************Threshold: 0.6****************************\n",
      "Init\n",
      "127 182 0.6978021978021978\n",
      "107 395 0.2708860759493671\n",
      "MSP\n",
      "123 182 0.6758241758241759\n",
      "113 395 0.28607594936708863\n",
      "*********************Threshold: 0.7****************************\n",
      "Init\n",
      "120 167 0.718562874251497\n",
      "114 410 0.2780487804878049\n",
      "MSP\n",
      "117 167 0.7005988023952096\n",
      "119 410 0.29024390243902437\n",
      "*********************Threshold: 0.8****************************\n",
      "Init\n",
      "103 139 0.7410071942446043\n",
      "131 438 0.2990867579908676\n",
      "MSP\n",
      "101 139 0.7266187050359713\n",
      "135 438 0.3082191780821918\n",
      "*********************Threshold: 0.9****************************\n",
      "Init\n",
      "76 93 0.8172043010752689\n",
      "158 484 0.32644628099173556\n",
      "MSP\n",
      "75 93 0.8064516129032258\n",
      "161 484 0.33264462809917356\n"
     ]
    }
   ],
   "source": [
    "# Weighted\n",
    "for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:\n",
    "    uncertain_init_mask = ood_score_init_list < threshold\n",
    "    uncertain_target_mask = ood_score_target_list < threshold\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # init prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_msp_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "175 282 0.6205673758865248\n",
      "69 295 0.23389830508474577\n",
      "MSP\n",
      "180 282 0.6382978723404256\n",
      "59 295 0.2\n",
      "*********************Threshold: 0.6****************************\n",
      "Init\n",
      "169 266 0.6353383458646616\n",
      "75 311 0.24115755627009647\n",
      "MSP\n",
      "173 266 0.650375939849624\n",
      "66 311 0.21221864951768488\n",
      "*********************Threshold: 0.7****************************\n",
      "Init\n",
      "156 236 0.6610169491525424\n",
      "88 341 0.25806451612903225\n",
      "MSP\n",
      "159 236 0.673728813559322\n",
      "80 341 0.23460410557184752\n",
      "*********************Threshold: 0.8****************************\n",
      "Init\n",
      "152 219 0.6940639269406392\n",
      "92 358 0.2569832402234637\n",
      "MSP\n",
      "153 219 0.6986301369863014\n",
      "86 358 0.24022346368715083\n",
      "*********************Threshold: 0.9****************************\n",
      "Init\n",
      "139 191 0.7277486910994765\n",
      "105 386 0.27202072538860106\n",
      "MSP\n",
      "140 191 0.7329842931937173\n",
      "99 386 0.25647668393782386\n"
     ]
    }
   ],
   "source": [
    "for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:\n",
    "    uncertain_init_mask = ood_score_init_list < threshold\n",
    "    uncertain_target_mask = ood_score_target_list < threshold\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # init prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_msp_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item(), (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "175 282 0.6205673758865248\n",
      "69 295 0.23389830508474577\n",
      "MSP\n",
      "180 282 0.6382978723404256\n",
      "59 295 0.2\n",
      "MSP & Init\n",
      "175\n",
      "29\n",
      "99 0.33559322033898303\n",
      "Permutation\n",
      "194 282 0.6879432624113475\n",
      "128 295 0.43389830508474575\n",
      "*********************Threshold: 0.8****************************\n",
      "Init\n",
      "152 219 0.6940639269406392\n",
      "92 358 0.2569832402234637\n",
      "MSP\n",
      "153 219 0.6986301369863014\n",
      "86 358 0.24022346368715083\n",
      "MSP & Init\n",
      "152\n",
      "52\n",
      "126 0.35195530726256985\n",
      "Permutation\n",
      "162 219 0.7397260273972602\n",
      "160 358 0.44692737430167595\n",
      "*********************Threshold: 0.9****************************\n",
      "Init\n",
      "139 191 0.7277486910994765\n",
      "105 386 0.27202072538860106\n",
      "MSP\n",
      "140 191 0.7329842931937173\n",
      "99 386 0.25647668393782386\n",
      "MSP & Init\n",
      "139\n",
      "65\n",
      "139 0.3601036269430052\n",
      "Permutation\n",
      "147 191 0.7696335078534031\n",
      "175 386 0.4533678756476684\n",
      "*********************Threshold: 0.95****************************\n",
      "Init\n",
      "118 159 0.7421383647798742\n",
      "126 418 0.3014354066985646\n",
      "MSP\n",
      "119 159 0.7484276729559748\n",
      "120 418 0.28708133971291866\n",
      "MSP & Init\n",
      "118\n",
      "86\n",
      "160 0.3827751196172249\n",
      "Permutation\n",
      "123 159 0.7735849056603774\n",
      "199 418 0.47607655502392343\n",
      "*********************Threshold: 0.99****************************\n",
      "Init\n",
      "71 93 0.7634408602150538\n",
      "173 484 0.3574380165289256\n",
      "MSP\n",
      "71 93 0.7634408602150538\n",
      "168 484 0.34710743801652894\n",
      "MSP & Init\n",
      "71\n",
      "133\n",
      "208 0.4297520661157025\n",
      "Permutation\n",
      "73 93 0.7849462365591398\n",
      "249 484 0.5144628099173554\n"
     ]
    }
   ],
   "source": [
    "for threshold in [0.5, 0.8, 0.9, 0.95, 0.99]:\n",
    "    uncertain_init_mask = ood_score_init_list < threshold\n",
    "    uncertain_target_mask = ood_score_target_list < threshold\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # init prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_msp_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP & Init\")\n",
    "    print(((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask))&(init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask))).sum().item())\n",
    "    print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item())\n",
    "    print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)|(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item(),\n",
    "          ((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)|(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"Permutation\")\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_permutation_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_permutation_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_permutation_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_permutation_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "140 209 0.6698564593301436\n",
      "94 368 0.2554347826086957\n",
      "MSP\n",
      "136 209 0.6507177033492823\n",
      "100 368 0.2717391304347826\n",
      "MSP & Init\n",
      "135\n",
      "62\n",
      "132 0.358695652173913\n",
      "Permutation\n",
      "150 209 0.7177033492822966\n",
      "161 368 0.4375\n",
      "*********************Threshold: 0.8****************************\n",
      "Init\n",
      "103 139 0.7410071942446043\n",
      "131 438 0.2990867579908676\n",
      "MSP\n",
      "101 139 0.7266187050359713\n",
      "135 438 0.3082191780821918\n",
      "MSP & Init\n",
      "101\n",
      "96\n",
      "170 0.3881278538812785\n",
      "Permutation\n",
      "108 139 0.7769784172661871\n",
      "203 438 0.4634703196347032\n",
      "*********************Threshold: 0.9****************************\n",
      "Init\n",
      "76 93 0.8172043010752689\n",
      "158 484 0.32644628099173556\n",
      "MSP\n",
      "75 93 0.8064516129032258\n",
      "161 484 0.33264462809917356\n",
      "MSP & Init\n",
      "75\n",
      "122\n",
      "197 0.40702479338842973\n",
      "Permutation\n",
      "77 93 0.8279569892473119\n",
      "234 484 0.4834710743801653\n"
     ]
    }
   ],
   "source": [
    "# weighted\n",
    "for threshold in [0.5, 0.8, 0.9]:\n",
    "    uncertain_init_mask = ood_score_init_list < threshold\n",
    "    uncertain_target_mask = ood_score_target_list < threshold\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # init prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_msp_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"MSP & Init\")\n",
    "    print(((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_init_mask))&(init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_msp_mask))).sum().item())\n",
    "    print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item())\n",
    "    print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)|(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item(),\n",
    "          ((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)|(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())\n",
    "    print(\"Permutation\")\n",
    "    print((init_target_mask&((~uncertain_init_mask & ~uncertain_target_mask)&correct_permutation_mask)).sum().item(), \n",
    "        (init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item(), \n",
    "        (init_target_mask&correct_permutation_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item())\n",
    "    # msp prediction\n",
    "    print((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_permutation_mask).sum().item(), \n",
    "          (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item(), \n",
    "        (~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_permutation_mask).sum().item()/(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149"
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(condition_mask).sum().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n",
      "Init\n",
      "189 317 0.5962145110410094\n",
      "45 260 0.17307692307692307\n",
      "MSP\n",
      "185 317 0.583596214511041\n",
      "51 260 0.19615384615384615\n",
      "MSP & Init\n",
      "173\n",
      "201 317 0.6340694006309149\n",
      "Permutation\n",
      "222 317 0.7003154574132492\n",
      "89 260 0.3423076923076923\n",
      "*********************Threshold: 0.8****************************\n",
      "Init\n",
      "148 213 0.6948356807511737\n",
      "86 364 0.23626373626373626\n",
      "MSP\n",
      "143 213 0.6713615023474179\n",
      "93 364 0.2554945054945055\n",
      "MSP & Init\n",
      "140\n",
      "151 213 0.7089201877934272\n",
      "Permutation\n",
      "164 213 0.7699530516431925\n",
      "147 364 0.40384615384615385\n",
      "*********************Threshold: 0.9****************************\n",
      "Init\n",
      "115 149 0.7718120805369127\n",
      "119 428 0.2780373831775701\n",
      "MSP\n",
      "111 149 0.7449664429530202\n",
      "125 428 0.29205607476635514\n",
      "MSP & Init\n",
      "110\n",
      "116 149 0.7785234899328859\n",
      "Permutation\n",
      "122 149 0.8187919463087249\n",
      "189 428 0.441588785046729\n"
     ]
    }
   ],
   "source": [
    "# weighted, only uncertain init\n",
    "\n",
    "for threshold in [0.5, 0.8, 0.9]:\n",
    "    uncertain_init_mask = ood_score_init_list < threshold\n",
    "    uncertain_target_mask = ood_score_target_list < threshold\n",
    "    condition_mask = ~uncertain_init_mask\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    print(\"Init\")\n",
    "    # init prediction\n",
    "    print((condition_mask&correct_init_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_init_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # init prediction\n",
    "    print((~condition_mask&correct_init_mask).sum().item(), \n",
    "        (~condition_mask).sum().item(), \n",
    "       (~condition_mask&correct_init_mask).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"MSP\")\n",
    "    # msp prediction\n",
    "    print((condition_mask&correct_msp_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_msp_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # msp prediction\n",
    "    print((~condition_mask&correct_msp_mask).sum().item(), \n",
    "        (~condition_mask).sum().item(), \n",
    "        (~condition_mask&correct_msp_mask).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"MSP & Init\")\n",
    "    print(((condition_mask&correct_init_mask)&(condition_mask&correct_msp_mask)).sum().item())\n",
    "    # print(((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask)).sum().item())\n",
    "    print(((condition_mask&correct_init_mask)|(condition_mask&correct_msp_mask)).sum().item(),\n",
    "          (condition_mask).sum().item(), \n",
    "          ((condition_mask&correct_init_mask)|(condition_mask&correct_msp_mask)).sum().item()/(condition_mask).sum().item())\n",
    "    print(((~condition_mask&correct_init_mask)|(~condition_mask&correct_msp_mask)).sum().item(),\n",
    "        (~condition_mask).sum().item(), \n",
    "        ((~condition_mask&correct_init_mask)|(~condition_mask&correct_msp_mask)).sum().item()/(~condition_mask).sum().item())\n",
    "    print(\"Permutation\")\n",
    "    print((condition_mask&correct_permutation_mask).sum().item(), \n",
    "        (condition_mask).sum().item(), \n",
    "        (condition_mask&correct_permutation_mask).sum().item()/(condition_mask).sum().item())\n",
    "    # msp prediction\n",
    "    print((~condition_mask&correct_permutation_mask).sum().item(), \n",
    "         (~condition_mask).sum().item(), \n",
    "        (~condition_mask&correct_permutation_mask).sum().item()/(~condition_mask).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n"
     ]
    }
   ],
   "source": [
    "# MSP is wrong, but init is correct\n",
    "target_col_idx_msp = idx_list[((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& ~correct_msp_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_init_mask))]\n",
    "print(len(target_col_idx_msp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    }
   ],
   "source": [
    "# MSP is wrong, but init is correct\n",
    "target_col_idx_init = idx_list[((~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& correct_msp_mask)&(~(init_target_mask&(~uncertain_init_mask & ~uncertain_target_mask))& ~correct_init_mask))]\n",
    "print(len(target_col_idx_init))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  45,  119,  124,  125,  128,  200,  218,  239,  260,  302,  321,  346,\n",
       "         355,  358,  392,  401,  415,  419,  439,  460,  569,  602,  622,  698,\n",
       "         718,  723,  772,  809,  867,  890,  922,  937,  949,  976,  989,  994,\n",
       "         995,  997, 1037, 1049])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_col_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "191 0.546875\n"
     ]
    }
   ],
   "source": [
    "print((init_target_mask&correct_init_mask&(~uncertain_init_mask | ~uncertain_target_mask)).sum().item(), (init_target_mask&correct_init_mask&(~uncertain_init_mask & ~uncertain_target_mask)).sum().item()/(init_target_mask&(~uncertain_init_mask | ~uncertain_target_mask)).sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164 0.28422876949740034 26 138 0.15853658536585366 0.10655737704918032\n"
     ]
    }
   ],
   "source": [
    "print(uncertain_init_mask.sum().item(), uncertain_init_mask.sum().item()/len(uncertain_init_mask), (correct_init_mask&uncertain_init_mask).sum().item(), uncertain_init_mask.sum().item()-(correct_init_mask&uncertain_init_mask).sum().item(), (correct_init_mask&uncertain_init_mask).sum().item()/uncertain_init_mask.sum().item(), (correct_init_mask&uncertain_init_mask).sum().item()/correct_init_mask.sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "413 0.5278450363196125 -195\n"
     ]
    }
   ],
   "source": [
    "print((~uncertain_init_mask).sum().item(), (correct_init_mask&(~uncertain_init_mask)).sum().item()/(~uncertain_init_mask).sum().item(), (correct_init_mask&(~uncertain_init_mask)).sum().item()-(~uncertain_init_mask).sum().item(), )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90 0.1559792027729636 16 0.17777777777777778 0.06986899563318777\n"
     ]
    }
   ],
   "source": [
    "print(uncertain_target_mask.sum().item(), uncertain_target_mask.sum().item()/len(uncertain_target_mask), (correct_target_mask&uncertain_target_mask).sum().item(), (correct_target_mask&uncertain_target_mask).sum().item()/uncertain_target_mask.sum().item(), (correct_target_mask&uncertain_target_mask).sum().item()/correct_target_mask.sum().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72\n"
     ]
    }
   ],
   "source": [
    "print(len(idx_list[(uncertain_init_mask | uncertain_target_mask) & correct_permutation_mask]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\n"
     ]
    }
   ],
   "source": [
    "print(len(idx_list[(uncertain_init_mask | uncertain_target_mask) & correct_permutation_mask & ~correct_msp_mask ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    }
   ],
   "source": [
    "print(len(idx_list[(uncertain_init_mask | uncertain_target_mask) & correct_permutation_mask & correct_msp_mask ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "change_log = torch.tensor(change_log)\n",
    "score_init = torch.tensor(score_init)\n",
    "score_best = torch.tensor(score_best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 178,  260,  321,  346,  460,  767,  772,  836, 1037])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_id_list_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 12, 115, 172, 217, 259, 339, 452, 470, 473, 481, 519, 522, 532, 545,\n",
       "        630, 647, 652, 711, 838, 874])"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  45,   52,   93,   97,  107,  194,  239,  248,  258,  358,  373,  392,\n",
       "         419,  439,  444,  454,  465,  547,  595,  622,  637,  683,  718,  723,\n",
       "         727,  842,  867,  935,  942,  949,  960,  976,  980,  995,  997, 1071])"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_col_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 12, 115, 172, 217, 259, 339, 452, 470, 473, 481, 519, 522, 532, 545,\n",
       "        630, 647, 652, 711, 838, 874])"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msp_new_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[1, 0] 0.0169745534658432 35\n",
      "********************************************************\n",
      "(0,) tensor(0.9979, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(1,) tensor(0.5204, device='cuda:1', grad_fn=<MaxBackward1>) tensor(35, device='cuda:1')\n",
      "(0, 1) tensor(0.9986, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "(1, 0) tensor(0.7587, device='cuda:1', grad_fn=<MaxBackward1>) tensor(0, device='cuda:1')\n",
      "********************************************************\n",
      "(0, 1) 0.9985560774803162 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/3464288220.py:36: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_955267/3464288220.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_955267/3464288220.py:57: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_955267/3464288220.py:58: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 12:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(batch[\"label\"].item())\n",
    "print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  45,  119,  124,  125,  128,  200,  218,  239,  260,  302,  321,  346,\n",
       "         355,  358,  392,  401,  415,  419,  439,  460,  569,  602,  622,  698,\n",
       "         718,  723,  772,  809,  867,  890,  922,  937,  949,  976,  989,  994,\n",
       "         995,  997, 1037, 1049])"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_freq = torch.zeros(args.num_classes)\n",
    "for batch_idx, batch in enumerate(train_dataloader_iter):\n",
    "    class_freq[batch[\"label\"].item()] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([790., 439., 198., 164.,  86.,  70.,  61.,  69.,  64.,  46.,  61.,  53.,\n",
       "         49.,  46.,  45.,  46.,  30.,  36.,  42.,  43.,  41.,  40.,  34.,  30.,\n",
       "         32.,  29.,  27.,  25.,  23.,  20.,  19.,  26.,  14.,  23.,  21.,  18.,\n",
       "         16.,  22.,  16.,  17.,  19.,  17.,  12.,  12.,  11.,  16.,  13.,  13.,\n",
       "          8.,  13.,   9.,  10.,   9.,  11.,  10.,   9.,   9.,  11.,  13.,   7.,\n",
       "         12.,  11.,   6.,  11.,   6.,  10.,   7.,   7.,  11.,  12.,   8.,  10.,\n",
       "          9.,   7.,   9.,  10.,   8.,   6.,   9.,   7.,   9.,   9.,   6.,   4.,\n",
       "          7.,   8.,   7.,   4.,   9.,   4.,   5.,   8.,   5.,   6.,   7.,   4.,\n",
       "          3.,   3.,   5.,   6.,   5.])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.25\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"label\"].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def reweight_logits(logits, class_weights):\n",
    "#     # Reweight the logits by multiplying with class weights\n",
    "#     reweighted_logits = logits * torch.sqrt(class_weights)\n",
    "    \n",
    "#     # Apply softmax to the reweighted logits\n",
    "#     reweighted_probs = F.softmax(reweighted_logits, dim=-1)\n",
    "    \n",
    "#     return reweighted_probs\n",
    "def reweight_logits(logits, class_weights):\n",
    "    # Step 1: Apply exp(logits)\n",
    "    exp_logits = torch.exp(logits)\n",
    "    \n",
    "    # Step 2: Multiply by class weights\n",
    "    # reweighted_exp = exp_logits * torch.sqrt(class_weights)\n",
    "    reweighted_exp = exp_logits * class_weights\n",
    "    # Step 3: Normalize to get valid probabilities (like softmax)\n",
    "    reweighted_probs = reweighted_exp / reweighted_exp.sum(dim=-1, keepdim=True)\n",
    "    \n",
    "    return reweighted_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  45,  119,  124,  125,  128,  200,  218,  239,  260,  302,  321,  346,\n",
       "         355,  358,  392,  401,  415,  419,  439,  460,  569,  602,  622,  698,\n",
       "         718,  723,  772,  809,  867,  890,  922,  937,  949,  976,  989,  994,\n",
       "         995,  997, 1037, 1049])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_col_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/1066959887.py:36: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_msp = []\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    # if batch_idx == 355:\n",
    "    if batch_idx not in target_col_idx:\n",
    "        continue\n",
    "\n",
    "    col_idx_set = target_col_mask.unique().tolist()\n",
    "    successs = False\n",
    "    init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "    init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "    init_msp = F.softmax(logits_init).max().item()\n",
    "    assert -1 not in col_idx_set\n",
    "    max_msp = 0 \n",
    "    for r in range(1, len(col_idx_set) + 1):\n",
    "        for subset in itertools.combinations(col_idx_set, r):\n",
    "            if 0 not in subset:\n",
    "                continue\n",
    "            for x in itertools.permutations(subset):\n",
    "                new_batch_data = []\n",
    "                for col_i in x:\n",
    "                    if col_i == 0:\n",
    "                        if len(new_batch_data) == 0:\n",
    "                            cls_indexes_value = 0\n",
    "                        else:\n",
    "                            cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                    new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "                # print(\"****origin****************\")\n",
    "                # msp_temp = F.softmax(logits_temp).max().item()\n",
    "                # predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "                # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "                # print(\"****weighted****************\")\n",
    "                logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "                msp_temp = logits_temp.max().item()\n",
    "                predict_temp = logits_temp.argmax().item()\n",
    "                # print(x, msp_temp, predict_temp)\n",
    "                if msp_temp > max_msp and 0 in x:\n",
    "                    max_msp = msp_temp\n",
    "                    best_msp_perm = x\n",
    "                    msp_predict = predict_temp\n",
    "    if msp_predict == batch[\"label\"].item():\n",
    "        corrected_msp.append(True)\n",
    "    else:\n",
    "        corrected_msp.append(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40 13 0.325\n"
     ]
    }
   ],
   "source": [
    "print(len(corrected_msp), sum(corrected_msp), sum(corrected_msp)/len(corrected_msp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: check the rest wrong msp repdictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor(93),\n",
       " tensor(213),\n",
       " tensor(229),\n",
       " tensor(373),\n",
       " tensor(380),\n",
       " tensor(465),\n",
       " tensor(480),\n",
       " tensor(490),\n",
       " tensor(530),\n",
       " tensor(547),\n",
       " tensor(649),\n",
       " tensor(683),\n",
       " tensor(796),\n",
       " tensor(797),\n",
       " tensor(894),\n",
       " tensor(905),\n",
       " tensor(960)]"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rest_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 2)"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****weighted****************\n",
      "0\n",
      "[0, 1, 2, 3] 0.9965016841888428 0\n",
      "********************************************************\n",
      "(0,) 0.565597414970398 8\n",
      "(0, 1) 0.7253597974777222 0\n",
      "(1, 0) 0.7149863839149475 1\n",
      "(0, 2) 0.9622083902359009 0\n",
      "(2, 0) 0.9209644794464111 8\n",
      "(0, 3) 0.6232067346572876 0\n",
      "(3, 0) 0.21673724055290222 8\n",
      "(0, 1, 2) 0.9424116015434265 0\n",
      "(0, 2, 1) 0.8189032673835754 0\n",
      "(1, 0, 2) 0.5844040513038635 8\n",
      "(1, 2, 0) 0.8372729420661926 8\n",
      "(2, 0, 1) 0.9242938160896301 8\n",
      "(2, 1, 0) 0.5104241967201233 8\n",
      "(0, 1, 3) 0.7963503003120422 0\n",
      "(0, 3, 1) 0.8788531422615051 0\n",
      "(1, 0, 3) 0.5939408540725708 2\n",
      "(1, 3, 0) 0.17285317182540894 8\n",
      "(3, 0, 1) 0.12025757133960724 43\n",
      "(3, 1, 0) 0.6485477089881897 2\n",
      "(0, 2, 3) 0.882753849029541 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/1269595593.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 3, 2) 0.9586843252182007 0\n",
      "(2, 0, 3) 0.7722136974334717 8\n",
      "(2, 3, 0) 0.15759515762329102 17\n",
      "(3, 0, 2) 0.1284075230360031 8\n",
      "(3, 2, 0) 0.2780799865722656 8\n",
      "(0, 1, 2, 3) 0.9922066330909729 0\n",
      "(0, 1, 3, 2) 0.9914277195930481 0\n",
      "(0, 2, 1, 3) 0.9866529107093811 0\n",
      "(0, 2, 3, 1) 0.9879024624824524 0\n",
      "(0, 3, 1, 2) 0.9925208687782288 0\n",
      "(0, 3, 2, 1) 0.9918546676635742 0\n",
      "(1, 0, 2, 3) 0.3617860972881317 2\n",
      "(1, 0, 3, 2) 0.4722830653190613 2\n",
      "(1, 2, 0, 3) 0.7849287986755371 8\n",
      "(1, 2, 3, 0) 0.21763253211975098 8\n",
      "(1, 3, 0, 2) 0.2503493130207062 17\n",
      "(1, 3, 2, 0) 0.5672049522399902 8\n",
      "(2, 0, 1, 3) 0.7476243376731873 8\n",
      "(2, 0, 3, 1) 0.7449418902397156 8\n",
      "(2, 1, 0, 3) 0.28633183240890503 11\n",
      "(2, 1, 3, 0) 0.34214460849761963 17\n",
      "(2, 3, 0, 1) 0.12016561627388 17\n",
      "(2, 3, 1, 0) 0.3709506094455719 2\n",
      "(3, 0, 1, 2) 0.09460796415805817 43\n",
      "(3, 0, 2, 1) 0.12393748760223389 43\n",
      "(3, 1, 0, 2) 0.4354759752750397 2\n",
      "(3, 1, 2, 0) 0.6392737627029419 8\n",
      "(3, 2, 0, 1) 0.2111869752407074 8\n",
      "(3, 2, 1, 0) 0.17329688370227814 16\n",
      "********************************************************\n",
      "(0, 3, 1, 2) 0.9925208687782288 0\n",
      "****origin****************\n",
      "(0,) tensor(0.4433) tensor(8)\n",
      "(0, 1) tensor(0.8296) tensor(0)\n",
      "(1, 0) tensor(0.7979) tensor(1)\n",
      "(0, 2) tensor(0.9797) tensor(0)\n",
      "(2, 0) tensor(0.8980) tensor(8)\n",
      "(0, 3) tensor(0.7515) tensor(0)\n",
      "(3, 0) tensor(0.2624) tensor(8)\n",
      "(0, 1, 2) tensor(0.9693) tensor(0)\n",
      "(0, 2, 1) tensor(0.8962) tensor(0)\n",
      "(1, 0, 2) tensor(0.5303) tensor(8)\n",
      "(1, 2, 0) tensor(0.8429) tensor(8)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/1269595593.py:82: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_955267/1269595593.py:102: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_955267/1269595593.py:103: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_955267/1269595593.py:104: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 0, 1) tensor(0.9228) tensor(8)\n",
      "(2, 1, 0) tensor(0.5198) tensor(8)\n",
      "(0, 1, 3) tensor(0.8954) tensor(0)\n",
      "(0, 3, 1) tensor(0.9240) tensor(0)\n",
      "(1, 0, 3) tensor(0.6565) tensor(2)\n",
      "(1, 3, 0) tensor(0.1947) tensor(8)\n",
      "(3, 0, 1) tensor(0.1085) tensor(43)\n",
      "(3, 1, 0) tensor(0.7339) tensor(2)\n",
      "(0, 2, 3) tensor(0.9421) tensor(0)\n",
      "(0, 3, 2) tensor(0.9801) tensor(0)\n",
      "(2, 0, 3) tensor(0.7799) tensor(8)\n",
      "(2, 3, 0) tensor(0.1723) tensor(17)\n",
      "(3, 0, 2) tensor(0.1681) tensor(8)\n",
      "(3, 2, 0) tensor(0.3541) tensor(8)\n",
      "(0, 1, 2, 3) tensor(0.9965) tensor(0)\n",
      "(0, 1, 3, 2) tensor(0.9962) tensor(0)\n",
      "(0, 2, 1, 3) tensor(0.9939) tensor(0)\n",
      "(0, 2, 3, 1) tensor(0.9944) tensor(0)\n",
      "(0, 3, 1, 2) tensor(0.9966) tensor(0)\n",
      "(0, 3, 2, 1) tensor(0.9963) tensor(0)\n",
      "(1, 0, 2, 3) tensor(0.4459) tensor(2)\n",
      "(1, 0, 3, 2) tensor(0.5658) tensor(2)\n",
      "(1, 2, 0, 3) tensor(0.7969) tensor(8)\n",
      "(1, 2, 3, 0) tensor(0.2510) tensor(8)\n",
      "(1, 3, 0, 2) tensor(0.2521) tensor(17)\n",
      "(1, 3, 2, 0) tensor(0.5896) tensor(8)\n",
      "(2, 0, 1, 3) tensor(0.7549) tensor(8)\n",
      "(2, 0, 3, 1) tensor(0.7277) tensor(8)\n",
      "(2, 1, 0, 3) tensor(0.2952) tensor(11)\n",
      "(2, 1, 3, 0) tensor(0.3426) tensor(17)\n",
      "(2, 3, 0, 1) tensor(0.1397) tensor(17)\n",
      "(2, 3, 1, 0) tensor(0.4768) tensor(2)\n",
      "(3, 0, 1, 2) tensor(0.0881) tensor(43)\n",
      "(3, 0, 2, 1) tensor(0.1268) tensor(8)\n",
      "(3, 1, 0, 2) tensor(0.5472) tensor(2)\n",
      "(3, 1, 2, 0) tensor(0.6622) tensor(8)\n",
      "(3, 2, 0, 1) tensor(0.2785) tensor(8)\n",
      "(3, 2, 1, 0) tensor(0.1827) tensor(2)\n",
      "********************************************************\n",
      "(0, 3, 1, 2) 0.9965726137161255 0\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    # if batch_idx == 355:\n",
    "    if batch_idx == 960:\n",
    "        break\n",
    "\n",
    "print(\"****weighted****************\")\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(batch[\"label\"].item())\n",
    "print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        if 0 not in subset:\n",
    "            continue\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "            # print(\"****origin****************\")\n",
    "            # msp_temp = F.softmax(logits_temp).max().item()\n",
    "            # predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            # print(\"****weighted****************\")\n",
    "            logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "            msp_temp = logits_temp.max().item()\n",
    "            predict_temp = logits_temp.argmax().item()\n",
    "            print(x, msp_temp, predict_temp)\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)\n",
    "\n",
    "\n",
    "print(\"****origin****************\")\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        if 0 not in subset:\n",
    "            continue        \n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "            \n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            \n",
    "            # logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "            # msp_temp = logits_temp.max().item()\n",
    "            # predict_temp = logits_temp.argmax().item()\n",
    "            # print(x, msp_temp, predict_temp)\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   4,   12,   96,  115,  131,  143,  154,  172,  217,  221,  259,  324,\n",
       "         339,  452,  470,  473,  481,  498,  519,  522,  532,  540,  545,  573,\n",
       "         580,  630,  647,  652,  671,  684,  711,  747,  748,  767,  838,  874,\n",
       "         964, 1038, 1049])"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_col_idx_init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 45,  52,  93, 194, 373, 392, 439, 444, 454, 547, 595, 622, 637, 718,\n",
       "        935, 942, 960, 976, 980, 995])"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_new_idx\n",
    "# 45, 52, 392,439, 444, 622, 637,\n",
    "# 935, 942 980 model no knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n",
      "[1, 2, 3, 0] 0.1787736713886261 23\n",
      "********************************************************\n",
      "(0,) 0.5114062428474426 6\n",
      "(1,) 0.9494367837905884 0\n",
      "(2,) 0.7153188586235046 8\n",
      "(3,) 0.8116299510002136 0\n",
      "(0, 1) 0.9376112818717957 0\n",
      "(1, 0) 0.2014230191707611 10\n",
      "(0, 2) 0.1857820749282837 23\n",
      "(2, 0) 0.4104098677635193 23\n",
      "(0, 3) 0.41415852308273315 0\n",
      "(3, 0) 0.1487245261669159 23\n",
      "(1, 2) 0.2603535056114197 8\n",
      "(2, 1) 0.4953395128250122 8\n",
      "(1, 3) 0.0964953675866127 46\n",
      "(3, 1) 0.12780238687992096 37\n",
      "(2, 3) 0.09776917099952698 37\n",
      "(3, 2) 0.38815242052078247 28\n",
      "(0, 1, 2) 0.8890414237976074 0\n",
      "(0, 2, 1) 0.20967409014701843 23\n",
      "(1, 0, 2) 0.11068226397037506 65\n",
      "(1, 2, 0) 0.6806222796440125 23\n",
      "(2, 0, 1) 0.25590312480926514 23\n",
      "(2, 1, 0) 0.388788640499115 23\n",
      "(0, 1, 3) 0.8084651231765747 0\n",
      "(0, 3, 1) 0.11917047202587128 0\n",
      "(1, 0, 3) 0.2300780862569809 65\n",
      "(1, 3, 0) 0.2495071142911911 23\n",
      "(3, 0, 1) 0.14841237664222717 23\n",
      "(3, 1, 0) 0.09925299882888794 65\n",
      "(0, 2, 3) 0.7576421499252319 0\n",
      "(0, 3, 2) 0.8447192907333374 0\n",
      "(2, 0, 3) 0.19699996709823608 23\n",
      "(2, 3, 0) 0.7239934206008911 23\n",
      "(3, 0, 2) 0.3288387358188629 23\n",
      "(3, 2, 0) 0.6753793358802795 23\n",
      "(1, 2, 3) 0.16110365092754364 16\n",
      "(1, 3, 2) 0.2077295184135437 46\n",
      "(2, 1, 3) 0.2706303000450134 24\n",
      "(2, 3, 1) 0.2889205515384674 58\n",
      "(3, 1, 2) 0.2249068319797516 65\n",
      "(3, 2, 1) 0.18729594349861145 37\n",
      "(0, 1, 2, 3) 0.9230952262878418 0\n",
      "(0, 1, 3, 2) 0.8954490423202515 0\n",
      "(0, 2, 1, 3) 0.6206144690513611 0\n",
      "(0, 2, 3, 1) 0.8091468214988708 0\n",
      "(0, 3, 1, 2) 0.4451865255832672 0\n",
      "(0, 3, 2, 1) 0.8825711011886597 0\n",
      "(1, 0, 2, 3) 0.21326154470443726 65\n",
      "(1, 0, 3, 2) 0.21284206211566925 65\n",
      "(1, 2, 0, 3) 0.2777409553527832 23\n",
      "(1, 2, 3, 0) 0.1787736713886261 23\n",
      "(1, 3, 0, 2) 0.6970555186271667 23\n",
      "(1, 3, 2, 0) 0.5800784826278687 23\n",
      "(2, 0, 1, 3) 0.41641274094581604 23\n",
      "(2, 0, 3, 1) 0.1857418268918991 23\n",
      "(2, 1, 0, 3) 0.15408900380134583 58\n",
      "(2, 1, 3, 0) 0.12323631346225739 23\n",
      "(2, 3, 0, 1) 0.6678833961486816 23\n",
      "(2, 3, 1, 0) 0.32364338636398315 49\n",
      "(3, 0, 1, 2) 0.3885897099971771 23\n",
      "(3, 0, 2, 1) 0.42636775970458984 23\n",
      "(3, 1, 0, 2) 0.09561468660831451 65\n",
      "(3, 1, 2, 0) 0.648291826248169 23\n",
      "(3, 2, 0, 1) 0.7420974969863892 23\n",
      "(3, 2, 1, 0) 0.08319099992513657 100\n",
      "********************************************************\n",
      "(0, 1) 0.9376112818717957 0\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "  \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "alpha = 1.0\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,).detach().cpu()\n",
    "    logits = reweight_logits(logits, class_weights)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    # if batch_idx == 355:\n",
    "    if batch_idx == 995:\n",
    "        break\n",
    "\n",
    "# print(\"****weighted****************\")\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = logits_init.max().item()\n",
    "print(batch[\"label\"].item())\n",
    "print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        # if 0 not in subset:\n",
    "        #     continue\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "            # print(\"****origin****************\")\n",
    "            # msp_temp = F.softmax(logits_temp).max().item()\n",
    "            # predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            # print(\"****weighted****************\")\n",
    "            logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "            msp_temp = logits_temp.max().item()\n",
    "            predict_temp = logits_temp.argmax().item()\n",
    "            print(x, msp_temp, predict_temp)\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  45,   52,   93,   97,  107,  194,  239,  248,  258,  358,  373,  392,\n",
       "         419,  439,  444,  454,  465,  547,  595,  622,  637,  683,  718,  723,\n",
       "         727,  842,  867,  935,  942,  949,  960,  976,  980,  995,  997, 1071])"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_col_idx_msp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94\n",
      "[1, 0] 0.2327449470758438 94\n",
      "********************************************************\n",
      "(0,) 0.21304257214069366 34\n",
      "(1,) 0.8460359573364258 13\n",
      "(0, 1) 0.2356746941804886 34\n",
      "(1, 0) 0.2327449470758438 94\n",
      "********************************************************\n",
      "(0, 1) 0.2356746941804886 34\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "  \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "alpha = 1.0\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,).detach().cpu()\n",
    "    logits = reweight_logits(logits, class_weights)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    # if batch_idx == 355:\n",
    "    if batch_idx == 258:\n",
    "        break\n",
    "\n",
    "# print(\"****weighted****************\")\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = logits_init.max().item()\n",
    "print(batch[\"label\"].item())\n",
    "print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        # if 0 not in subset:\n",
    "        #     continue\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "            # print(\"****origin****************\")\n",
    "            # msp_temp = F.softmax(logits_temp).max().item()\n",
    "            # predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            # print(\"****weighted****************\")\n",
    "            logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "            msp_temp = logits_temp.max().item()\n",
    "            predict_temp = logits_temp.argmax().item()\n",
    "            print(x, msp_temp, predict_temp)\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/3447887029.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_955267/3447887029.py:83: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_955267/3447887029.py:103: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_955267/3447887029.py:104: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "correted_rest = []\n",
    "\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    # if batch_idx == 355:\n",
    "    if batch_idx not in rest_idx:\n",
    "        continue\n",
    "\n",
    "    # print(\"****weighted****************\")\n",
    "    col_idx_set = target_col_mask.unique().tolist()\n",
    "    successs = False\n",
    "    init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "    init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "    init_msp = F.softmax(logits_init).max().item()\n",
    "    # print(batch[\"label\"].item())\n",
    "    # print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "    # print(\"********************************************************\")\n",
    "    assert -1 not in col_idx_set\n",
    "    max_msp = 0 \n",
    "    for r in range(1, len(col_idx_set) + 1):\n",
    "        for subset in itertools.combinations(col_idx_set, r):\n",
    "            if 0 not in subset:\n",
    "                continue\n",
    "            for x in itertools.permutations(subset):\n",
    "                new_batch_data = []\n",
    "                for col_i in x:\n",
    "                    if col_i == 0:\n",
    "                        if len(new_batch_data) == 0:\n",
    "                            cls_indexes_value = 0\n",
    "                        else:\n",
    "                            cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                    new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "                # print(\"****origin****************\")\n",
    "                # msp_temp = F.softmax(logits_temp).max().item()\n",
    "                # predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "                # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "                # print(\"****weighted****************\")\n",
    "                logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "                msp_temp = logits_temp.max().item()\n",
    "                predict_temp = logits_temp.argmax().item()\n",
    "                # print(x, msp_temp, predict_temp)\n",
    "                if msp_temp > max_msp and 0 in x:\n",
    "                    max_msp = msp_temp\n",
    "                    best_msp_perm = x\n",
    "                    msp_predict_weighted = predict_temp\n",
    "    # print(\"********************************************************\")\n",
    "    # print(best_msp_perm, max_msp, msp_predict)\n",
    "\n",
    "\n",
    "    # print(\"****origin****************\")\n",
    "    col_idx_set = target_col_mask.unique().tolist()\n",
    "    successs = False\n",
    "    init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "    init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "    init_msp = F.softmax(logits_init).max().item()\n",
    "    assert -1 not in col_idx_set\n",
    "    max_msp = 0 \n",
    "    for r in range(1, len(col_idx_set) + 1):\n",
    "        for subset in itertools.combinations(col_idx_set, r):\n",
    "            if 0 not in subset:\n",
    "                continue\n",
    "            for x in itertools.permutations(subset):\n",
    "                new_batch_data = []\n",
    "                for col_i in x:\n",
    "                    if col_i == 0:\n",
    "                        if len(new_batch_data) == 0:\n",
    "                            cls_indexes_value = 0\n",
    "                        else:\n",
    "                            cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                    new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "                new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,).detach().cpu()\n",
    "                \n",
    "                msp_temp = F.softmax(logits_temp).max().item()\n",
    "                predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "                # print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "                \n",
    "                # logits_temp = reweight_logits(logits_temp, class_weights)\n",
    "                # msp_temp = logits_temp.max().item()\n",
    "                # predict_temp = logits_temp.argmax().item()\n",
    "                # print(x, msp_temp, predict_temp)\n",
    "                if msp_temp > max_msp and 0 in x:\n",
    "                    max_msp = msp_temp\n",
    "                    best_msp_perm = x\n",
    "                    msp_predict = predict_temp\n",
    "    if msp_predict == msp_predict_weighted:\n",
    "        correted_rest.append(True)\n",
    "    else:\n",
    "        correted_rest.append(False)\n",
    "    # print(\"********************************************************\")\n",
    "    # print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(correted_rest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(correted_rest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "[0, 1] 0.08097616583108902 2\n",
      "********************************************************\n",
      "(0,) tensor(0.9377, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "(1,) tensor(0.5614, device='cuda:1', grad_fn=<MaxBackward1>) tensor(52, device='cuda:1')\n",
      "(0, 1) tensor(0.0810, device='cuda:1', grad_fn=<MaxBackward1>) tensor(2, device='cuda:1')\n",
      "(1, 0) tensor(0.5527, device='cuda:1', grad_fn=<MaxBackward1>) tensor(3, device='cuda:1')\n",
      "********************************************************\n",
      "(0,) 0.9376919269561768 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_955267/3269090917.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  init_msp = F.softmax(logits_init).max().item()\n",
      "/tmp/ipykernel_955267/3269090917.py:55: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n",
      "/tmp/ipykernel_955267/3269090917.py:56: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  predict_temp = F.softmax(logits_temp).argmax().item()\n",
      "/tmp/ipykernel_955267/3269090917.py:57: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n"
     ]
    }
   ],
   "source": [
    "# use target as head, the MSP context\n",
    "import itertools\n",
    "\n",
    "\n",
    "    \n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "init_permutation = defaultdict(list)\n",
    "corrected_permutation = defaultdict(list)\n",
    "init_logits = defaultdict(list)\n",
    "corrected_logits = defaultdict(list)\n",
    "\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    logits_init = logits.clone()\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    if batch_idx == 143:\n",
    "        break\n",
    "\n",
    "col_idx_set = target_col_mask.unique().tolist()\n",
    "successs = False\n",
    "init_permutation[batch_idx].append(get_permutation(target_col_mask))\n",
    "init_logits[batch_idx].append(logits_init.detach().cpu())     \n",
    "init_msp = F.softmax(logits_init).max().item()\n",
    "print(batch[\"label\"].item())\n",
    "print(get_permutation(target_col_mask), init_msp, init_logits[batch_idx][0].argmax().item())   \n",
    "print(\"********************************************************\")\n",
    "assert -1 not in col_idx_set\n",
    "max_msp = 0 \n",
    "for r in range(1, len(col_idx_set) + 1):\n",
    "    for subset in itertools.combinations(col_idx_set, r):\n",
    "        for x in itertools.permutations(subset):\n",
    "            new_batch_data = []\n",
    "            for col_i in x:\n",
    "                if col_i == 0:\n",
    "                    if len(new_batch_data) == 0:\n",
    "                        cls_indexes_value = 0\n",
    "                    else:\n",
    "                        cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                new_batch_data.append(batch[\"data\"].T[target_col_mask==col_i])\n",
    "            new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "            cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "            logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "            msp_temp = F.softmax(logits_temp).max().item()\n",
    "            predict_temp = F.softmax(logits_temp).argmax().item()\n",
    "            print(x, F.softmax(logits_temp).max(), F.softmax(logits_temp).argmax())\n",
    "            if msp_temp > max_msp and 0 in x:\n",
    "                max_msp = msp_temp\n",
    "                best_msp_perm = x\n",
    "                msp_predict = predict_temp\n",
    "print(\"********************************************************\")\n",
    "print(best_msp_perm, max_msp, msp_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.9345, 1.6622, 2.4728, 1.5530, 1.7454, 1.5963, 2.6106, 2.2574, 1.8556,\n",
       "        2.3706, 2.4085, 2.8997, 2.6628])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_init[change_log == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.9345, 0.0149, 0.5363, 1.5530, 1.7454, 0.7366, 0.4971, 0.0093, 0.1013,\n",
       "        1.3824, 0.1368, 0.4204, 0.7160])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_best[change_log == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.5975, 2.2764, 1.9519, 1.6245, 1.8343, 1.9780, 1.6903, 2.6139, 1.8942,\n",
       "        2.1679, 2.1553, 1.6920, 1.5104, 1.6143, 1.6281, 2.7796, 2.4067, 2.6658])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_init[change_log == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.6363, 0.7895, 0.0909, 1.6245, 0.1558, 0.3021, 0.0241, 0.0160, 0.0072,\n",
       "        2.1679, 0.0843, 0.0077, 1.4630, 0.0821, 1.6281, 0.2794, 0.0186, 2.6262])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_best[change_log == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAARgklEQVR4nO3dfYxldX3H8ffHZUETqaTuRNd9cGwwTdX6gCPy0DQUYoLWsmnFimkFjHSNSoXU2IhNMPKfaaNGMOIGiGCNrsWHLBZqaMSnWFcHuiCw2q6muoukDIsuUhW79ts/5qjj5c7MZZlzrzO/9ys52XPO73fPfH9zdvdzz8O9J1WFJKldj5t0AZKkyTIIJKlxBoEkNc4gkKTGGQSS1LijJl3Ao7Vhw4aanp6edBmStKrceuut91fV1LC2VRcE09PTzM7OTroMSVpVknx3sTZPDUlS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTG9R4ESdYl+fcknxnSdkySnUn2JdmdZLrveiRJv24cRwQXAXsXaXsd8IOqOh54D/CuMdQjSVqg1yBIshn4Y+CqRbpsA67t5q8HzkiSPmuSJP26vj9Z/F7gb4FjF2nfBOwHqKrDSQ4BTwbuX9gpyXZgO8DWrVv7qlVatTZt2cr3D+yfdBljtW79Mfz8fx+edBlj9bTNW7hn//dWfLu9BUGSlwP3VdWtSU57LNuqqh3ADoCZmRkfqSYN+P6B/bzqg1+ZdBljtfP1pzQ55j70eWroVOCsJP8FfAw4Pck/DvS5B9gCkOQo4EnAwR5rkiQN6C0IquqSqtpcVdPAOcDnquovB7rtAs7r5s/u+viOX5LGaOzfPprkMmC2qnYBVwMfTrIPeID5wJAkjdFYgqCqPg98vpu/dMH6nwKvHEcNkqTh/GSxJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxvQVBkscn+VqS25PcleSdQ/qcn2QuyZ5uuqCveiRJw/X5hLKHgdOr6qEk64EvJ7mpqr460G9nVV3YYx2SpCX0FgTdQ+gf6hbXd5MPppek3zC9XiNIsi7JHuA+4Oaq2j2k2yuS3JHk+iRb+qxHkvRIvQZBVf28qp4PbAZOTPKcgS43ANNV9VzgZuDaYdtJsj3JbJLZubm5PkuWpOaM5a6hqvohcAtw5sD6g1X1cLd4FfDCRV6/o6pmqmpmamqq11olqTV93jU0leS4bv4JwEuAbw702bhg8Sxgb1/1SJKG6/OuoY3AtUnWMR84H6+qzyS5DJitql3Am5OcBRwGHgDO77EeSdIQfd41dAfwgiHrL10wfwlwSV81SJKW5yeLJalxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXF9PrP48Um+luT2JHcleeeQPsck2ZlkX5LdSab7qkeSNFyfRwQPA6dX1fOA5wNnJjlpoM/rgB9U1fHAe4B39ViPJGmI3oKg5j3ULa7vphrotg24tpu/HjgjSfqqSZL0SL1eI0iyLske4D7g5qraPdBlE7AfoKoOA4eAJw/ZzvYks0lm5+bmjrieTVu2kqSpadOWrUf8+5LUhqP63HhV/Rx4fpLjgE8leU5V3XkE29kB7ACYmZkZPKoY2fcP7OdVH/zKkb58Vdr5+lMmXYKk33BjuWuoqn4I3AKcOdB0D7AFIMlRwJOAg+OoSZI0r8+7hqa6IwGSPAF4CfDNgW67gPO6+bOBz1XVEb/jlyQ9en2eGtoIXJtkHfOB8/Gq+kySy4DZqtoFXA18OMk+4AHgnB7rkSQN0VsQVNUdwAuGrL90wfxPgVf2VYMkaXl+sliSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIa1+czi7ckuSXJ3UnuSnLRkD6nJTmUZE83XTpsW5Kk/vT5zOLDwFuq6rYkxwK3Jrm5qu4e6Pelqnp5j3VIkpbQ2xFBVd1bVbd18z8C9gKb+vp5kqQjM5ZrBEmmmX+Q/e4hzScnuT3JTUmevcjrtyeZTTI7NzfXZ6mS1JzegyDJE4FPABdX1YMDzbcBT6+q5wGXA58eto2q2lFVM1U1MzU11Wu9ktSaXoMgyXrmQ+AjVfXJwfaqerCqHurmbwTWJ9nQZ02SpF/X511DAa4G9lbVuxfp89SuH0lO7Oo52FdNkqRH6vOuoVOB1wDfSLKnW/d2YCtAVV0JnA28Iclh4CfAOVVVPdYkSRrQWxBU1ZeBLNPnCuCKvmqQJC3PTxZLUuMMAklqnEEgSY0bKQiSnDrKOknS6jPqEcHlI66TJK0yS941lORk4BRgKsnfLGj6LWBdn4VJksZjudtHjwae2PU7dsH6B5n/DIAkaZVbMgiq6gvAF5J8qKq+O6aaJEljNOoHyo5JsgOYXviaqjq9j6IkSeMzahD8E3AlcBXw8/7KkSSN26hBcLiqPtBrJZKkiRj19tEbkrwxycYkv/2LqdfKJEljMeoRwXndn29dsK6A31nZciRJ4zZSEFTVM/ouRJI0GSMFQZJzh62vqutWthxJ0riNemroRQvmHw+cwfzzhg0CSVrlRj019NcLl5McB3ysj4IkSeN1pF9D/T/AktcNkmxJckuSu5PcleSiIX2S5H1J9iW5I8kJR1iPJOkIjXqN4Abm7xKC+S+b+z3g48u87DDwlqq6LcmxwK1Jbq6quxf0eSnwzG56MfCB7k9J0piMeo3gHxbMHwa+W1UHlnpBVd0L3NvN/yjJXmATsDAItgHXdQ+s/2qS45Js7F4rSRqDUa8RfCHJU/jVReP/fDQ/JMk08AJg90DTJmD/guUD3bpfC4Ik24HtAFu3bn00P1qPO4okk65irJ62eQv37P/epMuQVo1RTw39OfD3wOeBAJcneWtVXT/Ca58IfAK4uKoePJIiq2oHsANgZmamlumuhf7vMK/64FcmXcVY7Xz9KZMuQVpVRj019HfAi6rqPoAkU8C/AksGQZL1zIfAR6rqk0O63ANsWbC8uVsnSRqTUe8aetwvQqBzcLnXZv58xNXA3qp69yLddgHndncPnQQc8vqAJI3XqEcE/5Lks8BHu+VXATcu85pTgdcA30iyp1v3dmArQFVd2W3jZcA+4MfAa0euXJK0IpZ7ZvHxwFOq6q1J/gz4g67p34CPLPXaqvoy89cTlupTwJtGL1eStNKWOyJ4L3AJQHeO/5MASX6/a/uTHmuTJI3BctcInlJV3xhc2a2b7qUiSdJYLRcExy3R9oQVrEOSNCHLBcFskr8aXJnkAuDWfkqSJI3TctcILgY+leQv+NV//DPA0cCf9liXJGlMlgyCqvpv4JQkfwQ8p1v9z1X1ud4rkySNxajfNXQLcEvPtUiSJuBIn0cgSVojDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWpcb0GQ5Jok9yW5c5H205IcSrKnmy7tqxZJ0uJGfWbxkfgQcAVw3RJ9vlRVL++xBknSMno7IqiqLwIP9LV9SdLKmPQ1gpOT3J7kpiTPXqxTku1JZpPMzs3NjbM+SVrzJhkEtwFPr6rnAZcDn16sY1XtqKqZqpqZmpoaV32S1ISJBUFVPVhVD3XzNwLrk2yYVD2S1KqJBUGSpyZJN39iV8vBSdUjSa3q7a6hJB8FTgM2JDkAvANYD1BVVwJnA29Ichj4CXBOVVVf9UiShustCKrq1cu0X8H87aWSpAma9F1DkqQJMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY3rLQiSXJPkviR3LtKeJO9Lsi/JHUlO6KsWSdLi+jwi+BBw5hLtLwWe2U3bgQ/0WIskaRG9BUFVfRF4YIku24Drat5XgeOSbOyrHknScJO8RrAJ2L9g+UC37hGSbE8ym2R2bm5uLMVJUitWxcXiqtpRVTNVNTM1NTXpciRpTZlkENwDbFmwvLlbJ0kao0kGwS7g3O7uoZOAQ1V17wTrkaQmHdXXhpN8FDgN2JDkAPAOYD1AVV0J3Ai8DNgH/Bh4bV+1SJIW11sQVNWrl2kv4E19/XxJ0mhWxcViSVJ/DAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuN6DYIkZyb5VpJ9Sd42pP38JHNJ9nTTBX3WI0l6pD6fWbwOeD/wEuAA8PUku6rq7oGuO6vqwr7qkCQtrc8jghOBfVX1nar6GfAxYFuPP0+SdAT6DIJNwP4Fywe6dYNekeSOJNcn2TJsQ0m2J5lNMjs3N9dHrZLUrElfLL4BmK6q5wI3A9cO61RVO6pqpqpmpqamxlqgJK11fQbBPcDCd/ibu3W/VFUHq+rhbvEq4IU91iNJGqLPIPg68Mwkz0hyNHAOsGthhyQbFyyeBeztsR5J0hC93TVUVYeTXAh8FlgHXFNVdyW5DJitql3Am5OcBRwGHgDO76seSdJwvQUBQFXdCNw4sO7SBfOXAJf0WYMkaWmTvlgsSZowg0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1LhegyDJmUm+lWRfkrcNaT8myc6ufXeS6T7rkSQ9Um9BkGQd8H7gpcCzgFcnedZAt9cBP6iq44H3AO/qqx5J0nB9HhGcCOyrqu9U1c+AjwHbBvpsA67t5q8HzkiSHmuSJA1IVfWz4eRs4MyquqBbfg3w4qq6cEGfO7s+B7rlb3d97h/Y1nZge7f4u8C3jrCsDcD9y/Zae1ocd4tjhjbH7ZhH8/SqmhrWcNRjr6d/VbUD2PFYt5NktqpmVqCkVaXFcbc4Zmhz3I75sevz1NA9wJYFy5u7dUP7JDkKeBJwsMeaJEkD+gyCrwPPTPKMJEcD5wC7BvrsAs7r5s8GPld9nauSJA3V26mhqjqc5ELgs8A64JqquivJZcBsVe0CrgY+nGQf8ADzYdGnx3x6aZVqcdwtjhnaHLdjfox6u1gsSVod/GSxJDXOIJCkxq3JIGjxqy1GGPP5SeaS7OmmCyZR50pKck2S+7rPowxrT5L3db+TO5KcMO4a+zDCuE9LcmjBvr503DWutCRbktyS5O4kdyW5aEifNbW/RxzzyuzrqlpTE/MXpr8N/A5wNHA78KyBPm8EruzmzwF2TrruMYz5fOCKSde6wuP+Q+AE4M5F2l8G3AQEOAnYPemaxzTu04DPTLrOFR7zRuCEbv5Y4D+G/B1fU/t7xDGvyL5ei0cELX61xShjXnOq6ovM3222mG3AdTXvq8BxSTaOp7r+jDDuNaeq7q2q27r5HwF7gU0D3dbU/h5xzCtiLQbBJmD/guUDPPKX98s+VXUYOAQ8eSzV9WOUMQO8ojtkvj7JliHta82ov5e16OQktye5KcmzJ13MSupO5b4A2D3QtGb39xJjhhXY12sxCDTcDcB0VT0XuJlfHRFp7bmN+e+VeR5wOfDpyZazcpI8EfgEcHFVPTjpesZhmTGvyL5ei0HQ4ldbLDvmqjpYVQ93i1cBLxxTbZM0yt+FNaeqHqyqh7r5G4H1STZMuKzHLMl65v9D/EhVfXJIlzW3v5cb80rt67UYBC1+tcWyYx44V3oW8+cb17pdwLnd3SQnAYeq6t5JF9W3JE/9xTWvJCcy/+98Nb/RoRvP1cDeqnr3It3W1P4eZcwrta9XxbePPhr1m/nVFr0accxvTnIWcJj5MZ8/sYJXSJKPMn/XxIYkB4B3AOsBqupK4Ebm7yTZB/wYeO1kKl1ZI4z7bOANSQ4DPwHOWeVvdABOBV4DfCPJnm7d24GtsGb39yhjXpF97VdMSFLj1uKpIUnSo2AQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMb9P4ynL3wLYAD/AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(score_init[change_log == 1] - score_best[change_log == 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAANqklEQVR4nO3dfaze5V3H8feHtrA5cERbEUtrWWaIOJ3gGY6yLBtEg1MhKhEWZWMRS1w2IRqM06jRfzXLJi5Cw9iDInQyMIDAxPCUhVE9ZbCVhymSYQsoBxZ5UAOWff3j3F1Py+k5N+fc17l7rvN+JXf6e76+1/1rP/n1un/3705VIUnqz2HjLkCS1IYBL0mdMuAlqVMGvCR1yoCXpE6tHncBM61du7Y2bdo07jIkadnYsWPHs1W1brZ1h1TAb9q0icnJyXGXIUnLRpInDrbOIRpJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUqaYBn+ToJNcleTTJI0lObdmeJGmf1vfBfxK4rarOSXI48F2N25MkDTQL+CRvBt4NXABQVa8Ar7RqT5K0v5ZDNMcDU8Bnknw1yZVJ3nTgRkm2JJlMMjk1NdWwHGnh1m/YSJIuXus3bBz326klkla/6JRkArgPOK2qtif5JPBCVf3BwfaZmJgoH1WgQ1ESzr3i3nGXMRLbLtqMv+TWjyQ7qmpitnUtr+B3A7uravtg/jrg5IbtSZJmaBbwVfUfwK4kJwwWnQE83Ko9SdL+Wt9F81Hg6sEdNI8DH2rcniRpoGnAV9UDwKxjQ5KktvwmqyR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1anXLgyf5JvAi8Cqwp6omWrYnSdqnacAPvLeqnl2CdiRJMzhEI0mdah3wBfxDkh1Jtsy2QZItSSaTTE5NTS24ofUbNpKki9f6DRsX/D5I0l6th2jeVVVPJvk+4PYkj1bVPTM3qKqtwFaAiYmJWmhDT+3exblX3Lu4ag8R2y7aPO4SJHWg6RV8VT05+PMZ4AbglJbtSZL2aRbwSd6U5Ki908BPAztbtSdJ2l/LIZpjgBuS7G3nb6rqtobtSZJmaBbwVfU48PZWx5ckzc3bJCWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHWqecAnWZXkq0lubt2WJGmfpbiCvxh4ZAnakSTN0DTgkxwH/CxwZct2JEmv1foK/hPA7wDfPtgGSbYkmUwyOTU11bgcSVo5mgV8kp8DnqmqHXNtV1Vbq2qiqibWrVvXqhxJWnFaXsGfBpyV5JvAtcDpSf66YXuSpBmaBXxVfayqjquqTcB5wB1V9aut2pMk7c/74CWpU6uXopGqugu4aynakiRN8wpekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnq1FABn+S0YZZJkg4dw17BXzbkMknSIWLOp0kmORXYDKxL8lszVn03sKplYZKkxZnvccGHA0cOtjtqxvIXgHNaFSVJWrw5A76q7gbuTvLZqnpiiWqSJI3AsD/4cUSSrcCmmftU1ektipIkLd6wAf+3wOXAlcCr7cqRJI3KsAG/p6r+smklkqSRGvY2yZuSfDjJsUm+Z++raWWSpEUZ9gr+g4M/L52xrIC3jLYcSdKoDBXwVXV860IkSaM1VMAn+cBsy6vq86MtR5I0KsMO0bxjxvQbgDOA+wEDXpIOUcMO0Xx05nySo4FrWxQkSRqNhT4u+L+BOcflk7whyT8leTDJQ0n+eIFtSZIWYNgx+JuYvmsGph8y9sPAF+bZ7WXg9Kp6Kcka4MtJbq2q+xZcrSRpaMOOwf/ZjOk9wBNVtXuuHaqqgJcGs2sGrzr4HpKkURp2DP7uJMew78PWfx1mvySrgB3AW4FPVdX2WbbZAmwB2Lhx4zCHlbQYh60mybirWLRVa47g1f97edxljMQPHLeBJ3f9+8iPO+wQzS8DfwrcBQS4LMmlVXXdXPtV1avAjw8+lL0hyduqaucB22wFtgJMTEx4hS+19u09nHvFveOuYtG2XbS5i37AdF9aGHaI5veBd1TVMwBJ1gH/CMwZ8HtV1X8luRM4E9g53/aSpMUb9i6aw/aG+8Bz8+2bZN3gyp0kbwR+Cnh0IUVKkl6/Ya/gb0vyJeCawfy5wC3z7HMs8LnBOPxhwBeq6uaFlSlJer3m+03WtwLHVNWlSX4ReNdg1VeAq+fat6q+Bpw0kiolSa/bfFfwnwA+BlBV1wPXAyT50cG6n29YmyRpEeYbgz+mqr5+4MLBsk1NKpIkjcR8AX/0HOveOMI6JEkjNl/ATyb59QMXJrmQ6S8wSZIOUfONwV/C9BeUfoV9gT4BHA78QsO6JEmLNGfAV9V/ApuTvBd422Dx31fVHc0rkyQtyrDPorkTuLNxLZKkEVro8+AlSYc4A16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqeaBXySDUnuTPJwkoeSXNyqLUnSaw31o9sLtAf47aq6P8lRwI4kt1fVww3blCQNNLuCr6qnq+r+wfSLwCPA+lbtSZL2tyRj8Ek2AScB22dZtyXJZJLJqamppShHklaE5gGf5Ejgi8AlVfXCgeuramtVTVTVxLp161qXI0krRtOAT7KG6XC/uqqub9mWJGl/Le+iCfBp4JGq+nirdiRJs2t5BX8acD5wepIHBq/3NWxPkjRDs9skq+rLQFodX5I0N7/JKkmdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdahbwSa5K8kySna3akCQdXMsr+M8CZzY8viRpDs0CvqruAb7V6viSpLmtHncBSbYAWwA2btw45mo0Sus3bOSp3bvGXYa0Yo094KtqK7AVYGJiosZcjkboqd27OPeKe8ddxkhsu2jzuEuQXjfvopGkThnwktSplrdJXgN8BTghye4kv9aqLUnSazUbg6+q97c6tiRpfg7RSFKnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6lTTgE9yZpJvJHksye+2bEuStL9mAZ9kFfAp4GeAE4H3JzmxVXuSpP21vII/BXisqh6vqleAa4GzG7YnSZohVdXmwMk5wJlVdeFg/nzgJ6vqIwdstwXYMpg9AfjGAptcCzy7wH2Xk5XST7CvPVop/YSl6+sPVtW62VasXoLG51RVW4Gtiz1OksmqmhhBSYe0ldJPsK89Win9hEOjry2HaJ4ENsyYP26wTJK0BFoG/D8DP5Tk+CSHA+cBNzZsT5I0Q7Mhmqrak+QjwJeAVcBVVfVQq/YYwTDPMrFS+gn2tUcrpZ9wCPS12YeskqTx8pusktQpA16SOrXsAn6+xx8kOSLJtsH67Uk2jaHMRRuinxckmUrywOB14TjqXKwkVyV5JsnOg6xPkj8fvA9fS3LyUtc4KkP09T1Jnp9xTv9wqWschSQbktyZ5OEkDyW5eJZtlv15HbKf4z2nVbVsXkx/WPtvwFuAw4EHgRMP2ObDwOWD6fOAbeOuu1E/LwD+Yty1jqCv7wZOBnYeZP37gFuBAO8Eto+75oZ9fQ9w87jrHEE/jwVOHkwfBfzLLH9/l/15HbKfYz2ny+0KfpjHH5wNfG4wfR1wRpIsYY2jsGIe81BV9wDfmmOTs4HP17T7gKOTHLs01Y3WEH3tQlU9XVX3D6ZfBB4B1h+w2bI/r0P2c6yWW8CvB3bNmN/Na9/Q72xTVXuA54HvXZLqRmeYfgL80uC/t9cl2TDL+h4M+1704tQkDya5NcmPjLuYxRoMkZ4EbD9gVVfndY5+whjP6XILeO1zE7Cpqn4MuJ19/2vR8nU/088VeTtwGfB34y1ncZIcCXwRuKSqXhh3Pa3M08+xntPlFvDDPP7gO9skWQ28GXhuSaobnXn7WVXPVdXLg9krgZ9YotqW2op55EVVvVBVLw2mbwHWJFk75rIWJMkapkPv6qq6fpZNujiv8/Vz3Od0uQX8MI8/uBH44GD6HOCOGnzasYzM288DxivPYnr8r0c3Ah8Y3HXxTuD5qnp63EW1kOT7935elOQUpv99LreLEwZ9+DTwSFV9/CCbLfvzOkw/x31Ox/40ydejDvL4gyR/AkxW1Y1Mv+F/leQxpj/QOm98FS/MkP38zSRnAXuY7ucFYyt4EZJcw/SdBmuT7Ab+CFgDUFWXA7cwfcfFY8D/AB8aT6WLN0RfzwF+I8ke4H+B85bhxQnAacD5wNeTPDBY9nvARujqvA7Tz7GeUx9VIEmdWm5DNJKkIRnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVP/D6kgXA27yU18AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(score_init[change_log == 2] - score_best[change_log == 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"label\"].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2, 3)\n",
      "(1, 3, 2)\n",
      "(2, 1, 3)\n",
      "(2, 3, 1)\n",
      "(3, 1, 2)\n",
      "(3, 2, 1)\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "for x in itertools.permutations([1, 2, 3]):\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>table_name</th>\n",
       "      <th>column_index</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Product_michaelirvine.com_September2020_CTA.js...</td>\n",
       "      <td>4</td>\n",
       "      <td>currency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Product_michaelirvine.com_September2020_CTA.js...</td>\n",
       "      <td>6</td>\n",
       "      <td>Date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Product_heavenlybia.com_September2020_CTA.json.gz</td>\n",
       "      <td>3</td>\n",
       "      <td>currency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Product_heavenlybia.com_September2020_CTA.json.gz</td>\n",
       "      <td>5</td>\n",
       "      <td>Date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Product_magpieandlondon.com_September2020_CTA....</td>\n",
       "      <td>3</td>\n",
       "      <td>price</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32999</th>\n",
       "      <td>MusicRecording_journey-north.com_September2020...</td>\n",
       "      <td>3</td>\n",
       "      <td>MusicAlbum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33000</th>\n",
       "      <td>MusicRecording_journey-north.com_September2020...</td>\n",
       "      <td>4</td>\n",
       "      <td>MusicArtistAT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33001</th>\n",
       "      <td>SportsEvent_4smenssoccer.com_September2020_CTA...</td>\n",
       "      <td>2</td>\n",
       "      <td>SportsTeam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33002</th>\n",
       "      <td>SportsEvent_4smenssoccer.com_September2020_CTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>SportsEvent/name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33003</th>\n",
       "      <td>JobPosting_ksl.com_September2020_CTA.json.gz</td>\n",
       "      <td>5</td>\n",
       "      <td>EducationalOccupationalCredential</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33004 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              table_name  column_index  \\\n",
       "0      Product_michaelirvine.com_September2020_CTA.js...             4   \n",
       "1      Product_michaelirvine.com_September2020_CTA.js...             6   \n",
       "2      Product_heavenlybia.com_September2020_CTA.json.gz             3   \n",
       "3      Product_heavenlybia.com_September2020_CTA.json.gz             5   \n",
       "4      Product_magpieandlondon.com_September2020_CTA....             3   \n",
       "...                                                  ...           ...   \n",
       "32999  MusicRecording_journey-north.com_September2020...             3   \n",
       "33000  MusicRecording_journey-north.com_September2020...             4   \n",
       "33001  SportsEvent_4smenssoccer.com_September2020_CTA...             2   \n",
       "33002  SportsEvent_4smenssoccer.com_September2020_CTA...             0   \n",
       "33003       JobPosting_ksl.com_September2020_CTA.json.gz             5   \n",
       "\n",
       "                                   label  \n",
       "0                               currency  \n",
       "1                                   Date  \n",
       "2                               currency  \n",
       "3                                   Date  \n",
       "4                                  price  \n",
       "...                                  ...  \n",
       "32999                         MusicAlbum  \n",
       "33000                      MusicArtistAT  \n",
       "33001                         SportsTeam  \n",
       "33002                   SportsEvent/name  \n",
       "33003  EducationalOccupationalCredential  \n",
       "\n",
       "[33004 rows x 3 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"/data/yongkang/TU/SOTAB/CTA_training_small_gt.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"/data/yongkang/TU/SOTAB/comma_train_sotab.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>table_id</th>\n",
       "      <th>column_index</th>\n",
       "      <th>label</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>Homer William Bedell Stanford W. Stanford, Dav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>Unlimited [Historical Fiction Book] ↠ The Odys...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>2</td>\n",
       "      <td>36</td>\n",
       "      <td>Paperback, Hardcover, Paperback, None, Paperba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>2020-06-27T15:00:54+00:00, 2020-06-01T00:07:00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>418, 309, 139, 491, 461, 272, 211, 365, 153, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91315</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>In the hope that Griffin has left materials an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91316</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>BANANA FISH, None, None, None, None, None, Non...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91317</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0, nan, nan, nan, nan, nan, nan, nan, nan, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91318</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>4</td>\n",
       "      <td>33</td>\n",
       "      <td>None, 2019-11-06, 2019-10-23, 2019-10-23, 1986...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91319</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>5</td>\n",
       "      <td>-1</td>\n",
       "      <td>6, 5, 3, 1, 19, 18, 21, 11, 20, 19, 15, 11, 2,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91320 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                table_id  column_index  label  \\\n",
       "0      Book_1carpetcleaning.co.uk_September2020_CTA.j...             0      5   \n",
       "1      Book_1carpetcleaning.co.uk_September2020_CTA.j...             1     10   \n",
       "2      Book_1carpetcleaning.co.uk_September2020_CTA.j...             2     36   \n",
       "3      Book_1carpetcleaning.co.uk_September2020_CTA.j...             3     -1   \n",
       "4      Book_1carpetcleaning.co.uk_September2020_CTA.j...             4     -1   \n",
       "...                                                  ...           ...    ...   \n",
       "91315      TVEpisode_yidio.com_September2020_CTA.json.gz             1     19   \n",
       "91316      TVEpisode_yidio.com_September2020_CTA.json.gz             2     35   \n",
       "91317      TVEpisode_yidio.com_September2020_CTA.json.gz             3     -1   \n",
       "91318      TVEpisode_yidio.com_September2020_CTA.json.gz             4     33   \n",
       "91319      TVEpisode_yidio.com_September2020_CTA.json.gz             5     -1   \n",
       "\n",
       "                                                    data  \n",
       "0      Homer William Bedell Stanford W. Stanford, Dav...  \n",
       "1      Unlimited [Historical Fiction Book] ↠ The Odys...  \n",
       "2      Paperback, Hardcover, Paperback, None, Paperba...  \n",
       "3      2020-06-27T15:00:54+00:00, 2020-06-01T00:07:00...  \n",
       "4      418, 309, 139, 491, 461, 272, 211, 365, 153, 2...  \n",
       "...                                                  ...  \n",
       "91315  In the hope that Griffin has left materials an...  \n",
       "91316  BANANA FISH, None, None, None, None, None, Non...  \n",
       "91317  1.0, nan, nan, nan, nan, nan, nan, nan, nan, n...  \n",
       "91318  None, 2019-11-06, 2019-10-23, 2019-10-23, 1986...  \n",
       "91319  6, 5, 3, 1, 19, 18, 21, 11, 20, 19, 15, 11, 2,...  \n",
       "\n",
       "[91320 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['table_id', 'column_index', 'label', 'data'], dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = ['table_id', 'col_idx', 'class_id', 'data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>table_id</th>\n",
       "      <th>col_idx</th>\n",
       "      <th>class_id</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>Homer William Bedell Stanford W. Stanford, Dav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>Unlimited [Historical Fiction Book] ↠ The Odys...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>2</td>\n",
       "      <td>36</td>\n",
       "      <td>Paperback, Hardcover, Paperback, None, Paperba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>2020-06-27T15:00:54+00:00, 2020-06-01T00:07:00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Book_1carpetcleaning.co.uk_September2020_CTA.j...</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>418, 309, 139, 491, 461, 272, 211, 365, 153, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91315</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>In the hope that Griffin has left materials an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91316</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>BANANA FISH, None, None, None, None, None, Non...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91317</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0, nan, nan, nan, nan, nan, nan, nan, nan, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91318</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>4</td>\n",
       "      <td>33</td>\n",
       "      <td>None, 2019-11-06, 2019-10-23, 2019-10-23, 1986...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91319</th>\n",
       "      <td>TVEpisode_yidio.com_September2020_CTA.json.gz</td>\n",
       "      <td>5</td>\n",
       "      <td>-1</td>\n",
       "      <td>6, 5, 3, 1, 19, 18, 21, 11, 20, 19, 15, 11, 2,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91320 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                table_id  col_idx  class_id  \\\n",
       "0      Book_1carpetcleaning.co.uk_September2020_CTA.j...        0         5   \n",
       "1      Book_1carpetcleaning.co.uk_September2020_CTA.j...        1        10   \n",
       "2      Book_1carpetcleaning.co.uk_September2020_CTA.j...        2        36   \n",
       "3      Book_1carpetcleaning.co.uk_September2020_CTA.j...        3        -1   \n",
       "4      Book_1carpetcleaning.co.uk_September2020_CTA.j...        4        -1   \n",
       "...                                                  ...      ...       ...   \n",
       "91315      TVEpisode_yidio.com_September2020_CTA.json.gz        1        19   \n",
       "91316      TVEpisode_yidio.com_September2020_CTA.json.gz        2        35   \n",
       "91317      TVEpisode_yidio.com_September2020_CTA.json.gz        3        -1   \n",
       "91318      TVEpisode_yidio.com_September2020_CTA.json.gz        4        33   \n",
       "91319      TVEpisode_yidio.com_September2020_CTA.json.gz        5        -1   \n",
       "\n",
       "                                                    data  \n",
       "0      Homer William Bedell Stanford W. Stanford, Dav...  \n",
       "1      Unlimited [Historical Fiction Book] ↠ The Odys...  \n",
       "2      Paperback, Hardcover, Paperback, None, Paperba...  \n",
       "3      2020-06-27T15:00:54+00:00, 2020-06-01T00:07:00...  \n",
       "4      418, 309, 139, 491, 461, 272, 211, 365, 153, 2...  \n",
       "...                                                  ...  \n",
       "91315  In the hope that Griffin has left materials an...  \n",
       "91316  BANANA FISH, None, None, None, None, None, Non...  \n",
       "91317  1.0, nan, nan, nan, nan, nan, nan, nan, nan, n...  \n",
       "91318  None, 2019-11-06, 2019-10-23, 2019-10-23, 1986...  \n",
       "91319  6, 5, 3, 1, 19, 18, 21, 11, 20, 19, 15, 11, 2,...  \n",
       "\n",
       "[91320 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tokenizer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtokenizer\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tokenizer' is not defined"
     ]
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from functools import reduce\n",
    "# import operator\n",
    "# random_sample = False\n",
    "# max_unlabeled = 8\n",
    "\n",
    "# data_list = []\n",
    "\n",
    "# df['class_id'] = df['class_id'].astype(int)\n",
    "# df.drop(df[(df['data'].isna()) & (df['class_id'] == -1)].index, inplace=True)\n",
    "# df['col_idx'] = df['col_idx'].astype(int)\n",
    "# df['data'] = df['data'].astype(str)\n",
    "\n",
    "# num_tables = len(df.groupby(\"table_id\"))\n",
    " \n",
    "\n",
    "# # df.drop(df[(df['data'] == '') & (df['class_id'] == -1)].index, inplace=True)\n",
    "# total_num_cols = 0\n",
    "# for i, (index, group_df) in enumerate(df.groupby(\"table_id\")):\n",
    "#     #     break\n",
    "#     labeled_columns = group_df[group_df['class_id'] > -1]\n",
    "#     unlabeled_columns = group_df[group_df['class_id'] == -1]\n",
    "#     num_unlabeled = min(max(max_unlabeled-len(labeled_columns), 0), len(unlabeled_columns))\n",
    "#     unlabeled_columns = unlabeled_columns.sample(num_unlabeled) if random_sample else unlabeled_columns[0:num_unlabeled]\n",
    "#     # group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns.sample(min(10-len(labeled_columns), len(unlabeled_columns)))])\n",
    "#     # group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns[0:min(max(10-len(labeled_columns), 0), len(unlabeled_columns))]])\n",
    "#     group_df = pd.concat([group_df[group_df['class_id'] > -1], unlabeled_columns]) # TODO\n",
    "#     group_df.sort_values(by=['col_idx'], inplace=True)\n",
    "\n",
    "#     if max_length <= 128:\n",
    "#         cur_maxlen = min(max_length, 512 // len(list(group_df[\"class_id\"].values)) - 1)\n",
    "#     else:\n",
    "#         cur_maxlen = max(1, max_length // len(list(group_df[\"class_id\"].values)) - 1)\n",
    "        \n",
    "#     token_ids_list = group_df[\"data\"].apply(lambda x: tokenizer.encode(\n",
    "#         tokenizer.cls_token + \" \" + x, add_special_tokens=False, max_length=cur_maxlen, truncation=True)).tolist(\n",
    "#         )\n",
    "#     token_ids = torch.LongTensor(reduce(operator.add,\n",
    "#                                         token_ids_list)).to(device)\n",
    "#     cls_index_list = [0] + np.cumsum(\n",
    "#         np.array([len(x) for x in token_ids_list])).tolist()[:-1]\n",
    "#     for cls_index in cls_index_list:\n",
    "#         assert token_ids[\n",
    "#             cls_index] == tokenizer.cls_token_id, \"cls_indexes validation\"\n",
    "#     cls_indexes = torch.LongTensor(cls_index_list).to(device)\n",
    "#     class_ids = torch.LongTensor(\n",
    "#         group_df[\"class_id\"].values).to(device)\n",
    "#     data_list.append(\n",
    "#         [index,\n",
    "#             len(group_df), token_ids, class_ids, cls_indexes])\n",
    "# table_df = pd.DataFrame(data_list,\n",
    "#                                 columns=[\n",
    "#                                     \"table_id\", \"num_col\", \"data_tensor\",\n",
    "#                                     \"label_tensor\", \"cls_indexes\"\n",
    "#                                 ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dirpath = \"/data/yongkang/TU/SOTAB\"\n",
    "df = pd.read_csv(os.path.join(base_dirpath, \"CTA_training_small_gt.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'data_folder' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [65]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m table_files \u001b[38;5;241m=\u001b[39m [f \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m os\u001b[38;5;241m.\u001b[39mlistdir(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(base_dirpath, \u001b[43mdata_folder\u001b[49m)) \u001b[38;5;28;01mif\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m gt_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtable_name\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mvalues]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'data_folder' is not defined"
     ]
    }
   ],
   "source": [
    "table_files = [f for f in os.listdir(os.path.join(base_dirpath, data_folder)) if f in gt_df['table_name'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 64])"
      ]
     },
     "execution_count": 450,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"data\"].T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 2])"
      ]
     },
     "execution_count": 454,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_indexes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 477,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save target training embeddings\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "num_single_fail = 0\n",
    "num_good_context = 0\n",
    "# training_batches = []\n",
    "# training_embeddings = []\n",
    "# training_data = []\n",
    "# training_col_mask = []\n",
    "training_labels = []\n",
    "class_context = defaultdict(list)\n",
    "for batch_idx, batch in enumerate(train_dataloader_iter):\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    if 1 in target_col_mask:\n",
    "        cls_indexes = torch.tensor([0, 0]).reshape(1, -1).to(device)\n",
    "        # logits_temp, embs = model(batch[\"data\"].T[target_col_mask==0].reshape(1, -1), cls_indexes=cls_indexes, get_enc=True)  \n",
    "        # training_embeddings.append(embs.detach().cpu())\n",
    "        # training_data.append(batch[\"data\"].T.cpu())\n",
    "        # training_col_mask.append(target_col_mask.cpu())\n",
    "        training_labels.append(batch[\"label\"].item())\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3463"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(training_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([768])"
      ]
     },
     "execution_count": 458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_embeddings[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_embeddings_all = torch.vstack(training_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = F.cosine_similarity(training_embeddings[0].reshape(-1), training_embeddings_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.return_types.topk(\n",
       "values=tensor([1.0000, 0.9984, 0.9981, 0.9971, 0.9967]),\n",
       "indices=tensor([   0, 2614, 2017, 2702,  160]))"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.topk(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([768])"
      ]
     },
     "execution_count": 467,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_embs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_values, sim_indices = F.cosine_similarity(target_embs.reshape(-1), training_embeddings_all).topk(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.6808, 0.6634, 0.6417, 0.6390, 0.6309])"
      ]
     },
     "execution_count": 475,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sim_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 478,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 57, 3, 0, 0]"
      ]
     },
     "execution_count": 478,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[training_labels[i] for i in sim_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data \n",
    "training_col_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 508,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols_test = []\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    num_cols_test.append(batch[\"target_col_mask\"].max().item()+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.908755760368663\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(num_cols_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 511,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAS70lEQVR4nO3df6zd9X3f8eerOISEpDEkd5ZrOzNTLDq0KUBvGT+iSMOlAprFbOoIUResyJ0jlUZhndqR7Y8pUjWlUtWkVBOrB2lMxyCEgHAzlIYZ2ixqIL0GCgkmwmE4vg7g2yRAk6xjpO/9cT7+cjDX917b93vOvdznQ/rqfL6f74/zNsJ+nc/n+z3fk6pCkiSAnxp3AZKkpcNQkCR1DAVJUsdQkCR1DAVJUmfVuAs4Ee94xztq48aN4y5DkpaVPXv2/HVVTcy2bVmHwsaNG5mamhp3GZK0rCTZf7RtTh9Jkjq9hUKSM5M8MrS8mOTaJKcnuTfJk+31tLZ/klyfZF+SR5Oc21dtkqTZ9RYKVfWtqjq7qs4Gfg74MXAXcB2wu6o2AbvbOsBlwKa2bAdu6Ks2SdLsRjV9tBn4dlXtB7YAO1v/TuCK1t4C3FwDDwCrk6wdUX2SJEYXClcBt7b2mqp6prWfBda09jrgwNAx063vVZJsTzKVZGpmZqaveiVpReo9FJKcDLwf+PyR22rwNL5jeiJfVe2oqsmqmpyYmPWOKknScRrFSOEy4KGqeq6tP3d4Wqi9Hmr9B4ENQ8etb32SpBEZRSh8kFemjgB2AVtbeytw91D/1e0upPOBF4ammSRJI9Drl9eSnApcAnxkqPuTwO1JtgH7gStb/z3A5cA+BncqfbjP2iRJr9VrKFTVj4C3H9H3PQZ3Ix25bwHX9FmPJC0F6za8k+9OH5h/xzn8zPoNHDzwnUWq6BXL+jEXkrQcfXf6AB/4w784oXN87iMXLlI1r+ZjLiRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktQxFCRJHUNBktTpNRSSrE5yR5InkuxNckGS05Pcm+TJ9npa2zdJrk+yL8mjSc7tszZJ0mv1PVL4feBLVfWzwLuBvcB1wO6q2gTsbusAlwGb2rIduKHn2iRJR+gtFJK8DXgvcBNAVb1UVc8DW4CdbbedwBWtvQW4uQYeAFYnWdtXfZKk1+pzpHAGMAP8UZKHk9yY5FRgTVU90/Z5FljT2uuAA0PHT7e+V0myPclUkqmZmZkey5eklafPUFgFnAvcUFXnAD/ilakiAKqqgDqWk1bVjqqarKrJiYmJRStWktRvKEwD01X1YFu/g0FIPHd4Wqi9HmrbDwIbho5f3/okSSPSWyhU1bPAgSRntq7NwOPALmBr69sK3N3au4Cr211I5wMvDE0zSZJGYFXP5/8ocEuSk4GngA8zCKLbk2wD9gNXtn3vAS4H9gE/bvtKkkao11CoqkeAyVk2bZ5l3wKu6bMeSdLc/EazJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKnTaygkeTrJY0keSTLV+k5Pcm+SJ9vraa0/Sa5Psi/Jo0nO7bM2SdJrjWKk8E+r6uyqmmzr1wG7q2oTsLutA1wGbGrLduCGEdQmSRoyjumjLcDO1t4JXDHUf3MNPACsTrJ2DPVJ0orVdygU8OUke5Jsb31rquqZ1n4WWNPa64ADQ8dOt75XSbI9yVSSqZmZmb7qlqQVaVXP539PVR1M8veAe5M8MbyxqipJHcsJq2oHsANgcnLymI6VJM2t15FCVR1sr4eAu4DzgOcOTwu110Nt94PAhqHD17c+SdKI9BYKSU5N8tbDbeAXgW8Au4CtbbetwN2tvQu4ut2FdD7wwtA0kyRpBPqcPloD3JXk8Pv896r6UpK/BG5Psg3YD1zZ9r8HuBzYB/wY+HCPtUmSZtFbKFTVU8C7Z+n/HrB5lv4CrumrHknS/PxGsySpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjq9h0KSk5I8nOSLbf2MJA8m2Zfkc0lObv1vbOv72vaNfdcmSXq1UYwUPgbsHVr/HeBTVfUu4AfAtta/DfhB6/9U20+SNEK9hkKS9cAvATe29QAXA3e0XXYCV7T2lrZO27657S9JGpEFhUKSixbSN4tPA78F/F1bfzvwfFW93NangXWtvQ44ANC2v9D2P/J9tyeZSjI1MzOzkPIlSQu00JHCHyywr5PkfcChqtpzzFXNoap2VNVkVU1OTEws5qklacVbNdfGJBcAFwITSX5jaNNPAyfNc+6LgPcnuRw4pR3z+8DqJKvaaGA9cLDtfxDYAEwnWQW8DfjeMf55JEknYL6RwsnAWxiEx1uHlheBX57rwKr6eFWtr6qNwFXAfVX1K8D9Q8duBe5u7V1tnbb9vqqqY/rTSJJOyJwjhar6c+DPk3y2qvYv0nv+O+C2JL8NPAzc1PpvAv44yT7g+wyCRJI0QnOGwpA3JtkBbBw+pqouXsjBVfVnwJ+19lPAebPs87fAv1xgPZKkHiw0FD4P/BcGt5b+pL9yJEnjtNBQeLmqbui1EknS2C30ltQ/SfJrSdYmOf3w0mtlkqSRW+hI4fBdQb851FfAP1jcciRJ47SgUKiqM/ouRJI0fgsKhSRXz9ZfVTcvbjmSpHFa6PTRzw+1TwE2Aw8BhoIkvY4sdProo8PrSVYDt/VRkCRpfI730dk/ArzOIEmvMwu9pvAnDO42gsGD8P4hcHtfRUmSxmOh1xR+d6j9MrC/qqZ7qEeSNEYLmj5qD8Z7gsETUk8DXuqzKEnSeCz0l9euBL7O4IF1VwIPJpnz0dmSpOVnodNH/wH4+ao6BJBkAvifvPJby5Kk14GF3n30U4cDofneMRwrSVomFjpS+FKSPwVubesfAO7ppyRJ0rjM9xvN7wLWVNVvJvkXwHvapq8Bt/RdnCRptOYbKXwa+DhAVd0J3AmQ5B+3bf+sx9okSSM233WBNVX12JGdrW9jLxVJksZmvlBYPce2Ny1iHZKkJWC+UJhK8q+P7Ezyq8CefkqSJI3LfNcUrgXuSvIrvBICk8DJwD+f68AkpwBfAd7Y3ueOqvqPSc5g8ITVt7dzfqiqXkryRgaP4v45Bre8fqCqnj6eP5Qk6fjMOVKoqueq6kLgE8DTbflEVV1QVc/Oc+7/C1xcVe8GzgYuTXI+8DvAp6rqXcAPgG1t/23AD1r/p9p+kqQRWuizj+6vqj9oy30LPKaq6odt9Q1tKeBiXvkm9E7gitbe0tZp2zcnyULeS5K0OHr9VnKSk5I8AhwC7gW+DTxfVS+3XaaBda29DjgA0La/wGCK6chzbk8ylWRqZmamz/IlacXpNRSq6idVdTawHjgP+NlFOOeOqpqsqsmJiYkTPZ0kachInl9UVc8D9wMXAKuTHL7AvR442NoHgQ0AbfvbGFxwliSNSG+hkGSi/ZYzSd4EXALsZRAOhx+7vRW4u7V3tXXa9vuqqpAkjcxCH4h3PNYCO5OcxCB8bq+qLyZ5HLgtyW8DDwM3tf1vAv44yT7g+8BVPdYmSZpFb6FQVY8C58zS/xSD6wtH9v8tgx/xkSSNib+JIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnq9BYKSTYkuT/J40m+meRjrf/0JPcmebK9ntb6k+T6JPuSPJrk3L5qkyTNrs+RwsvAv62qs4DzgWuSnAVcB+yuqk3A7rYOcBmwqS3bgRt6rE2SNIveQqGqnqmqh1r7b4C9wDpgC7Cz7bYTuKK1twA318ADwOoka/uqT5L0WiO5ppBkI3AO8CCwpqqeaZueBda09jrgwNBh063vyHNtTzKVZGpmZqa/oiVpBeo9FJK8BfgCcG1VvTi8raoKqGM5X1XtqKrJqpqcmJhYxEolSb2GQpI3MAiEW6rqztb93OFpofZ6qPUfBDYMHb6+9UmSRqTPu48C3ATsrarfG9q0C9ja2luBu4f6r253IZ0PvDA0zSRJGoFVPZ77IuBDwGNJHml9/x74JHB7km3AfuDKtu0e4HJgH/Bj4MM91iZJmkVvoVBVXwVylM2bZ9m/gGv6qkeSND+/0SxJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6vQWCkk+k+RQkm8M9Z2e5N4kT7bX01p/klyfZF+SR5Oc21ddkqSj63Ok8Fng0iP6rgN2V9UmYHdbB7gM2NSW7cANPdYlSTqK3kKhqr4CfP+I7i3AztbeCVwx1H9zDTwArE6ytq/aJEmzG/U1hTVV9UxrPwusae11wIGh/aZbnyRphMZ2obmqCqhjPS7J9iRTSaZmZmZ6qEySVq5Rh8Jzh6eF2uuh1n8Q2DC03/rW9xpVtaOqJqtqcmJiotdiJWmlGXUo7AK2tvZW4O6h/qvbXUjnAy8MTTNJkkakz1tSbwW+BpyZZDrJNuCTwCVJngR+oa0D3AM8BewD/ivwa33VJS0X6za8kyQntKzb8M7XTR0ajVV9nbiqPniUTZtn2beAa/qqRVqOvjt9gA/84V+c0Dk+95ELXzd1aDT8RrNed/xkKx2/3kYK0rj4yVY6fo4UJK0YjiLn50hB0orhKHJ+jhQkSR1DQYvGobm0/Dl9pEXj0Fxa/hwpSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6hoIkqWMoSJI6SyoUklya5FtJ9iW5btz1SNJKs2RCIclJwH8GLgPOAj6Y5KzxViVJK8uSCQXgPGBfVT1VVS8BtwFb+nozfzpSkl4rVTXuGgBI8svApVX1q239Q8A/qapfP2K/7cD2tnom8K3jfMt3AH99nMeOw3KqdznVCsur3uVUKyyvepdTrXBi9f79qpqYbcOy+43mqtoB7DjR8ySZqqrJRShpJJZTvcupVlhe9S6nWmF51bucaoX+6l1K00cHgQ1D6+tbnyRpRJZSKPwlsCnJGUlOBq4Cdo25JklaUZbM9FFVvZzk14E/BU4CPlNV3+zxLU94CmrEllO9y6lWWF71LqdaYXnVu5xqhZ7qXTIXmiVJ47eUpo8kSWNmKEiSOisuFJJ8JsmhJN8Ydy3zSbIhyf1JHk/yzSQfG3dNc0lySpKvJ/mrVu8nxl3TfJKclOThJF8cdy3zSfJ0kseSPJJkatz1zCXJ6iR3JHkiyd4kF4y7pqNJcmb7b3p4eTHJteOu62iS/Jv29+sbSW5Ncsqinn+lXVNI8l7gh8DNVfWPxl3PXJKsBdZW1UNJ3grsAa6oqsfHXNqskgQ4tap+mOQNwFeBj1XVA2Mu7aiS/AYwCfx0Vb1v3PXMJcnTwGRVLfkvWCXZCfyvqrqx3U345qp6fsxlzas9bucggy/O7h93PUdKso7B36uzqur/JLkduKeqPrtY77HiRgpV9RXg++OuYyGq6pmqeqi1/wbYC6wbb1VHVwM/bKtvaMuS/dSRZD3wS8CN467l9STJ24D3AjcBVNVLyyEQms3At5diIAxZBbwpySrgzcB3F/PkKy4UlqskG4FzgAfHXMqc2nTMI8Ah4N6qWsr1fhr4LeDvxlzHQhXw5SR72uNelqozgBngj9rU3I1JTh13UQt0FXDruIs4mqo6CPwu8B3gGeCFqvryYr6HobAMJHkL8AXg2qp6cdz1zKWqflJVZzP4Rvp5SZbkFF2S9wGHqmrPuGs5Bu+pqnMZPEn4mjYVuhStAs4Fbqiqc4AfAUv+Ufhtmuv9wOfHXcvRJDmNwYNCzwB+Bjg1yb9azPcwFJa4Njf/BeCWqrpz3PUsVJsuuB+4dMylHM1FwPvbPP1twMVJ/tt4S5pb+5RIVR0C7mLwZOGlaBqYHhol3sEgJJa6y4CHquq5cRcyh18A/ndVzVTV/wPuBC5czDcwFJawduH2JmBvVf3euOuZT5KJJKtb+03AJcATYy3qKKrq41W1vqo2MpgyuK+qFvUT12JKcmq72YA2FfOLwJK8g66qngUOJDmzdW0GluTNEUf4IEt46qj5DnB+kje3fx82M7jWuGhWXCgkuRX4GnBmkukk28Zd0xwuAj7E4FPs4dvlLh93UXNYC9yf5FEGz7K6t6qW/K2ey8Qa4KtJ/gr4OvA/qupLY65pLh8Fbmn/L5wN/KfxljO3FrSXMPjkvWS10dcdwEPAYwz+DV/Ux12suFtSJUlHt+JGCpKkozMUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1Pn/TElIKkHiTa8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(num_cols_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols_train = []\n",
    "for batch_idx, batch in enumerate(train_dataloader_iter):\n",
    "    num_cols_train.append(batch[\"target_col_mask\"].max().item()+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 512,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 512,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAD4CAYAAAAdIcpQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQGklEQVR4nO3df+xddX3H8edLKv5AHSV0Te2PlW2NGXMZkooIxqhMBOYEF4eQTRqiK8nAyFxc0P3BpjHxD+eMi2NW6CgZgogQcWvEDpnOOJAWGb8NHYJtKbSKA38sc+B7f9zTeFe/7ef7/fbe7/n+eD6Sm3vu+/y47xNSXt/zOeeek6pCkqSDeU7fDUiSZj/DQpLUZFhIkpoMC0lSk2EhSWpa1HcD43D00UfX6tWr+25DkuaUbdu2fa+qlkw0b16GxerVq9m6dWvfbUjSnJLk0QPNcxhKktRkWEiSmgwLSVKTYSFJajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhI0iy1fOUqkkzptXzlqrH0Mi9v9yFJ88FjO3fw9k99Y0rrfPaCk8bSi0cWkqQmw0KS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpybCQJDUZFpKkJsNCktRkWEiSmgwLSVKTYSFJajIsJElNhoUkqWlsYZFkZZJbk9yf5L4k7+nqRyXZkuSh7n1xV0+STyTZnuTuJMcPbWtdt/xDSdaNq2dJ0sTGeWTxDPBnVXUscCJwYZJjgUuAW6pqDXBL9xngdGBN91oPXAaDcAEuBV4FnABcui9gJEkzY2xhUVW7q+rObvqHwAPAcuBMYFO32CbgrG76TOCqGrgNODLJMuBNwJaqerKqfgBsAU4bV9+SpF80I+cskqwGXgHcDiytqt3drMeBpd30cmDH0Go7u9qB6vt/x/okW5Ns3bt372h3QJIWuLGHRZIXAZ8HLq6qp4fnVVUBNYrvqaoNVbW2qtYuWbJkFJuUJHXGGhZJnssgKK6uqhu68hPd8BLd+56uvgtYObT6iq52oLokaYaM82qoAFcAD1TVx4Zm3QTsu6JpHfCFofp53VVRJwJPdcNVNwOnJlncndg+tatJkmbIojFu+2TgHcA9Se7qah8APgJcl+SdwKPA2d28zcAZwHbgJ8D5AFX1ZJIPAXd0y32wqp4cY9+SpP2MLSyq6utADjD7lAmWL+DCA2xrI7BxdN1JkqbCX3BLkpoMC0lSk2EhSWoyLCRJTYaFJKnJsJAkNRkWkqQmw0KS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpybCQJDUZFpKkJsNCktRkWEiSmgwLSVKTYSFJajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhIkpoMC0lSk2EhSWoyLCRJTYaFJKnJsJAkNRkWkqQmw0KS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpaWxhkWRjkj1J7h2q/WWSXUnu6l5nDM17f5LtSb6d5E1D9dO62vYkl4yrX0nSgY3zyOJK4LQJ6n9TVcd1r80ASY4FzgF+s1vn75IcluQw4JPA6cCxwLndspKkGbRoXBuuqq8lWT3Jxc8Erq2q/wG+k2Q7cEI3b3tVPQyQ5Npu2ftH3a8k6cD6OGdxUZK7u2GqxV1tObBjaJmdXe1AdUnSDJrpsLgM+DXgOGA38Nej2nCS9Um2Jtm6d+/eUW1WksQMh0VVPVFVz1bVz4BP8/Ohpl3AyqFFV3S1A9Un2vaGqlpbVWuXLFky+uYlaQGb0bBIsmzo41uBfVdK3QSck+R5SY4B1gDfBO4A1iQ5JsnhDE6C3zSTPUuSxniCO8k1wOuAo5PsBC4FXpfkOKCAR4ALAKrqviTXMThx/QxwYVU9223nIuBm4DBgY1XdN66eJUkTG+fVUOdOUL7iIMt/GPjwBPXNwOYRtiZJmiJ/wS1JajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhIkpoMC0lSk2EhSWoyLCRJTYaFJKnJsJAkNRkWkqQmw0KS1DSpsEhy8mRqkqT5abJHFn87yZokaR466MOPkrwaOAlYkuS9Q7NewuDJdZKkBaD1pLzDgRd1y714qP408LZxNSVJml0OGhZV9VXgq0murKpHZ6gnSdIsM9lncD8vyQZg9fA6VfWGcTQlSZpdJhsWnwP+HrgceHZ87UiSZqPJhsUzVXXZWDuRJM1ak7109otJ/iTJsiRH7XuNtTNJ0qwx2SOLdd37+4ZqBfzqaNuRJM1GkwqLqjpm3I1IkmavSYVFkvMmqlfVVaNtR5I0G012GOqVQ9PPB04B7gQMC0laACY7DPXu4c9JjgSuHUdDkqTZZ7q3KP8x4HkMSVogJnvO4osMrn6CwQ0EfwO4blxNSZJml8mes/jo0PQzwKNVtXMM/UiSZqFJDUN1NxR8kMGdZxcDPx1nU5Kk2WWyT8o7G/gm8AfA2cDtSbxFuSQtEJMdhvoL4JVVtQcgyRLgX4Drx9WYJGn2mOzVUM/ZFxSd709hXUnSHDfZI4svJbkZuKb7/HZg83hakiTNNq1ncP86sLSq3pfk94HXdLP+Hbh63M1JkmaH1pHFx4H3A1TVDcANAEl+q5v3e2PsTZI0S7TOOyytqnv2L3a11QdbMcnGJHuS3DtUOyrJliQPde+Lu3qSfCLJ9iR3Jzl+aJ113fIPJVk30XdJksarFRZHHmTeCxrrXgmctl/tEuCWqloD3NJ9BjgdWNO91gOXwSBcgEuBVwEnAJfuCxhJ0sxphcXWJH+8fzHJu4BtB1uxqr4GPLlf+UxgUze9CThrqH5VDdwGHJlkGfAmYEtVPVlVPwC28IsBJEkas9Y5i4uBG5P8IT8Ph7XA4cBbp/F9S6tqdzf9OLC0m14O7BhabmdXO1D9FyRZz+CohFWrVk2jNUnSgRw0LKrqCeCkJK8HXt6V/7mqvnKoX1xVlaTaS056exuADQBr164d2XYlSZN/nsWtwK0j+L4nkiyrqt3dMNO+H/rtAlYOLbeiq+0CXrdf/V9H0IckaQpm+lfYNwH7rmhaB3xhqH5ed1XUicBT3XDVzcCpSRZ3J7ZP7WqSpBk02V9wT1mSaxgcFRydZCeDq5o+AlyX5J3AowxuSgiDX4OfAWwHfgKcD1BVTyb5EHBHt9wHq2r/k+aSpDEbW1hU1bkHmHXKBMsWcOEBtrMR2DjC1iRJU+TNACVJTYaFJKnJsJAkNRkWkqQmw0KS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpybCQJDUZFpKkJsNCktRkWEiSmgwLSVKTYSFJajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhIkpoMC0lSk2EhSWoyLCRJTYaFJKnJsJAkNRkWkqQmw0KS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpybCQJDUZFpKkJsNCktRkWEiSmnoJiySPJLknyV1Jtna1o5JsSfJQ9764qyfJJ5JsT3J3kuP76FmSFrI+jyxeX1XHVdXa7vMlwC1VtQa4pfsMcDqwpnutBy6b8U4laYGbTcNQZwKbuulNwFlD9atq4DbgyCTLeuhPkhasvsKigC8n2ZZkfVdbWlW7u+nHgaXd9HJgx9C6O7va/5NkfZKtSbbu3bt3XH1L0oK0qKfvfU1V7Uryy8CWJA8Oz6yqSlJT2WBVbQA2AKxdu3ZK60qSDq6XI4uq2tW97wFuBE4Antg3vNS97+kW3wWsHFp9RVeTJM2QGQ+LJEckefG+aeBU4F7gJmBdt9g64Avd9E3Aed1VUScCTw0NV0mSZkAfw1BLgRuT7Pv+z1TVl5LcAVyX5J3Ao8DZ3fKbgTOA7cBPgPNnvmVJWthmPCyq6mHgtyeofx84ZYJ6ARfOQGuSpAOYTZfOSpJmKcNCktRkWEiSmgwLSVKTYSFJajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhIkpoMC0lSk2EhSWoyLCRJTYaFpElZvnIVSab0Wr5yVd9ta0T6ega3pDnmsZ07ePunvjGldT57wUlj6mY0lq9cxWM7d0xpnZeuWMmuHd8dU0ezl2EhacGajwE4Lg5DqXfTGd5wiEOaWR5ZqHfT+esOFu5feFIfPLKQxsCTwZpvPLKQxsCxcM03HllIkpoMC0lSk2EhSWoyLOYYLzOV1AdPcM8xXmYqqQ8eWUiSmgwLSVKTYSFJajIsJElNhoUkqcmwkCQ1GRaSpCbDQpLUZFhIkpoMC0lSk2EhSWoyLCRJTXMmLJKcluTbSbYnuaTvfiRpIZkTYZHkMOCTwOnAscC5SY7ttytJWjjmRFgAJwDbq+rhqvopcC1w5ri+bDrPjPB5EZLms1RV3z00JXkbcFpVvav7/A7gVVV10dAy64H13ceXAd8+hK88GvjeIaw/W8yX/QD3ZbaaL/syX/YDDm1ffqWqlkw0Y948/KiqNgAbRrGtJFurau0ottWn+bIf4L7MVvNlX+bLfsD49mWuDEPtAlYOfV7R1SRJM2CuhMUdwJokxyQ5HDgHuKnnniRpwZgTw1BV9UySi4CbgcOAjVV13xi/ciTDWbPAfNkPcF9mq/myL/NlP2BM+zInTnBLkvo1V4ahJEk9MiwkSU2GRSfJyiS3Jrk/yX1J3tN3T9OV5PlJvpnkP7p9+au+ezoUSQ5L8q0k/9R3L4ciySNJ7klyV5KtffdzKJIcmeT6JA8meSDJq/vuaTqSvKz777Hv9XSSi/vua7qS/Gn3b/7eJNckef7Itu05i4Eky4BlVXVnkhcD24Czqur+nlubsiQBjqiqHyV5LvB14D1VdVvPrU1LkvcCa4GXVNWb++5nupI8Aqytqjn/468km4B/q6rLuysUX1hV/9VzW4eku63QLgY/+H20736mKslyBv/Wj62q/05yHbC5qq4cxfY9suhU1e6qurOb/iHwALC8366mpwZ+1H18bveak38VJFkB/C5wed+9aCDJLwGvBa4AqKqfzvWg6JwC/OdcDIohi4AXJFkEvBB4bFQbNiwmkGQ18Arg9p5bmbZu6OYuYA+wparm6r58HPhz4Gc99zEKBXw5ybbu9jRz1THAXuAfuuHBy5Mc0XdTI3AOcE3fTUxXVe0CPgp8F9gNPFVVXx7V9g2L/SR5EfB54OKqerrvfqarqp6tquMY/Nr9hCQv77mlKUvyZmBPVW3ru5cReU1VHc/g7skXJnlt3w1N0yLgeOCyqnoF8GNgTj82oBtKewvwub57ma4kixncYPUY4KXAEUn+aFTbNyyGdOP7nweurqob+u5nFLrhgVuB03puZTpOBt7SjfVfC7whyT/229L0dX/5UVV7gBsZ3E15LtoJ7Bw6Wr2eQXjMZacDd1bVE303cgh+B/hOVe2tqv8FbgBOGtXGDYtOd1L4CuCBqvpY3/0ciiRLkhzZTb8AeCPwYK9NTUNVvb+qVlTVagZDBF+pqpH9pTSTkhzRXThBN2RzKnBvv11NT1U9DuxI8rKudAow5y4E2c+5zOEhqM53gROTvLD7/9kpDM69jsScuN3HDDkZeAdwTzfWD/CBqtrcX0vTtgzY1F3d8Rzguqqa05edzgNLgRsH/4ZZBHymqr7Ub0uH5N3A1d3wzcPA+T33M21deL8RuKDvXg5FVd2e5HrgTuAZ4FuM8NYfXjorSWpyGEqS1GRYSJKaDAtJUpNhIUlqMiwkSU2GhSSpybCQJDX9HxjxDWhJ8ms+AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(num_cols_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 494,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.977476176725383\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(num_cols_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************K:5; Threshold: 0.9; msp: 0.99, debias: 0.8****************************\n",
      "2 [0]\n",
      "Sim tensor(3175) Table [0, 1, 2, 3, 4, 5, 6]\n",
      "Sim tensor(3188) Table [0, 1, 2, 3, 4, 5, 6, 7]\n",
      "Sim tensor(2787) Table [0, 1, 2, 3, 4, 5, 6, 7]\n",
      "Sim tensor(1264) Table [0, 1, 2, 3, 4, 5, 6, 7]\n",
      "Sim tensor(3197) Table [0, 1, 2, 3, 4, 5, 6, 7]\n",
      "ts_micro_f1=1.0000, ts_macro_f1=1.0000\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "The shape of the mask [3] at index 0 does not match the shape of the indexed tensor [2] at index 0",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [490]\u001b[0m, in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     89\u001b[0m ts_macro_f1 \u001b[38;5;241m=\u001b[39m f1_score(labels_test\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mnumpy()\u001b[38;5;241m.\u001b[39mtolist(),\n\u001b[1;32m     90\u001b[0m                     ts_pred_list,\n\u001b[1;32m     91\u001b[0m                     average\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmacro\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mts_micro_f1=\u001b[39m\u001b[38;5;132;01m{:.4f}\u001b[39;00m\u001b[38;5;124m, ts_macro_f1=\u001b[39m\u001b[38;5;132;01m{:.4f}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(ts_micro_f1, ts_macro_f1))\n\u001b[0;32m---> 93\u001b[0m ts_pred_list \u001b[38;5;241m=\u001b[39m \u001b[43mlogits_test\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margmax\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     94\u001b[0m \u001b[43m                            \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpu\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdetach\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmask\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mnumpy()\u001b[38;5;241m.\u001b[39mtolist()\n\u001b[1;32m     95\u001b[0m ts_micro_f1 \u001b[38;5;241m=\u001b[39m f1_score(labels_test\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)[mask]\u001b[38;5;241m.\u001b[39mnumpy()\u001b[38;5;241m.\u001b[39mtolist(),\n\u001b[1;32m     96\u001b[0m                     ts_pred_list,\n\u001b[1;32m     97\u001b[0m                     average\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmicro\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     98\u001b[0m ts_macro_f1 \u001b[38;5;241m=\u001b[39m f1_score(labels_test\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)[mask]\u001b[38;5;241m.\u001b[39mnumpy()\u001b[38;5;241m.\u001b[39mtolist(),\n\u001b[1;32m     99\u001b[0m                     ts_pred_list,\n\u001b[1;32m    100\u001b[0m                     average\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmacro\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mIndexError\u001b[0m: The shape of the mask [3] at index 0 does not match the shape of the indexed tensor [2] at index 0"
     ]
    }
   ],
   "source": [
    "alpha = 0.125\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "\n",
    "for k in [5]:\n",
    "    for msp_threshold in [0.99]:\n",
    "        for threshold in [0.9]:\n",
    "            for debias_threshold in [0.8]:\n",
    "                print(f\"*********************K:{k}; Threshold: {threshold}; msp: {msp_threshold}, debias: {debias_threshold}****************************\")\n",
    "                model = model.to(device)\n",
    "                ft_embs_test = []\n",
    "                labels_test = []\n",
    "                logits_test = []\n",
    "                log = defaultdict(list)\n",
    "                num_cols = []\n",
    "                for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                    target_col_mask = batch[\"target_col_mask\"].T\n",
    "                    logits, target_embs = model(batch[\"data\"].T, cls_indexes=cls_indexes, get_enc=True)\n",
    "                    target_embs = target_embs.detach().cpu()\n",
    "                    logits =  reweight_logits(logits.detach().cpu(), class_weights)\n",
    "                    msp_init = logits.max().item()\n",
    "                    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                    max_msp = 0\n",
    "                    if 1 not in target_col_mask and msp_init < threshold:\n",
    "                        sim_values, sim_indices = F.cosine_similarity(target_embs.reshape(-1), training_embeddings_all).topk(k)\n",
    "                        print(batch_idx, target_col_mask.unique().tolist())\n",
    "                        for sim_i in sim_indices:\n",
    "                            new_data = training_data[sim_i].to(device)\n",
    "                            new_target_col_mask = training_col_mask[sim_i].to(device)\n",
    "                            col_idx_set = new_target_col_mask.unique().tolist()\n",
    "                            print(\"Sim\", sim_i, \"Table\", col_idx_set)\n",
    "                            debias_classes = []\n",
    "                            assert -1 not in col_idx_set\n",
    "                            for r in range(1, len(col_idx_set) + 1):\n",
    "                                for subset in itertools.combinations(col_idx_set, r):\n",
    "                                    # if 0 not in subset:\n",
    "                                    #     continue\n",
    "                                    for x in itertools.permutations(subset):\n",
    "                                        new_batch_data = []\n",
    "                                        for col_i in x:\n",
    "                                            if col_i == 0:\n",
    "                                                if len(new_batch_data) == 0:\n",
    "                                                    cls_indexes_value = 0\n",
    "                                                else:\n",
    "                                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                                new_batch_data.append(batch[\"data\"].T.reshape(-1))\n",
    "                                            else:\n",
    "                                                new_batch_data.append(new_data[new_target_col_mask==col_i].to(device))\n",
    "                                        if 0 not in x:\n",
    "                                            cls_indexes_value = 0\n",
    "                                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                        logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                        msp_temp = logits_temp.max().item()\n",
    "                                        predict_temp = logits_temp.argmax().item()\n",
    "                                        if len(x) == 1 and 0 in x:\n",
    "                                            predict_target = predict_temp\n",
    "                                            msp_target = msp_temp\n",
    "                                        # print(x, msp_temp, predict_temp)\n",
    "                                        if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                            debias_classes.append(predict_temp)\n",
    "                                            continue\n",
    "                                        if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                            max_msp = msp_temp\n",
    "                                            best_msp_perm = x\n",
    "                                            msp_predict = predict_temp\n",
    "                                            logits_msp = logits_temp.clone()\n",
    "                        break\n",
    "                    if max_msp > msp_threshold:\n",
    "                        logits = logits_msp.clone()\n",
    "                    labels_test.append(batch[\"label\"].cpu())\n",
    "                    logits_test.append(logits.detach().cpu())\n",
    "                labels_test = torch.cat(labels_test, dim=0)\n",
    "                logits_test = torch.stack(logits_test, dim=0)\n",
    "                preds_test = torch.argmax(logits_test, dim=1)\n",
    "                num_cols = torch.tensor(num_cols)\n",
    "\n",
    "                from sklearn.metrics import confusion_matrix, f1_score\n",
    "                mask = num_cols > 0\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach().numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************K:5; Threshold: 0.8; msp: 0.8, debias: 0.8****************************\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [488]\u001b[0m, in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     53\u001b[0m new_batch_data \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat(new_batch_data, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     54\u001b[0m cls_indexes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor([\u001b[38;5;241m0\u001b[39m, cls_indexes_value])\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 55\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_batch_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcls_indexes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     56\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m reweight_logits(logits_temp\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mcpu(), class_weights)\n\u001b[1;32m     57\u001b[0m msp_temp \u001b[38;5;241m=\u001b[39m logits_temp\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;241m.\u001b[39mitem()\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:571\u001b[0m, in \u001b[0;36mBertForMultiOutputClassification.forward\u001b[0;34m(self, input_ids, get_enc, cls_indexes, token_type_ids)\u001b[0m\n\u001b[1;32m    569\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    570\u001b[0m \u001b[38;5;66;03m# BertModelMultiOutput\u001b[39;00m\n\u001b[0;32m--> 571\u001b[0m bert_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbert\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m    572\u001b[0m \u001b[43m                        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    573\u001b[0m table_length \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mlen\u001b[39m(input_ids[i]\u001b[38;5;241m.\u001b[39mnonzero()) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(input_ids))]\n\u001b[1;32m    574\u001b[0m \u001b[38;5;66;03m# Note: returned tensor contains pooled_output of all tokens (to make the tensor size consistent)\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:1141\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1134\u001b[0m \u001b[38;5;66;03m# Prepare head mask if needed\u001b[39;00m\n\u001b[1;32m   1135\u001b[0m \u001b[38;5;66;03m# 1.0 in head_mask indicate we keep the head\u001b[39;00m\n\u001b[1;32m   1136\u001b[0m \u001b[38;5;66;03m# attention_probs has shape bsz x n_heads x N x N\u001b[39;00m\n\u001b[1;32m   1137\u001b[0m \u001b[38;5;66;03m# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]\u001b[39;00m\n\u001b[1;32m   1138\u001b[0m \u001b[38;5;66;03m# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]\u001b[39;00m\n\u001b[1;32m   1139\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m-> 1141\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1142\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1143\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1144\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1145\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1146\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1147\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1148\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1149\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1150\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1151\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1152\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1153\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1154\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:694\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    683\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    684\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    685\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    691\u001b[0m         output_attentions,\n\u001b[1;32m    692\u001b[0m     )\n\u001b[1;32m    693\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 694\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    695\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    696\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    697\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    698\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    699\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    700\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    701\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    702\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    704\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    705\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:584\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    572\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    573\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    574\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    581\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m    582\u001b[0m     \u001b[38;5;66;03m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[39;00m\n\u001b[1;32m    583\u001b[0m     self_attn_past_key_value \u001b[38;5;241m=\u001b[39m past_key_value[:\u001b[38;5;241m2\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 584\u001b[0m     self_attention_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    585\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    586\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    587\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    588\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mself_attn_past_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    590\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m self_attention_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    593\u001b[0m     \u001b[38;5;66;03m# if decoder, the last output is tuple of self-attn cache\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:523\u001b[0m, in \u001b[0;36mBertAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    504\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    505\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    506\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    512\u001b[0m     output_attentions: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    513\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m    514\u001b[0m     self_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mself(\n\u001b[1;32m    515\u001b[0m         hidden_states,\n\u001b[1;32m    516\u001b[0m         attention_mask,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    521\u001b[0m         output_attentions,\n\u001b[1;32m    522\u001b[0m     )\n\u001b[0;32m--> 523\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput\u001b[49m\u001b[43m(\u001b[49m\u001b[43mself_outputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    524\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (attention_output,) \u001b[38;5;241m+\u001b[39m self_outputs[\u001b[38;5;241m1\u001b[39m:]  \u001b[38;5;66;03m# add attentions if we output them\u001b[39;00m\n\u001b[1;32m    525\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:465\u001b[0m, in \u001b[0;36mBertSelfOutput.forward\u001b[0;34m(self, hidden_states, input_tensor)\u001b[0m\n\u001b[1;32m    464\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor, input_tensor: torch\u001b[38;5;241m.\u001b[39mTensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m torch\u001b[38;5;241m.\u001b[39mTensor:\n\u001b[0;32m--> 465\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdense\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    466\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(hidden_states)\n\u001b[1;32m    467\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mLayerNorm(hidden_states \u001b[38;5;241m+\u001b[39m input_tensor)\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/linear.py:117\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 117\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "alpha = 0.125\n",
    "class_weights = (1.0 / class_freq) ** alpha\n",
    "\n",
    "# Normalize the weights\n",
    "class_weights /= class_weights.sum()\n",
    "\n",
    "for k in [5,3, 1, 10]:\n",
    "    for msp_threshold in [0.8, 0.9]:\n",
    "        for threshold in [ 0.8, 0.9]:\n",
    "            for debias_threshold in [0.8, 0.9, 1.0]:\n",
    "                print(f\"*********************K:{k}; Threshold: {threshold}; msp: {msp_threshold}, debias: {debias_threshold}****************************\")\n",
    "                model = model.to(device)\n",
    "                ft_embs_test = []\n",
    "                labels_test = []\n",
    "                logits_test = []\n",
    "                log = defaultdict(list)\n",
    "                num_cols = []\n",
    "                for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "                    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "                    target_col_mask = batch[\"target_col_mask\"].T\n",
    "                    logits, target_embs = model(batch[\"data\"].T, cls_indexes=cls_indexes, get_enc=True)\n",
    "                    target_embs = target_embs.detach().cpu()\n",
    "                    logits =  reweight_logits(logits.detach().cpu(), class_weights)\n",
    "                    msp_init = logits.max().item()\n",
    "                    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "                    max_msp = 0\n",
    "                    if 1 not in target_col_mask and msp_init < threshold:\n",
    "                        sim_values, sim_indices = F.cosine_similarity(target_embs.reshape(-1), training_embeddings_all).topk(k)\n",
    "                        \n",
    "                        for sim_i in sim_indices:\n",
    "                            new_data = training_data[sim_i].to(device)\n",
    "                            new_target_col_mask = training_col_mask[sim_i].to(device)\n",
    "                            col_idx_set = new_target_col_mask.unique().tolist()\n",
    "                            debias_classes = []\n",
    "                            assert -1 not in col_idx_set\n",
    "                            for r in range(1, len(col_idx_set) + 1):\n",
    "                                for subset in itertools.combinations(col_idx_set, r):\n",
    "                                    # if 0 not in subset:\n",
    "                                    #     continue\n",
    "                                    for x in itertools.permutations(subset):\n",
    "                                        new_batch_data = []\n",
    "                                        for col_i in x:\n",
    "                                            if col_i == 0:\n",
    "                                                if len(new_batch_data) == 0:\n",
    "                                                    cls_indexes_value = 0\n",
    "                                                else:\n",
    "                                                    cls_indexes_value = sum([len(new_batch_data[i]) for i in range(len(new_batch_data))])\n",
    "                                                new_batch_data.append(batch[\"data\"].T.reshape(-1))\n",
    "                                            else:\n",
    "                                                new_batch_data.append(new_data[new_target_col_mask==col_i].to(device))\n",
    "                                        if 0 not in x:\n",
    "                                            cls_indexes_value = 0\n",
    "                                        new_batch_data = torch.cat(new_batch_data, dim=-1).reshape(1, -1)\n",
    "                                        cls_indexes = torch.tensor([0, cls_indexes_value]).reshape(1, -1).to(device)\n",
    "                                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                                        logits_temp = reweight_logits(logits_temp.detach().cpu(), class_weights)\n",
    "                                        msp_temp = logits_temp.max().item()\n",
    "                                        predict_temp = logits_temp.argmax().item()\n",
    "                                        if len(x) == 1 and 0 in x:\n",
    "                                            predict_target = predict_temp\n",
    "                                            msp_target = msp_temp\n",
    "                                        # print(x, msp_temp, predict_temp)\n",
    "                                        if 0 not in x and msp_temp > debias_threshold and (predict_temp != predict_target):\n",
    "                                            debias_classes.append(predict_temp)\n",
    "                                            continue\n",
    "                                        if msp_temp > max_msp and 0 in x and predict_temp not in debias_classes:\n",
    "                                            max_msp = msp_temp\n",
    "                                            best_msp_perm = x\n",
    "                                            msp_predict = predict_temp\n",
    "                                            logits_msp = logits_temp.clone()\n",
    "                    if max_msp > msp_threshold:\n",
    "                        logits = logits_msp.clone()\n",
    "                    labels_test.append(batch[\"label\"].cpu())\n",
    "                    logits_test.append(logits.detach().cpu())\n",
    "                labels_test = torch.cat(labels_test, dim=0)\n",
    "                logits_test = torch.stack(logits_test, dim=0)\n",
    "                preds_test = torch.argmax(logits_test, dim=1)\n",
    "                num_cols = torch.tensor(num_cols)\n",
    "\n",
    "                from sklearn.metrics import confusion_matrix, f1_score\n",
    "                mask = num_cols > 0\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach().numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "                ts_pred_list = logits_test.argmax(\n",
    "                                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "                ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"micro\")\n",
    "                ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                                    ts_pred_list,\n",
    "                                    average=\"macro\")\n",
    "                print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "\n",
    "                        \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 62])"
      ]
     },
     "execution_count": 484,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"data\"].T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 62])\n",
      "torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "for data in new_batch_data:\n",
    "    print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 483,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save good context\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "num_single_fail = 0\n",
    "num_good_context = 0\n",
    "class_context = defaultdict(list)\n",
    "for batch_idx, batch in enumerate(train_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    label_i = batch[\"label\"].item()\n",
    "    if 1 in target_col_mask:\n",
    "        new_batch_data = batch[\"data\"].T[target_col_mask==0].reshape([1, -1]) # Target column only\n",
    "        cls_indexes = torch.tensor([[0, 0]]).to(device)\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        if logits.argmax().item() != label_i:\n",
    "            num_single_fail += 1\n",
    "            for col_i in target_col_mask.unique():\n",
    "                if col_i == 0 or col_i == -1:\n",
    "                    continue\n",
    "                new_batch_data = torch.cat([batch[\"data\"].T[target_col_mask==0].reshape([1, -1]), batch[\"data\"].T[target_col_mask==col_i].reshape([1, -1])], dim=1)\n",
    "                cls_indexes = torch.tensor([[0, 0]]).to(device)\n",
    "                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                    context_i = batch[\"data\"].T[target_col_mask==col_i].reshape([1, -1]).detach().cpu()\n",
    "                    class_context[label_i].append(context_i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2393"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_single_fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_class_context = {k: len(v) for k, v in class_context.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*********************Threshold: 0.5****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4618, ts_macro_f1=0.1841\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1781\n",
      "ts_micro_f1=0.5059, ts_macro_f1=0.1581\n",
      "*********************Threshold: 0.6****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4590, ts_macro_f1=0.1832\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1781\n",
      "ts_micro_f1=0.5000, ts_macro_f1=0.1499\n",
      "*********************Threshold: 0.7****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4562, ts_macro_f1=0.1828\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1781\n",
      "ts_micro_f1=0.4941, ts_macro_f1=0.1483\n",
      "*********************Threshold: 0.8****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4544, ts_macro_f1=0.1766\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1781\n",
      "ts_micro_f1=0.4902, ts_macro_f1=0.1365\n",
      "*********************Threshold: 0.9****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ts_micro_f1=0.4544, ts_macro_f1=0.1729\n",
      "ts_micro_f1=0.4229, ts_macro_f1=0.1781\n",
      "ts_micro_f1=0.4902, ts_macro_f1=0.1251\n",
      "*********************Threshold: 1.0****************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_4054970/861392142.py:28: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_init = msp_max = F.softmax(logits).max().item()\n",
      "/tmp/ipykernel_4054970/861392142.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  msp_temp = F.softmax(logits_temp).max().item()\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [123]\u001b[0m, in \u001b[0;36m<cell line: 16>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     35\u001b[0m new_batch_data \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat([batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mT[target_col_mask\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mreshape([\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]),temp_col\u001b[38;5;241m.\u001b[39mto(device)], dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     36\u001b[0m cls_indexes \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor([[\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m0\u001b[39m]])\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 37\u001b[0m logits_temp \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_batch_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcls_indexes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcls_indexes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     38\u001b[0m msp_temp \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39msoftmax(logits_temp)\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m     39\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m msp_temp \u001b[38;5;241m>\u001b[39m msp_max:\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/TU/watchog/watchog/model.py:571\u001b[0m, in \u001b[0;36mBertForMultiOutputClassification.forward\u001b[0;34m(self, input_ids, get_enc, cls_indexes, token_type_ids)\u001b[0m\n\u001b[1;32m    569\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    570\u001b[0m \u001b[38;5;66;03m# BertModelMultiOutput\u001b[39;00m\n\u001b[0;32m--> 571\u001b[0m bert_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbert\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m    572\u001b[0m \u001b[43m                        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    573\u001b[0m table_length \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mlen\u001b[39m(input_ids[i]\u001b[38;5;241m.\u001b[39mnonzero()) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(input_ids))]\n\u001b[1;32m    574\u001b[0m \u001b[38;5;66;03m# Note: returned tensor contains pooled_output of all tokens (to make the tensor size consistent)\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:1141\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1134\u001b[0m \u001b[38;5;66;03m# Prepare head mask if needed\u001b[39;00m\n\u001b[1;32m   1135\u001b[0m \u001b[38;5;66;03m# 1.0 in head_mask indicate we keep the head\u001b[39;00m\n\u001b[1;32m   1136\u001b[0m \u001b[38;5;66;03m# attention_probs has shape bsz x n_heads x N x N\u001b[39;00m\n\u001b[1;32m   1137\u001b[0m \u001b[38;5;66;03m# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]\u001b[39;00m\n\u001b[1;32m   1138\u001b[0m \u001b[38;5;66;03m# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]\u001b[39;00m\n\u001b[1;32m   1139\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m-> 1141\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1142\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1143\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1144\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1145\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1146\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1147\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1148\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1149\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1150\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1151\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1152\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1153\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1154\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:694\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    683\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    684\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    685\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    691\u001b[0m         output_attentions,\n\u001b[1;32m    692\u001b[0m     )\n\u001b[1;32m    693\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 694\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    695\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    696\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    697\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    698\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    699\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    700\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    701\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    702\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    704\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    705\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:584\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    572\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    573\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    574\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    581\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m    582\u001b[0m     \u001b[38;5;66;03m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[39;00m\n\u001b[1;32m    583\u001b[0m     self_attn_past_key_value \u001b[38;5;241m=\u001b[39m past_key_value[:\u001b[38;5;241m2\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 584\u001b[0m     self_attention_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    585\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    586\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    587\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    588\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mself_attn_past_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    590\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m self_attention_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    593\u001b[0m     \u001b[38;5;66;03m# if decoder, the last output is tuple of self-attn cache\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:523\u001b[0m, in \u001b[0;36mBertAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    504\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    505\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    506\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    512\u001b[0m     output_attentions: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    513\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m    514\u001b[0m     self_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mself(\n\u001b[1;32m    515\u001b[0m         hidden_states,\n\u001b[1;32m    516\u001b[0m         attention_mask,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    521\u001b[0m         output_attentions,\n\u001b[1;32m    522\u001b[0m     )\n\u001b[0;32m--> 523\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput\u001b[49m\u001b[43m(\u001b[49m\u001b[43mself_outputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    524\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (attention_output,) \u001b[38;5;241m+\u001b[39m self_outputs[\u001b[38;5;241m1\u001b[39m:]  \u001b[38;5;66;03m# add attentions if we output them\u001b[39;00m\n\u001b[1;32m    525\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/torch/nn/modules/module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/jupyterprojects/jupyter/lib/python3.8/site-packages/transformers/models/bert/modeling_bert.py:467\u001b[0m, in \u001b[0;36mBertSelfOutput.forward\u001b[0;34m(self, hidden_states, input_tensor)\u001b[0m\n\u001b[1;32m    465\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdense(hidden_states)\n\u001b[1;32m    466\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(hidden_states)\n\u001b[0;32m--> 467\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mLayerNorm(\u001b[43mhidden_states\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43minput_tensor\u001b[49m)\n\u001b[1;32m    468\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m hidden_states\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# MSP add\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "class_corrected ={k: 0 for k in class_context.keys()}\n",
    "class_mistakes = {k: 0 for k in class_context.keys()}\n",
    "\n",
    "for threshold in [ 0.8, 0.9]:\n",
    "    print(f\"*********************Threshold: {threshold}****************************\")\n",
    "    model = model.to(device)\n",
    "    ft_embs_test = []\n",
    "    labels_test = []\n",
    "    logits_test = []\n",
    "    log = defaultdict(list)\n",
    "    num_cols = []\n",
    "    for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "        cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "        target_col_mask = batch[\"target_col_mask\"].T\n",
    "        logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "        msp_init = msp_max = F.softmax(logits).max().item()\n",
    "        num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "        label_i = batch[\"label\"].item()\n",
    "        if batch[\"target_col_mask\"].max().item() == 0:\n",
    "            if msp_init < threshold:\n",
    "                for class_i in class_context.keys():          \n",
    "                    for temp_col in class_context[class_i]:\n",
    "                        new_batch_data = torch.cat([batch[\"data\"].T[target_col_mask==0].reshape([1, -1]),temp_col.to(device)], dim=1)\n",
    "                        cls_indexes = torch.tensor([[0, 0]]).to(device)\n",
    "                        logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                        msp_temp = F.softmax(logits_temp).max().item()\n",
    "                        if msp_temp > msp_max:\n",
    "                            logits = logits_temp.clone()\n",
    "                            msp_max = msp_temp\n",
    "            \n",
    "        labels_test.append(batch[\"label\"].cpu())\n",
    "        logits_test.append(logits.detach().cpu())\n",
    "    labels_test = torch.cat(labels_test, dim=0)\n",
    "    logits_test = torch.stack(logits_test, dim=0)\n",
    "    preds_test = torch.argmax(logits_test, dim=1)\n",
    "    num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    from sklearn.metrics import confusion_matrix, f1_score\n",
    "    mask = num_cols > 0\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach().numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "    ts_pred_list = logits_test.argmax(\n",
    "                                1).cpu().detach()[~mask].numpy().tolist()\n",
    "    ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"micro\")\n",
    "    ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                        ts_pred_list,\n",
    "                        average=\"macro\")\n",
    "    print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cheat add\n",
    "model.load_state_dict(best_state_dict, strict=False)\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "ft_embs_test = []\n",
    "labels_test = []\n",
    "logits_test = []\n",
    "log = defaultdict(list)\n",
    "num_cols = []\n",
    "threshold = 0.8\n",
    "corrected = 0\n",
    "total_mistakes = 0\n",
    "class_corrected ={k: 0 for k in class_context.keys()}\n",
    "class_mistakes = {k: 0 for k in class_context.keys()}\n",
    "for batch_idx, batch in enumerate(test_dataloader_iter):\n",
    "    cls_indexes = torch.LongTensor([[0, batch[\"cls_indexes\"].cpu().item()]]).to(device)\n",
    "    target_col_mask = batch[\"target_col_mask\"].T\n",
    "    logits = model(batch[\"data\"].T, cls_indexes=cls_indexes,)\n",
    "    num_cols.append(batch[\"target_col_mask\"].max().item())\n",
    "    label_i = batch[\"label\"].item()\n",
    "    if batch[\"target_col_mask\"].max().item() == 0:\n",
    "        if logits.argmax().item() != label_i:\n",
    "            class_mistakes[label_i] += 1\n",
    "            for temp_col in class_context[label_i]:\n",
    "                new_batch_data = torch.cat([batch[\"data\"].T[target_col_mask==0].reshape([1, -1]),temp_col.to(device)], dim=1)\n",
    "                cls_indexes = torch.tensor([[0, 0]]).to(device)\n",
    "                logits_temp = model(new_batch_data, cls_indexes=cls_indexes,)\n",
    "                if logits_temp.argmax().item() == batch[\"label\"].item():\n",
    "                    logits = logits_temp.clone()\n",
    "                    class_corrected[label_i] += 1\n",
    "                    break\n",
    "                \n",
    "    labels_test.append(batch[\"label\"].cpu())\n",
    "    logits_test.append(logits.detach().cpu())\n",
    "labels_test = torch.cat(labels_test, dim=0)\n",
    "logits_test = torch.stack(logits_test, dim=0)\n",
    "preds_test = torch.argmax(logits_test, dim=1)\n",
    "num_cols = torch.tensor(num_cols)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "mask = num_cols > 0\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach().numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1).numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))\n",
    "ts_pred_list = logits_test.argmax(\n",
    "                            1).cpu().detach()[~mask].numpy().tolist()\n",
    "ts_micro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"micro\")\n",
    "ts_macro_f1 = f1_score(labels_test.reshape(-1)[~mask].numpy().tolist(),\n",
    "                    ts_pred_list,\n",
    "                    average=\"macro\")\n",
    "print(\"ts_micro_f1={:.4f}, ts_macro_f1={:.4f}\".format(ts_micro_f1, ts_macro_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jupyter",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}