detect-fake-text-keras-nlp-infer (1).ipynb

{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":61542,"databundleVersionId":6888007,"sourceType":"competition"},{"sourceId":6300093,"sourceType":"datasetVersion","datasetId":3623988},{"sourceId":6857742,"sourceType":"datasetVersion","datasetId":3623154},{"sourceId":6987454,"sourceType":"datasetVersion","datasetId":3947266},{"sourceId":151069021,"sourceType":"kernelVersion"}],"dockerImageVersionId":30588,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":533.685872,"end_time":"2023-11-17T09:04:57.985286","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-11-17T08:56:04.299414","version":"2.4.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0527149485364ca58dd51b17cc928973":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0b711119808a49d9bac5a02344ff3bbc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1aee0e1637f6416a899549e28a4bc89d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_22b0a4eede8647abb4c262c13694973d","IPY_MODEL_a93b2ebb56ca4c5bb736c3bc32727a92","IPY_MODEL_366c2f9c836c4792bbcb09acffc5ead0"],"layout":"IPY_MODEL_0b711119808a49d9bac5a02344ff3bbc"}},"22b0a4eede8647abb4c262c13694973d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6e534f90bdb64d2e8c819c38e93f666a","placeholder":"​","style":"IPY_MODEL_9da411dd15bc438da86f32249c4ffcd6","value":"100%"}},"2b6af4142e7a42af819df54a15bce1f9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"366c2f9c836c4792bbcb09acffc5ead0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_96a389609e1c46bea4df4684981f032c","placeholder":"​","style":"IPY_MODEL_0527149485364ca58dd51b17cc928973","value":" 3/3 [07:35&lt;00:00, 150.77s/it]"}},"6e534f90bdb64d2e8c819c38e93f666a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"96a389609e1c46bea4df4684981f032c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9da411dd15bc438da86f32249c4ffcd6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a93b2ebb56ca4c5bb736c3bc32727a92":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f1111862dee64ff28623a810b1e89d9c","max":3,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2b6af4142e7a42af819df54a15bce1f9","value":3}},"f1111862dee64ff28623a810b1e89d9c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LLM - Detect AI Generated Text\n> Identify which essay was written by a large language model\n\n<img src=\"https://user-images.githubusercontent.com/36858976/279902422-b365f6ef-ef01-49ac-af7f-0bc2ca3ba835.png\">","metadata":{"papermill":{"duration":0.010064,"end_time":"2023-11-17T08:56:07.590406","exception":false,"start_time":"2023-11-17T08:56:07.580342","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 🎯 | Motivation\n\n* In this notebook, we will demonstrate the usage of the multi-backend capabilities of `KerasCore` and `KerasNLP` for the **Detect Fake Text** infernece.","metadata":{"papermill":{"duration":0.009197,"end_time":"2023-11-17T08:56:07.609413","exception":false,"start_time":"2023-11-17T08:56:07.600216","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 🛠 | Install Libraries ","metadata":{"papermill":{"duration":0.009162,"end_time":"2023-11-17T08:56:07.646601","exception":false,"start_time":"2023-11-17T08:56:07.637439","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install /kaggle/input/llm-science-exam-lib-ds/keras_core-0.1.7-py3-none-any.whl --no-deps\n!pip install /kaggle/input/llm-science-exam-lib-ds/keras_nlp-0.6.2-py3-none-any.whl --no-deps","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":5.293319,"end_time":"2023-11-17T08:56:12.949349","exception":false,"start_time":"2023-11-17T08:56:07.656030","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:36:05.182228Z","iopub.execute_input":"2023-11-27T13:36:05.183054Z","iopub.status.idle":"2023-11-27T13:36:50.405744Z","shell.execute_reply.started":"2023-11-27T13:36:05.183006Z","shell.execute_reply":"2023-11-27T13:36:50.404624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📚 | Import Libraries ","metadata":{"papermill":{"duration":0.011202,"end_time":"2023-11-17T08:56:12.972425","exception":false,"start_time":"2023-11-17T08:56:12.961223","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"torch\"  # or \"tensorflow\" or \"torch\"\n\nimport keras_nlp\nimport keras_core as keras \nimport keras_core.backend as K\n\n\nimport jax\nimport tensorflow as tf\n# from tensorflow import keras\n# import tensorflow.keras.backend as K\n\nimport numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport gc","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":16.426151,"end_time":"2023-11-17T08:56:29.408422","exception":false,"start_time":"2023-11-17T08:56:12.982271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:36:50.408738Z","iopub.execute_input":"2023-11-27T13:36:50.409140Z","iopub.status.idle":"2023-11-27T13:37:05.364094Z","shell.execute_reply.started":"2023-11-27T13:36:50.409105Z","shell.execute_reply":"2023-11-27T13:37:05.363274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Library Version","metadata":{"papermill":{"duration":0.009985,"end_time":"2023-11-17T08:56:29.429877","exception":false,"start_time":"2023-11-17T08:56:29.419892","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"TensorFlow:\", tf.__version__)\n# print(\"JAX:\", jax.__version__)\nprint(\"Keras:\", keras.__version__)\nprint(\"KerasNLP:\", keras_nlp.__version__)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.018222,"end_time":"2023-11-17T08:56:29.458303","exception":false,"start_time":"2023-11-17T08:56:29.440081","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:05.365195Z","iopub.execute_input":"2023-11-27T13:37:05.365861Z","iopub.status.idle":"2023-11-27T13:37:05.371342Z","shell.execute_reply.started":"2023-11-27T13:37:05.365826Z","shell.execute_reply":"2023-11-27T13:37:05.370364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚙️ | Configuration","metadata":{"papermill":{"duration":0.009918,"end_time":"2023-11-17T08:56:29.478661","exception":false,"start_time":"2023-11-17T08:56:29.468743","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    verbose = 0  # Verbosity\n    device = 'GPU'  # Device\n    seed = 42  # Random seed\n    batch_size = 6  # Batch size\n    drop_remainder = True  # Drop incomplete batches\n    ckpt_dir = \"/kaggle/input/daigt-kerasnlp-ckpt\"  # Name of pretrained models\n    sequence_length = 200  # Input sequence length\n    class_names = ['real','fake']  # Class names [A, B, C, D, E]\n    num_classes = len(class_names)  # Number of classes\n    class_labels = list(range(num_classes))  # Class labels [0, 1, 2, 3, 4]\n    label2name = dict(zip(class_labels, class_names))  # Label to class name mapping\n    name2label = {v: k for k, v in label2name.items()}  # Class name to label mapping","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.018567,"end_time":"2023-11-17T08:56:29.507387","exception":false,"start_time":"2023-11-17T08:56:29.488820","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:05.372634Z","iopub.execute_input":"2023-11-27T13:37:05.372986Z","iopub.status.idle":"2023-11-27T13:37:05.413740Z","shell.execute_reply.started":"2023-11-27T13:37:05.372947Z","shell.execute_reply":"2023-11-27T13:37:05.412897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ♻️ | Reproducibility \nSets value for random seed to produce similar result in each run.","metadata":{"papermill":{"duration":0.010021,"end_time":"2023-11-17T08:56:29.527616","exception":false,"start_time":"2023-11-17T08:56:29.517595","status":"completed"},"tags":[]}},{"cell_type":"code","source":"keras.utils.set_random_seed(CFG.seed)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.021272,"end_time":"2023-11-17T08:56:29.559054","exception":false,"start_time":"2023-11-17T08:56:29.537782","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:05.416490Z","iopub.execute_input":"2023-11-27T13:37:05.416842Z","iopub.status.idle":"2023-11-27T13:37:05.426046Z","shell.execute_reply.started":"2023-11-27T13:37:05.416811Z","shell.execute_reply":"2023-11-27T13:37:05.425147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 💾 | Hardware\nFollowing codes automatically detects hardware (TPU or GPU). ","metadata":{"papermill":{"duration":0.010001,"end_time":"2023-11-17T08:56:29.579120","exception":false,"start_time":"2023-11-17T08:56:29.569119","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_device():\n    \"Detect and intializes GPU/TPU automatically\"\n    try:\n        # Connect to TPU\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n        # Set TPU strategy\n        strategy = tf.distribute.TPUStrategy(tpu)\n        print(f'> Running on TPU', tpu.master(), end=' | ')\n        print('Num of TPUs: ', strategy.num_replicas_in_sync)\n        device=CFG.device\n    except:\n        # If TPU is not available, detect GPUs\n        gpus = tf.config.list_logical_devices('GPU')\n        ngpu = len(gpus)\n         # Check number of GPUs\n        if ngpu:\n            # Set GPU strategy\n            strategy = tf.distribute.MirroredStrategy(gpus) # single-GPU or multi-GPU\n            # Print GPU details\n            print(\"> Running on GPU\", end=' | ')\n            print(\"Num of GPUs: \", ngpu)\n            device='GPU'\n        else:\n            # If no GPUs are available, use CPU\n            print(\"> Running on CPU\")\n            strategy = tf.distribute.get_strategy()\n            device='CPU'\n    return strategy, device","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.019652,"end_time":"2023-11-17T08:56:29.608817","exception":false,"start_time":"2023-11-17T08:56:29.589165","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:05.427187Z","iopub.execute_input":"2023-11-27T13:37:05.427521Z","iopub.status.idle":"2023-11-27T13:37:05.436122Z","shell.execute_reply.started":"2023-11-27T13:37:05.427489Z","shell.execute_reply":"2023-11-27T13:37:05.435238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize GPU/TPU/TPU-VM\nstrategy, CFG.device = get_device()\nCFG.replicas = strategy.num_replicas_in_sync","metadata":{"papermill":{"duration":5.232155,"end_time":"2023-11-17T08:56:34.851173","exception":false,"start_time":"2023-11-17T08:56:29.619018","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:05.437159Z","iopub.execute_input":"2023-11-27T13:37:05.437524Z","iopub.status.idle":"2023-11-27T13:37:09.994058Z","shell.execute_reply.started":"2023-11-27T13:37:05.437501Z","shell.execute_reply":"2023-11-27T13:37:09.992469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📁 | Dataset Path ","metadata":{"papermill":{"duration":0.010403,"end_time":"2023-11-17T08:56:34.872147","exception":false,"start_time":"2023-11-17T08:56:34.861744","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/llm-detect-ai-generated-text'","metadata":{"papermill":{"duration":0.016915,"end_time":"2023-11-17T08:56:34.899204","exception":false,"start_time":"2023-11-17T08:56:34.882289","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:09.995228Z","iopub.execute_input":"2023-11-27T13:37:09.995533Z","iopub.status.idle":"2023-11-27T13:37:10.002667Z","shell.execute_reply.started":"2023-11-27T13:37:09.995509Z","shell.execute_reply":"2023-11-27T13:37:10.001803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📖 | Meta Data \n* `{test|train}_essays.csv`\n    * `id` - A unique identifier for each essay.\n    * `prompt_id` - Identifies the prompt the essay was written in response to.\n    * `text` - The essay text itself.\n    * `generated` - Whether the essay was written by a student (0) or generated by an LLM (1). This field is the target and is not present in test_essays.csv.\n* **sample_submission.csv** - is the valid sample submission.","metadata":{"papermill":{"duration":0.009855,"end_time":"2023-11-17T08:56:34.919164","exception":false,"start_time":"2023-11-17T08:56:34.909309","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Test Data","metadata":{"papermill":{"duration":0.009854,"end_time":"2023-11-17T08:56:34.939203","exception":false,"start_time":"2023-11-17T08:56:34.929349","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_df = pd.read_csv(f'{BASE_PATH}/test_essays.csv')  # Read CSV file into a DataFrame\n\n# Display information about the train data\nprint(\"# Test Data: {:,}\".format(len(test_df)))\nprint(\"# Sample:\")\ndisplay(test_df.head(2))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.040957,"end_time":"2023-11-17T08:56:34.990388","exception":false,"start_time":"2023-11-17T08:56:34.949431","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.003619Z","iopub.execute_input":"2023-11-27T13:37:10.003914Z","iopub.status.idle":"2023-11-27T13:37:10.033269Z","shell.execute_reply.started":"2023-11-27T13:37:10.003885Z","shell.execute_reply":"2023-11-27T13:37:10.032396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍽️ | Preprocessing\n\n**What it does:** The preprocessor takes input strings and transforms them into a dictionary (`token_ids`, `padding_mask`) containing preprocessed tensors. This process starts with tokenization, where input strings are converted into sequences of token IDs.\n\n**Why it's important:** Initially, raw text data is complex and challenging for modeling due to its high dimensionality. By converting text into a compact set of tokens, such as transforming `\"The quick brown fox\"` into `[\"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\"]`, we simplify the data. Many models rely on special tokens and additional tensors to understand input. These tokens help divide input and identify padding, among other tasks. Making all sequences the same length through padding boosts computational efficiency, making subsequent steps smoother.\n\nExplore the following pages to access the available preprocessing and tokenizer layers in **KerasNLP**:\n- [Preprocessing](https://keras.io/api/keras_nlp/preprocessing_layers/)\n- [Tokenizers](https://keras.io/api/keras_nlp/tokenizers/)","metadata":{"papermill":{"duration":0.010257,"end_time":"2023-11-17T08:56:35.011327","exception":false,"start_time":"2023-11-17T08:56:35.001070","status":"completed"},"tags":[]}},{"cell_type":"code","source":"vocab_path = '/kaggle/input/keras-nlp-deberta-v3-base-en-vocab-ds/vocab.spm'\ntokenizer= keras_nlp.models.DebertaV3Tokenizer(vocab_path)\npreprocessor= keras_nlp.models.DebertaV3Preprocessor(tokenizer, sequence_length=CFG.sequence_length)","metadata":{"papermill":{"duration":0.750082,"end_time":"2023-11-17T08:56:35.811755","exception":false,"start_time":"2023-11-17T08:56:35.061673","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.034712Z","iopub.execute_input":"2023-11-27T13:37:10.035107Z","iopub.status.idle":"2023-11-27T13:37:10.780114Z","shell.execute_reply.started":"2023-11-27T13:37:10.035076Z","shell.execute_reply":"2023-11-27T13:37:10.779316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's examine what the output shape of the preprocessing layer looks like. The output shape of the layer can be represented as $(num\\_choices, sequence\\_length)$.","metadata":{"papermill":{"duration":0.010604,"end_time":"2023-11-17T08:56:35.833390","exception":false,"start_time":"2023-11-17T08:56:35.822786","status":"completed"},"tags":[]}},{"cell_type":"code","source":"outs = preprocessor(test_df.text.iloc[0])  # Process options for the first row\n\n# Display the shape of each processed output\nfor k, v in outs.items():\n    print(k, \":\", v.shape)","metadata":{"papermill":{"duration":0.174217,"end_time":"2023-11-17T08:56:36.018247","exception":false,"start_time":"2023-11-17T08:56:35.844030","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.781199Z","iopub.execute_input":"2023-11-27T13:37:10.781485Z","iopub.status.idle":"2023-11-27T13:37:10.924296Z","shell.execute_reply.started":"2023-11-27T13:37:10.781461Z","shell.execute_reply":"2023-11-27T13:37:10.923401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll use the `preprocessing_fn` function to transform each text option using the `dataset.map(preprocessing_fn)` method.","metadata":{"papermill":{"duration":0.010316,"end_time":"2023-11-17T08:56:36.039399","exception":false,"start_time":"2023-11-17T08:56:36.029083","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocess_fn(text, label=None):\n    text = preprocessor(text)  # Preprocess text\n    return (text, label) if label is not None else text  # Return processed text and label if available","metadata":{"papermill":{"duration":0.017934,"end_time":"2023-11-17T08:56:36.068127","exception":false,"start_time":"2023-11-17T08:56:36.050193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.925530Z","iopub.execute_input":"2023-11-27T13:37:10.926051Z","iopub.status.idle":"2023-11-27T13:37:10.931307Z","shell.execute_reply.started":"2023-11-27T13:37:10.926018Z","shell.execute_reply":"2023-11-27T13:37:10.930415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍚 | DataLoader\n\nThe code below sets up a robust data flow pipeline using `tf.data.Dataset` for data processing. Notable aspects of `tf.data` include its ability to simplify pipeline construction and represent components in sequences.\n\nTo learn more about `tf.data`, refer to this [documentation](https://www.tensorflow.org/guide/data).","metadata":{"papermill":{"duration":0.010397,"end_time":"2023-11-17T08:56:36.089077","exception":false,"start_time":"2023-11-17T08:56:36.078680","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_dataset(texts, labels=None, batch_size=32,\n                  cache=False, drop_remainder=True,\n                  augment=False, repeat=False, shuffle=1024):\n    AUTO = tf.data.AUTOTUNE  # AUTOTUNE option\n    slices = (texts,) if labels is None else (texts, labels)  # Create slices\n    ds = tf.data.Dataset.from_tensor_slices(slices)  # Create dataset from slices\n    ds = ds.cache() if cache else ds  # Cache dataset if enabled\n    ds = ds.map(preprocess_fn, num_parallel_calls=AUTO)  # Map preprocessing function\n    ds = ds.repeat() if repeat else ds  # Repeat dataset if enabled\n    opt = tf.data.Options()  # Create dataset options\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)  # Shuffle dataset if enabled\n        opt.experimental_deterministic = False\n    ds = ds.with_options(opt)  # Set dataset options\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)  # Batch dataset\n    ds = ds.prefetch(AUTO)  # Prefetch next batch\n    return ds  # Return the built dataset","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.020825,"end_time":"2023-11-17T08:56:36.120453","exception":false,"start_time":"2023-11-17T08:56:36.099628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.932623Z","iopub.execute_input":"2023-11-27T13:37:10.932896Z","iopub.status.idle":"2023-11-27T13:37:10.945098Z","shell.execute_reply.started":"2023-11-27T13:37:10.932873Z","shell.execute_reply":"2023-11-27T13:37:10.944429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fetch Train/test Dataset\n\nThe function below generates the training and testation datasets for a given fold.","metadata":{"papermill":{"duration":0.010674,"end_time":"2023-11-17T08:56:36.141878","exception":false,"start_time":"2023-11-17T08:56:36.131204","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_test_dataset(test_df):\n    test_texts = test_df.text.tolist()  # Extract testation texts\n    \n    # Build testation dataset\n    test_ds = build_dataset(test_texts, labels=None,\n                             batch_size=min(CFG.batch_size*CFG.replicas, len(test_df)), cache=False,\n                             shuffle=False, drop_remainder=False, repeat=False)\n    \n    return test_ds  # Return datasets and dataframes","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.018123,"end_time":"2023-11-17T08:56:36.170771","exception":false,"start_time":"2023-11-17T08:56:36.152648","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.949230Z","iopub.execute_input":"2023-11-27T13:37:10.949470Z","iopub.status.idle":"2023-11-27T13:37:10.954926Z","shell.execute_reply.started":"2023-11-27T13:37:10.949449Z","shell.execute_reply":"2023-11-27T13:37:10.954066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🤖 | Modeling\n\n","metadata":{"papermill":{"duration":0.01047,"end_time":"2023-11-17T08:56:36.191662","exception":false,"start_time":"2023-11-17T08:56:36.181192","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_model():\n    # Create a DebertaV3Classifier model\n    classifier = keras_nlp.models.DebertaV3Classifier.from_preset(\n        CFG.preset,\n        load_weights=False,\n        preprocessor=None,\n        num_classes=1 # one output per one option, for five options total 5 outputs\n    )\n    inputs = classifier.input\n    logits = classifier(inputs)\n        \n    # Compute final output\n    outputs = keras.layers.Activation(\"sigmoid\")(logits)\n    model = keras.Model(inputs, outputs)\n    return model","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.018819,"end_time":"2023-11-17T08:56:36.221036","exception":false,"start_time":"2023-11-17T08:56:36.202217","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.956167Z","iopub.execute_input":"2023-11-27T13:37:10.956801Z","iopub.status.idle":"2023-11-27T13:37:10.964485Z","shell.execute_reply.started":"2023-11-27T13:37:10.956748Z","shell.execute_reply":"2023-11-27T13:37:10.963746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ckpt processing\nFor some reason, `keras.models.load_model` requires write access as `/kaggle/input` doesn't have that access it throws error. Workaround is to simply copy the `ckpts` to other directory then load the model.","metadata":{"papermill":{"duration":0.010531,"end_time":"2023-11-17T08:56:36.242130","exception":false,"start_time":"2023-11-17T08:56:36.231599","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get the checkpoint directory and name\nckpt_dir = CFG.ckpt_dir\nckpt_name = ckpt_dir.split('/')[3]\n\n# Copy the checkpoints to a new directory in the /kaggle directory\n!cp -r {ckpt_dir} /kaggle/{ckpt_name}\n\n# List all the checkpoint paths in the new directory\nnew_ckpt_dir = f\"/kaggle/{ckpt_name}\"\nckpt_paths = glob(os.path.join(new_ckpt_dir, '*.keras'))\n\nprint(\"Total CKPT:\", len(ckpt_paths))","metadata":{"papermill":{"duration":42.567175,"end_time":"2023-11-17T08:57:18.819825","exception":false,"start_time":"2023-11-17T08:56:36.252650","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:37:10.965444Z","iopub.execute_input":"2023-11-27T13:37:10.965692Z","iopub.status.idle":"2023-11-27T13:38:22.470147Z","shell.execute_reply.started":"2023-11-27T13:37:10.965671Z","shell.execute_reply":"2023-11-27T13:38:22.468863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🧪 | Prediction","metadata":{"papermill":{"duration":0.010729,"end_time":"2023-11-17T08:57:18.841466","exception":false,"start_time":"2023-11-17T08:57:18.830737","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Inference","metadata":{"papermill":{"duration":0.010593,"end_time":"2023-11-17T08:57:18.863152","exception":false,"start_time":"2023-11-17T08:57:18.852559","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Initialize an array to store predictions for each fold\nfold_preds = np.zeros(shape=(len(test_df),), dtype='float32')\n\n# # Build model\n# model = build_model()\n\n# Iterate through each checkpoint path\nfor ckpt_path in tqdm(ckpt_paths):\n    # Load the pre-trained model from the checkpoint\n    model = keras.models.load_model(\n        ckpt_path,\n        compile=False,\n    )\n#     model.load_weights(ckpt_path)\n    \n    # Get the test dataset\n    test_ds = get_test_dataset(test_df)\n    \n    # Generate predictions using the model\n    preds = model.predict(\n        test_ds,\n        batch_size=min(CFG.batch_size * CFG.replicas * 2, len(test_df)),  # Set batch size\n        verbose=1\n    )\n    \n    # Add predictions to fold_preds and average over checkpoints\n    fold_preds += preds.squeeze() / len(ckpt_paths)\n    \n    # Clean up by deleting the model and collecting garbage\n    del model\n    gc.collect()","metadata":{"papermill":{"duration":455.322197,"end_time":"2023-11-17T09:04:54.196046","exception":false,"start_time":"2023-11-17T08:57:18.873849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:38:22.471966Z","iopub.execute_input":"2023-11-27T13:38:22.472398Z","iopub.status.idle":"2023-11-27T13:46:09.676153Z","shell.execute_reply.started":"2023-11-27T13:38:22.472361Z","shell.execute_reply":"2023-11-27T13:46:09.675168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Prediction","metadata":{"papermill":{"duration":0.011286,"end_time":"2023-11-17T09:04:54.220295","exception":false,"start_time":"2023-11-17T09:04:54.209009","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Format predictions and true answers\npred_answers = (fold_preds > 0.5).astype(int).squeeze()\n\n# Check 5 Predictions\nprint(\"# Predictions\\n\")\nfor i in range(3):\n    row = test_df.iloc[i]\n    text  = row.text\n    pred_answer = CFG.label2name[pred_answers[i]]\n    print(f\"❓ Text {i+1}:\\n{text}\\n\")\n    print(f\"🤖 Predicted: {pred_answer}\\n\")\n    print(\"-\"*90, \"\\n\")","metadata":{"papermill":{"duration":0.021515,"end_time":"2023-11-17T09:04:54.253176","exception":false,"start_time":"2023-11-17T09:04:54.231661","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:46:09.677519Z","iopub.execute_input":"2023-11-27T13:46:09.677849Z","iopub.status.idle":"2023-11-27T13:46:09.686444Z","shell.execute_reply.started":"2023-11-27T13:46:09.677820Z","shell.execute_reply":"2023-11-27T13:46:09.685194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📮 | Submission","metadata":{"papermill":{"duration":0.011262,"end_time":"2023-11-17T09:04:54.275828","exception":false,"start_time":"2023-11-17T09:04:54.264566","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Create a DataFrame to store the submission\nsub_df = test_df[[\"id\"]].copy()\n\n# Add the formatted predictions to the submission DataFrame\nsub_df[\"generated\"] = fold_preds.squeeze()\n\n# Save Submission\nsub_df.to_csv('submission.csv',index=False)\n\n# Display the first 2 rows of the submission DataFrame\nsub_df.head(2)","metadata":{"papermill":{"duration":0.034166,"end_time":"2023-11-17T09:04:54.321527","exception":false,"start_time":"2023-11-17T09:04:54.287361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-27T13:46:09.688077Z","iopub.execute_input":"2023-11-27T13:46:09.688539Z","iopub.status.idle":"2023-11-27T13:46:09.717158Z","shell.execute_reply.started":"2023-11-27T13:46:09.688455Z","shell.execute_reply":"2023-11-27T13:46:09.716198Z"},"trusted":true},"execution_count":null,"outputs":[]}]}