quantize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try:\n",
    "  import google.colab\n",
    "  IN_COLAB = True\n",
    "  \n",
    "except:\n",
    "  IN_COLAB = False\n",
    "\n",
    "IN_COLAB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if IN_COLAB:\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-23 21:50:33.693102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-05-23 21:50:35.274007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/lib/wsl/lib:\n",
      "2023-05-23 21:50:35.274614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/lib/wsl/lib:\n",
      "2023-05-23 21:50:35.274628: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
    "from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path = './models/small_model_230517-19_30'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'NEGATIVE', '1': 'POSITIVE'}. The number of labels wil be overwritten to 2.\n",
      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n",
      "Some weights of BartForConditionalGeneration were not initialized from the model checkpoint at ./models/small_model_230517-19_30 and are newly initialized: ['model.decoder.layers.4.self_attn_layer_norm.bias', 'model.decoder.layers.5.final_layer_norm.bias', 'model.decoder.layers.3.self_attn.k_proj.bias', 'model.decoder.layers.5.encoder_attn.v_proj.weight', 'model.decoder.layers.5.encoder_attn_layer_norm.weight', 'model.decoder.layers.5.self_attn.q_proj.bias', 'model.decoder.layers.5.self_attn_layer_norm.bias', 'model.decoder.layers.5.final_layer_norm.weight', 'model.decoder.layers.3.self_attn.q_proj.bias', 'model.decoder.layers.4.encoder_attn.q_proj.bias', 'model.decoder.layers.3.self_attn.out_proj.weight', 'model.decoder.layers.5.self_attn.q_proj.weight', 'model.decoder.layers.4.encoder_attn.q_proj.weight', 'model.decoder.layers.3.fc1.weight', 'model.decoder.layers.5.self_attn.k_proj.bias', 'model.decoder.layers.3.self_attn.v_proj.weight', 'model.decoder.layers.4.self_attn.q_proj.weight', 'model.decoder.layers.4.encoder_attn.k_proj.bias', 'model.decoder.layers.3.encoder_attn.k_proj.weight', 'model.decoder.layers.3.encoder_attn.q_proj.bias', 'model.decoder.layers.4.self_attn.k_proj.bias', 'model.decoder.layers.3.fc2.weight', 'model.decoder.layers.3.encoder_attn.v_proj.bias', 'model.decoder.layers.3.self_attn_layer_norm.bias', 'model.decoder.layers.3.final_layer_norm.weight', 'model.decoder.layers.4.encoder_attn.out_proj.weight', 'model.decoder.layers.4.fc2.weight', 'model.decoder.layers.3.encoder_attn.q_proj.weight', 'model.decoder.layers.4.fc1.weight', 'model.decoder.layers.5.fc2.bias', 'model.decoder.layers.5.self_attn.v_proj.weight', 'model.decoder.layers.5.encoder_attn_layer_norm.bias', 'model.decoder.layers.4.encoder_attn.k_proj.weight', 'model.decoder.layers.3.encoder_attn.k_proj.bias', 'model.decoder.layers.3.fc1.bias', 'model.decoder.layers.4.encoder_attn_layer_norm.bias', 'model.decoder.layers.3.self_attn.out_proj.bias', 'model.decoder.layers.3.encoder_attn_layer_norm.weight', 'model.decoder.layers.4.self_attn.out_proj.bias', 'model.decoder.layers.4.encoder_attn_layer_norm.weight', 'model.decoder.layers.5.encoder_attn.out_proj.bias', 'model.decoder.layers.3.encoder_attn_layer_norm.bias', 'model.decoder.layers.4.encoder_attn.out_proj.bias', 'model.decoder.layers.5.fc1.bias', 'model.decoder.layers.3.self_attn.k_proj.weight', 'model.decoder.layers.4.fc2.bias', 'model.decoder.layers.4.fc1.bias', 'model.decoder.layers.3.self_attn.q_proj.weight', 'model.decoder.layers.5.fc1.weight', 'model.decoder.layers.5.self_attn.out_proj.bias', 'model.decoder.layers.4.self_attn.k_proj.weight', 'model.decoder.layers.5.encoder_attn.out_proj.weight', 'model.decoder.layers.4.self_attn.v_proj.weight', 'model.decoder.layers.3.fc2.bias', 'model.decoder.layers.4.self_attn_layer_norm.weight', 'model.decoder.layers.3.encoder_attn.v_proj.weight', 'model.decoder.layers.5.encoder_attn.v_proj.bias', 'model.decoder.layers.4.final_layer_norm.weight', 'model.decoder.layers.5.encoder_attn.k_proj.weight', 'model.decoder.layers.5.encoder_attn.k_proj.bias', 'model.decoder.layers.4.self_attn.out_proj.weight', 'model.decoder.layers.4.self_attn.v_proj.bias', 'model.decoder.layers.3.self_attn.v_proj.bias', 'model.decoder.layers.3.self_attn_layer_norm.weight', 'model.decoder.layers.3.encoder_attn.out_proj.weight', 'model.decoder.layers.5.self_attn.out_proj.weight', 'model.decoder.layers.3.encoder_attn.out_proj.bias', 'model.decoder.layers.5.self_attn_layer_norm.weight', 'model.decoder.layers.4.encoder_attn.v_proj.weight', 'model.decoder.layers.4.final_layer_norm.bias', 'model.decoder.layers.5.self_attn.v_proj.bias', 'model.decoder.layers.5.self_attn.k_proj.weight', 'model.decoder.layers.3.final_layer_norm.bias', 'model.decoder.layers.5.fc2.weight', 'model.decoder.layers.5.encoder_attn.q_proj.bias', 'model.decoder.layers.5.encoder_attn.q_proj.weight', 'model.decoder.layers.4.self_attn.q_proj.bias', 'model.decoder.layers.4.encoder_attn.v_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path, load_in_8bit=True, device_map='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=64,\n",
    "    per_device_eval_batch_size=64,\n",
    "    num_train_epochs=1,\n",
    "    weight_decay=0.01,\n",
    "    report_to=\"tensorboard\",\n",
    "    push_to_hub=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_args,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thesol1/.local/lib/python3.10/site-packages/transformers/modeling_utils.py:1690: UserWarning: You are calling `save_pretrained` to a 8-bit converted model you may likely encounter unexepected behaviors. If you want to save 8-bit models, make sure to have `bitsandbytes>0.37.2` installed.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "NAME_STR = datetime.datetime.now().strftime(f'%y%m%d-%H:%M-{checkpoint_path.split(\"/\")[-1]}-quantized')\n",
    "trainer.create_model_card(\n",
    "    language='Korean',\n",
    "    tags='Grammar',\n",
    "    #model='KoGrammar',\n",
    "    finetuned_from=checkpoint_path\n",
    ")\n",
    "trainer.save_model(f\"./models/{NAME_STR}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}