finetune_whisper_small_fa_v01.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2720e54b",
      "metadata": {
        "id": "2720e54b",
        "scrolled": true,
        "tags": []
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install datasets==2.8.0\n",
        "!pip install transformers==4.26\n",
        "!pip install librosa\n",
        "!pip install evaluate>=0.30\n",
        "!pip install audiomentations\n",
        "!pip install jiwer\n",
        "!pip install gradio\n",
        "!pip install torchaudio\n",
        "!pip install tensorboardX\n",
        "!pip install accelerate -U\n",
        "!pip install hazm==0.7.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "47d65032-2f55-49b6-91bd-104bd73ef0cd",
      "metadata": {
        "id": "47d65032-2f55-49b6-91bd-104bd73ef0cd",
        "outputId": "fe689e54-7696-46a7-ac89-db476c726a2a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
            "Token is valid (permission: write).\n",
            "Your token has been saved to /home/jupyter/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ],
      "source": [
        "!huggingface-cli login --token <\"YOUR_HF_TOKEN\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9377f469",
      "metadata": {
        "id": "9377f469",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import hazm\n",
        "import string\n",
        "import os\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from datasets import load_dataset, load_metric, Dataset, concatenate_datasets, load_from_disk"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3d6d74e2-610f-49cf-a960-ae5dd36c11ca",
      "metadata": {
        "id": "3d6d74e2-610f-49cf-a960-ae5dd36c11ca"
      },
      "outputs": [],
      "source": [
        "_normalizer = hazm.Normalizer()\n",
        "\n",
        "chars_to_ignore = [\n",
        "    \",\", \"?\", \".\", \"!\", \"-\", \";\", \":\", '\"\"', \"%\", \"'\", '\"', \"�\",\n",
        "    \"#\", \"!\", \"؟\", \"?\", \"«\", \"»\", \"،\", \"(\", \")\", \"؛\", \"'ٔ\", \"٬\",'ٔ', \",\", \"?\",\n",
        "    \".\", \"!\", \"-\", \";\", \":\",'\"',\"“\", \"%\", \"‘\", \"”\", \"�\", \"–\", \"…\", \"_\", \"”\", '“', '„',\n",
        "    'ā', 'š',\n",
        "]\n",
        "\n",
        "chars_to_ignore = chars_to_ignore + list(string.ascii_lowercase + string.digits)\n",
        "\n",
        "chars_to_mapping = {\n",
        "    'ك': 'ک', 'دِ': 'د', 'بِ': 'ب', 'زِ': 'ز', 'ذِ': 'ذ', 'شِ': 'ش', 'سِ': 'س', 'ى': 'ی',\n",
        "    'ي': 'ی', 'أ': 'ا', 'ؤ': 'و', \"ے\": \"ی\", \"ۀ\": \"ه\", \"ﭘ\": \"پ\", \"ﮐ\": \"ک\", \"ﯽ\": \"ی\",\n",
        "    \"ﺎ\": \"ا\", \"ﺑ\": \"ب\", \"ﺘ\": \"ت\", \"ﺧ\": \"خ\", \"ﺩ\": \"د\", \"ﺱ\": \"س\", \"ﻀ\": \"ض\", \"ﻌ\": \"ع\",\n",
        "    \"ﻟ\": \"ل\", \"ﻡ\": \"م\", \"ﻢ\": \"م\", \"ﻪ\": \"ه\", \"ﻮ\": \"و\", 'ﺍ': \"ا\", 'ة': \"ه\",\n",
        "    'ﯾ': \"ی\", 'ﯿ': \"ی\", 'ﺒ': \"ب\", 'ﺖ': \"ت\", 'ﺪ': \"د\", 'ﺮ': \"ر\", 'ﺴ': \"س\", 'ﺷ': \"ش\",\n",
        "    'ﺸ': \"ش\", 'ﻋ': \"ع\", 'ﻤ': \"م\", 'ﻥ': \"ن\", 'ﻧ': \"ن\", 'ﻭ': \"و\", 'ﺭ': \"ر\", \"ﮔ\": \"گ\",\n",
        "    \"۱۴ام\": \"۱۴ ام\",\n",
        "\n",
        "    \"a\": \" ای \", \"b\": \" بی \", \"c\": \" سی \", \"d\": \" دی \", \"e\": \" ایی \", \"f\": \" اف \",\n",
        "    \"g\": \" جی \", \"h\": \" اچ \", \"i\": \" آی \", \"j\": \" جی \", \"k\": \" کی \", \"l\": \" ال \",\n",
        "    \"m\": \" ام \", \"n\": \" ان \", \"o\": \" او \", \"p\": \" پی \", \"q\": \" کیو \", \"r\": \" آر \",\n",
        "    \"s\": \" اس \", \"t\": \" تی \", \"u\": \" یو \", \"v\": \" وی \", \"w\": \" دبلیو \", \"x\": \" اکس \",\n",
        "    \"y\": \" وای \", \"z\": \" زد \",\n",
        "    \"\\u200c\": \" \", \"\\u200d\": \" \", \"\\u200e\": \" \", \"\\u200f\": \" \", \"\\ufeff\": \" \",\n",
        "}\n",
        "\n",
        "\n",
        "def multiple_replace(text, chars_to_mapping):\n",
        "    pattern = \"|\".join(map(re.escape, chars_to_mapping.keys()))\n",
        "    return re.sub(pattern, lambda m: chars_to_mapping[m.group()], str(text))\n",
        "\n",
        "def remove_special_characters(text, chars_to_ignore_regex):\n",
        "    text = re.sub(chars_to_ignore_regex, '', text).lower() + \" \"\n",
        "    return text\n",
        "\n",
        "def normalizer(row, chars_to_ignore=chars_to_ignore, chars_to_mapping=chars_to_mapping):\n",
        "    text = row['sentence']\n",
        "    chars_to_ignore_regex = f\"\"\"[{\"\".join(chars_to_ignore)}]\"\"\"\n",
        "    text = text.lower().strip()\n",
        "\n",
        "    text = _normalizer.normalize(text)\n",
        "    text = multiple_replace(text, chars_to_mapping)\n",
        "    text = remove_special_characters(text, chars_to_ignore_regex)\n",
        "    text = re.sub(\" +\", \" \", text)\n",
        "    _text = []\n",
        "    for word in text.split():\n",
        "        try:\n",
        "            word = int(word)\n",
        "            _text.append(words(word))\n",
        "        except:\n",
        "            _text.append(word)\n",
        "\n",
        "    text = \" \".join(_text) + \" \"\n",
        "    text = text.strip()\n",
        "\n",
        "    if not len(text) > 0:\n",
        "        return None\n",
        "\n",
        "    row['sentence'] = text\n",
        "    return row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "82109dc1",
      "metadata": {
        "id": "82109dc1",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "from transformers import WhisperFeatureExtractor, WhisperTokenizer, WhisperProcessor\n",
        "\n",
        "feature_extractor = WhisperFeatureExtractor.from_pretrained(\"openai/whisper-small\")\n",
        "tokenizer = WhisperTokenizer.from_pretrained(\"openai/whisper-small\", language=\"persian\", task=\"transcribe\")\n",
        "processor = WhisperProcessor.from_pretrained(\"openai/whisper-small\", language=\"persian\", task=\"transcribe\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6036a32f",
      "metadata": {
        "id": "6036a32f",
        "scrolled": true,
        "outputId": "7d88c05b-9f15-4752-9df8-d277072e22ed",
        "colab": {
          "referenced_widgets": [
            "93102ddca622431db036695aaad559af",
            "aee03dc02347416380336035d25b26a9"
          ]
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Found cached dataset common_voice_11_0 (/home/jupyter/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/fa/11.0.0/3f27acf10f303eac5b6fbbbe02495aeddb46ecffdb0a2fe3507fcfbf89094631)\n",
            "Found cached dataset common_voice_11_0 (/home/jupyter/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/fa/11.0.0/3f27acf10f303eac5b6fbbbe02495aeddb46ecffdb0a2fe3507fcfbf89094631)\n",
            "Parameter 'function'=<function normalizer at 0x7fbc0e9bc280> of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93102ddca622431db036695aaad559af",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/26951 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aee03dc02347416380336035d25b26a9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/10288 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['audio', 'sentence'],\n",
            "        num_rows: 26951\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['audio', 'sentence'],\n",
            "        num_rows: 10288\n",
            "    })\n",
            "})\n"
          ]
        }
      ],
      "source": [
        "from datasets import load_dataset, DatasetDict\n",
        "\n",
        "common_voice = DatasetDict()\n",
        "\n",
        "common_voice[\"train\"] = load_dataset(\"mozilla-foundation/common_voice_11_0\", \"fa\", split=\"train\")\n",
        "common_voice[\"validation\"] = load_dataset(\"mozilla-foundation/common_voice_11_0\", \"fa\", split=\"validation\")\n",
        "\n",
        "common_voice = common_voice.map(normalizer)\n",
        "common_voice = common_voice.remove_columns([\"accent\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"path\", \"segment\", \"up_votes\"])\n",
        "\n",
        "print(common_voice)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2oDLkemSOp_V",
      "metadata": {
        "id": "2oDLkemSOp_V"
      },
      "outputs": [],
      "source": [
        "from datasets import Audio\n",
        "common_voice = common_voice.cast_column(\"audio\", Audio(sampling_rate=16000))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ZXbvUw8zS03t",
      "metadata": {
        "id": "ZXbvUw8zS03t",
        "outputId": "e5f2126f-fb86-4632-dcc3-59b329af13d4",
        "colab": {
          "referenced_widgets": [
            "3fb898c300c045a98737c53a55e35132",
            "5b7fd76ad4814e329d07e8dec0a86a4b",
            "fc581e5e1ffb43d2bdb4541bb16a05bf",
            "0e1bd9057483407cab787d77a5ecc64d",
            "2dff70abeb8e4449a34b919423c662e5",
            "88fc6fae53de46fc9abb054bba92728c",
            "157801c7abe542c79d1fc7bef57e0b20",
            "337bc48b9fe34564a05a77c19b67494b",
            "93c8bb526426416c81cb4149ea4bfb4a",
            "87a9f4d912d54b23aeb921d7a84153b9",
            "4806ab3bb89042dd848903cd8cfb91b8",
            "0c4b9b85d78d402a9bb611766fdec01e"
          ]
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "        "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3fb898c300c045a98737c53a55e35132",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#3:   0%|          | 0/4492 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b7fd76ad4814e329d07e8dec0a86a4b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#1:   0%|          | 0/4492 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc581e5e1ffb43d2bdb4541bb16a05bf",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#2:   0%|          | 0/4492 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0e1bd9057483407cab787d77a5ecc64d",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#0:   0%|          | 0/4492 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2dff70abeb8e4449a34b919423c662e5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#5:   0%|          | 0/4491 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "88fc6fae53de46fc9abb054bba92728c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#4:   0%|          | 0/4492 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "        "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "157801c7abe542c79d1fc7bef57e0b20",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#3:   0%|          | 0/1715 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "337bc48b9fe34564a05a77c19b67494b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#0:   0%|          | 0/1715 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93c8bb526426416c81cb4149ea4bfb4a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#2:   0%|          | 0/1715 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "87a9f4d912d54b23aeb921d7a84153b9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#4:   0%|          | 0/1714 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " "
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4806ab3bb89042dd848903cd8cfb91b8",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#5:   0%|          | 0/1714 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c4b9b85d78d402a9bb611766fdec01e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "#1:   0%|          | 0/1715 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def prepare_dataset(batch):\n",
        "    # load and resample audio data from 48 to 16kHz\n",
        "    audio = batch[\"audio\"]\n",
        "\n",
        "    # compute log-Mel input features from input audio array\n",
        "    batch[\"input_features\"] = feature_extractor(audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_features[0]\n",
        "\n",
        "    # encode target text to label ids\n",
        "    batch[\"labels\"] = tokenizer(batch[\"sentence\"]).input_ids\n",
        "    return batch\n",
        "\n",
        "common_voice = common_voice.map(prepare_dataset, num_proc=6)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "gFgOcs8iS8ds",
      "metadata": {
        "id": "gFgOcs8iS8ds"
      },
      "outputs": [],
      "source": [
        "common_voice = common_voice.remove_columns(['audio', 'sentence'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d90073a5",
      "metadata": {
        "id": "d90073a5"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "from dataclasses import dataclass\n",
        "from typing import Any, Dict, List, Union\n",
        "\n",
        "@dataclass\n",
        "class DataCollatorSpeechSeq2SeqWithPadding:\n",
        "    processor: Any\n",
        "\n",
        "    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
        "        # split inputs and labels since they have to be of different lengths and need different padding methods\n",
        "        # first treat the audio inputs by simply returning torch tensors\n",
        "        input_features = [{\"input_features\": feature[\"input_features\"]} for feature in features]\n",
        "        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n",
        "\n",
        "        # get the tokenized label sequences\n",
        "        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n",
        "        # pad the labels to max length\n",
        "        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n",
        "\n",
        "        # replace padding with -100 to ignore loss correctly\n",
        "        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n",
        "\n",
        "        # if bos token is appended in previous tokenization step,\n",
        "        # cut bos token here as it's append later anyways\n",
        "        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n",
        "            labels = labels[:, 1:]\n",
        "\n",
        "        batch[\"labels\"] = labels\n",
        "\n",
        "        return batch\n",
        "\n",
        "data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "49f47fa4",
      "metadata": {
        "id": "49f47fa4"
      },
      "outputs": [],
      "source": [
        "import evaluate\n",
        "metric = evaluate.load(\"wer\")\n",
        "\n",
        "def compute_metrics(pred):\n",
        "    pred_ids = pred.predictions\n",
        "    label_ids = pred.label_ids\n",
        "\n",
        "    # replace -100 with the pad_token_id\n",
        "    label_ids[label_ids == -100] = tokenizer.pad_token_id\n",
        "\n",
        "    # we do not want to group tokens when computing the metrics\n",
        "    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)\n",
        "    label_str = tokenizer.batch_decode(label_ids, skip_special_tokens=True)\n",
        "\n",
        "    wer = 100 * metric.compute(predictions=pred_str, references=label_str)\n",
        "\n",
        "    return {\"wer\": wer}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a6381e77",
      "metadata": {
        "id": "a6381e77",
        "scrolled": true,
        "tags": []
      },
      "outputs": [],
      "source": [
        "from transformers import WhisperForConditionalGeneration\n",
        "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small\", use_cache = False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c32057f3",
      "metadata": {
        "id": "c32057f3"
      },
      "outputs": [],
      "source": [
        "model.config.forced_decoder_ids = None\n",
        "model.config.suppress_tokens = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3a67bf33",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3a67bf33",
        "outputId": "c1f718b2-c628-4994-8846-4f4db0d740d2"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n"
          ]
        }
      ],
      "source": [
        "from transformers import Seq2SeqTrainingArguments\n",
        "\n",
        "training_args = Seq2SeqTrainingArguments(\n",
        "    output_dir=\"whisper_small-fa_v01\",\n",
        "    per_device_train_batch_size=8,\n",
        "    gradient_accumulation_steps=2,\n",
        "    learning_rate=1e-5,\n",
        "    warmup_steps=500,\n",
        "    max_steps=5000,\n",
        "    gradient_checkpointing=True,\n",
        "    fp16=True,\n",
        "    evaluation_strategy=\"steps\",\n",
        "    per_device_eval_batch_size=8,\n",
        "    predict_with_generate=True,\n",
        "    generation_max_length=225,\n",
        "    eval_steps=500,\n",
        "    save_steps=500,\n",
        "    logging_steps=500,\n",
        "    report_to=[\"tensorboard\"],\n",
        "    metric_for_best_model=\"wer\",\n",
        "    greater_is_better=False,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6241828e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6241828e",
        "outputId": "4ef16d3d-158f-46a4-b01a-fa549585484c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "max_steps is given, it will override any value given in num_train_epochs\n",
            "Using cuda_amp half precision backend\n"
          ]
        }
      ],
      "source": [
        "from transformers import Seq2SeqTrainer\n",
        "\n",
        "trainer = Seq2SeqTrainer(\n",
        "    args=training_args,\n",
        "    model=model,\n",
        "    train_dataset=common_voice[\"train\"],\n",
        "    eval_dataset=common_voice[\"validation\"],\n",
        "    data_collator=data_collator,\n",
        "    compute_metrics=compute_metrics,\n",
        "    tokenizer=processor.feature_extractor,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f655c06a-b072-4544-93aa-3862674a6c27",
      "metadata": {
        "scrolled": true,
        "tags": [],
        "id": "f655c06a-b072-4544-93aa-3862674a6c27",
        "outputId": "cf13cd06-362d-4e81-d132-0e36468eca1a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running training *****\n",
            "  Num examples = 26951\n",
            "  Num Epochs = 1\n",
            "  Instantaneous batch size per device = 4\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 5000\n",
            "  Number of trainable parameters = 241734912\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='5000' max='5000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [5000/5000 13:12:48, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Wer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>1.179600</td>\n",
              "      <td>0.504016</td>\n",
              "      <td>57.715051</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.341200</td>\n",
              "      <td>0.414977</td>\n",
              "      <td>45.909700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.269900</td>\n",
              "      <td>0.378401</td>\n",
              "      <td>41.061453</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.252000</td>\n",
              "      <td>0.342005</td>\n",
              "      <td>39.632134</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.222900</td>\n",
              "      <td>0.313812</td>\n",
              "      <td>37.457937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.201100</td>\n",
              "      <td>0.300456</td>\n",
              "      <td>34.493123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.191800</td>\n",
              "      <td>0.276289</td>\n",
              "      <td>32.461041</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.173200</td>\n",
              "      <td>0.262007</td>\n",
              "      <td>31.850109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.163400</td>\n",
              "      <td>0.254489</td>\n",
              "      <td>31.000686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.161000</td>\n",
              "      <td>0.250763</td>\n",
              "      <td>30.688686</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-500\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-500/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-500/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-500/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-500/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-1000\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-1000/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-1000/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-1000/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-1000/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-1500\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-1500/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-1500/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-1500/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-1500/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-2000\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-2000/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-2000/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-2000/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-2000/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-2500\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-2500/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-2500/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-2500/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-2500/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-3000\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-3000/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-3000/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-3000/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-3000/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-3500\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-3500/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-3500/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-3500/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-3500/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-4000\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-4000/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-4000/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-4000/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-4000/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-4500\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-4500/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-4500/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-4500/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-4500/preprocessor_config.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 10288\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to mohammadh128/whisper_small-fa_v01/checkpoint-5000\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-5000/config.json\n",
            "Configuration saved in mohammadh128/whisper_small-fa_v01/checkpoint-5000/generation_config.json\n",
            "Model weights saved in mohammadh128/whisper_small-fa_v01/checkpoint-5000/pytorch_model.bin\n",
            "Feature extractor saved in mohammadh128/whisper_small-fa_v01/checkpoint-5000/preprocessor_config.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=5000, training_loss=0.3156125183105469, metrics={'train_runtime': 47569.336, 'train_samples_per_second': 0.42, 'train_steps_per_second': 0.105, 'total_flos': 5.7717080064e+18, 'train_loss': 0.3156125183105469, 'epoch': 0.74})"
            ]
          },
          "execution_count": 92,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "93c05110-0319-42ad-bb07-0d38ec0491e6",
      "metadata": {
        "id": "93c05110-0319-42ad-bb07-0d38ec0491e6"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": []
    },
    "environment": {
      "kernel": "conda-env-pytorch-pytorch",
      "name": "workbench-notebooks.m110",
      "type": "gcloud",
      "uri": "gcr.io/deeplearning-platform-release/workbench-notebooks:m110"
    },
    "kernelspec": {
      "display_name": "PyTorch 1-13",
      "language": "python",
      "name": "conda-env-pytorch-pytorch"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}