VAD_PAPER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49c990de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from inaGVAD.vad_metrics import VadEval\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31f5e0e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ve = VadEval(collar=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "67eb017d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lsystems = ['inaspeechsegmenter', 'liumspkdiarization_csv', 'pyannote_vad', 'rvad_csv', 'silero_vad', 'speechbrain_vad']\n",
    "ldf = []\n",
    "for sys in lsystems:\n",
    "    _, summary = ve.compare_category('./automatic_baselines/%s/' % sys, 'channel_category', './annotations/filesplit/testset.csv')\n",
    "    ldf.append(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "933b7eef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>(generalist_radio, accuracy)</th>\n",
       "      <th>(generalist_radio, precision)</th>\n",
       "      <th>(generalist_radio, recall)</th>\n",
       "      <th>(music_radio, accuracy)</th>\n",
       "      <th>(music_radio, precision)</th>\n",
       "      <th>(music_radio, recall)</th>\n",
       "      <th>(generalist_tv, accuracy)</th>\n",
       "      <th>(generalist_tv, precision)</th>\n",
       "      <th>(generalist_tv, recall)</th>\n",
       "      <th>(news_tv, accuracy)</th>\n",
       "      <th>(news_tv, precision)</th>\n",
       "      <th>(news_tv, recall)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>system</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>inaspeechsegmenter</th>\n",
       "      <td>95.411575</td>\n",
       "      <td>95.502631</td>\n",
       "      <td>99.771684</td>\n",
       "      <td>98.126449</td>\n",
       "      <td>88.359260</td>\n",
       "      <td>97.772894</td>\n",
       "      <td>88.321832</td>\n",
       "      <td>87.717223</td>\n",
       "      <td>93.568961</td>\n",
       "      <td>94.956887</td>\n",
       "      <td>95.050142</td>\n",
       "      <td>99.454249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>liumspkdiarization_csv</th>\n",
       "      <td>94.336806</td>\n",
       "      <td>94.796721</td>\n",
       "      <td>99.373777</td>\n",
       "      <td>74.168262</td>\n",
       "      <td>32.203277</td>\n",
       "      <td>97.987616</td>\n",
       "      <td>81.396503</td>\n",
       "      <td>80.592478</td>\n",
       "      <td>90.732681</td>\n",
       "      <td>92.099945</td>\n",
       "      <td>92.740657</td>\n",
       "      <td>98.762540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pyannote_vad</th>\n",
       "      <td>96.263637</td>\n",
       "      <td>96.673692</td>\n",
       "      <td>99.408570</td>\n",
       "      <td>75.721974</td>\n",
       "      <td>33.804017</td>\n",
       "      <td>99.949320</td>\n",
       "      <td>89.533990</td>\n",
       "      <td>86.515467</td>\n",
       "      <td>97.726126</td>\n",
       "      <td>96.098912</td>\n",
       "      <td>96.076198</td>\n",
       "      <td>99.640856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rvad_csv</th>\n",
       "      <td>95.141496</td>\n",
       "      <td>97.929153</td>\n",
       "      <td>96.830910</td>\n",
       "      <td>18.805304</td>\n",
       "      <td>12.917960</td>\n",
       "      <td>96.625336</td>\n",
       "      <td>77.542536</td>\n",
       "      <td>74.444693</td>\n",
       "      <td>95.073936</td>\n",
       "      <td>91.835153</td>\n",
       "      <td>94.583746</td>\n",
       "      <td>96.242477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>silero_vad</th>\n",
       "      <td>96.106662</td>\n",
       "      <td>97.269449</td>\n",
       "      <td>98.587155</td>\n",
       "      <td>61.647499</td>\n",
       "      <td>23.883359</td>\n",
       "      <td>95.686857</td>\n",
       "      <td>86.903811</td>\n",
       "      <td>83.864255</td>\n",
       "      <td>96.700425</td>\n",
       "      <td>95.749382</td>\n",
       "      <td>96.957497</td>\n",
       "      <td>98.257996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>speechbrain_vad</th>\n",
       "      <td>94.076095</td>\n",
       "      <td>94.817379</td>\n",
       "      <td>99.053498</td>\n",
       "      <td>92.911399</td>\n",
       "      <td>64.236650</td>\n",
       "      <td>96.645233</td>\n",
       "      <td>87.141578</td>\n",
       "      <td>84.834100</td>\n",
       "      <td>95.579532</td>\n",
       "      <td>94.175754</td>\n",
       "      <td>94.042890</td>\n",
       "      <td>99.704007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MEAN</th>\n",
       "      <td>95.222712</td>\n",
       "      <td>96.164837</td>\n",
       "      <td>98.837599</td>\n",
       "      <td>70.230148</td>\n",
       "      <td>42.567420</td>\n",
       "      <td>97.444543</td>\n",
       "      <td>85.140042</td>\n",
       "      <td>82.994703</td>\n",
       "      <td>94.896943</td>\n",
       "      <td>94.152672</td>\n",
       "      <td>94.908522</td>\n",
       "      <td>98.677021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        (generalist_radio, accuracy)  \\\n",
       "system                                                 \n",
       "inaspeechsegmenter                         95.411575   \n",
       "liumspkdiarization_csv                     94.336806   \n",
       "pyannote_vad                               96.263637   \n",
       "rvad_csv                                   95.141496   \n",
       "silero_vad                                 96.106662   \n",
       "speechbrain_vad                            94.076095   \n",
       "MEAN                                       95.222712   \n",
       "\n",
       "                        (generalist_radio, precision)  \\\n",
       "system                                                  \n",
       "inaspeechsegmenter                          95.502631   \n",
       "liumspkdiarization_csv                      94.796721   \n",
       "pyannote_vad                                96.673692   \n",
       "rvad_csv                                    97.929153   \n",
       "silero_vad                                  97.269449   \n",
       "speechbrain_vad                             94.817379   \n",
       "MEAN                                        96.164837   \n",
       "\n",
       "                        (generalist_radio, recall)  (music_radio, accuracy)  \\\n",
       "system                                                                        \n",
       "inaspeechsegmenter                       99.771684                98.126449   \n",
       "liumspkdiarization_csv                   99.373777                74.168262   \n",
       "pyannote_vad                             99.408570                75.721974   \n",
       "rvad_csv                                 96.830910                18.805304   \n",
       "silero_vad                               98.587155                61.647499   \n",
       "speechbrain_vad                          99.053498                92.911399   \n",
       "MEAN                                     98.837599                70.230148   \n",
       "\n",
       "                        (music_radio, precision)  (music_radio, recall)  \\\n",
       "system                                                                    \n",
       "inaspeechsegmenter                     88.359260              97.772894   \n",
       "liumspkdiarization_csv                 32.203277              97.987616   \n",
       "pyannote_vad                           33.804017              99.949320   \n",
       "rvad_csv                               12.917960              96.625336   \n",
       "silero_vad                             23.883359              95.686857   \n",
       "speechbrain_vad                        64.236650              96.645233   \n",
       "MEAN                                   42.567420              97.444543   \n",
       "\n",
       "                        (generalist_tv, accuracy)  (generalist_tv, precision)  \\\n",
       "system                                                                          \n",
       "inaspeechsegmenter                      88.321832                   87.717223   \n",
       "liumspkdiarization_csv                  81.396503                   80.592478   \n",
       "pyannote_vad                            89.533990                   86.515467   \n",
       "rvad_csv                                77.542536                   74.444693   \n",
       "silero_vad                              86.903811                   83.864255   \n",
       "speechbrain_vad                         87.141578                   84.834100   \n",
       "MEAN                                    85.140042                   82.994703   \n",
       "\n",
       "                        (generalist_tv, recall)  (news_tv, accuracy)  \\\n",
       "system                                                                 \n",
       "inaspeechsegmenter                    93.568961            94.956887   \n",
       "liumspkdiarization_csv                90.732681            92.099945   \n",
       "pyannote_vad                          97.726126            96.098912   \n",
       "rvad_csv                              95.073936            91.835153   \n",
       "silero_vad                            96.700425            95.749382   \n",
       "speechbrain_vad                       95.579532            94.175754   \n",
       "MEAN                                  94.896943            94.152672   \n",
       "\n",
       "                        (news_tv, precision)  (news_tv, recall)  \n",
       "system                                                           \n",
       "inaspeechsegmenter                 95.050142          99.454249  \n",
       "liumspkdiarization_csv             92.740657          98.762540  \n",
       "pyannote_vad                       96.076198          99.640856  \n",
       "rvad_csv                           94.583746          96.242477  \n",
       "silero_vad                         96.957497          98.257996  \n",
       "speechbrain_vad                    94.042890          99.704007  \n",
       "MEAN                               94.908522          98.677021  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld = []\n",
    "for sys, df in zip(lsystems, ldf):\n",
    "    df = df.set_index('category')\n",
    "    d = {'system': sys}\n",
    "    for cat in ['generalist_radio', 'music_radio', 'generalist_tv', 'news_tv']:\n",
    "        for metric in ['accuracy', 'precision', 'recall']:\n",
    "            d[(cat, metric)] = df[metric][cat] * 100\n",
    "    ld.append(d)\n",
    "df = pd.DataFrame.from_dict(ld).set_index('system')\n",
    "#df = df * 100\n",
    "\n",
    "d = {'system' : 'MEAN'}\n",
    "d.update(df.mean())\n",
    "dmean = pd.DataFrame.from_dict([d]).set_index('system')\n",
    "dmean\n",
    "\n",
    "dall = pd.concat([df, dmean])\n",
    "#.to_latex()\n",
    "#df\n",
    "dall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a96a10cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrrrrrrr}\n",
      "\\toprule\n",
      " & ('generalistradio', 'accuracy') & ('generalistradio', 'precision') & ('generalistradio', 'recall') & ('musicradio', 'accuracy') & ('musicradio', 'precision') & ('musicradio', 'recall') & ('generalisttv', 'accuracy') & ('generalisttv', 'precision') & ('generalisttv', 'recall') & ('newstv', 'accuracy') & ('newstv', 'precision') & ('newstv', 'recall') \\\\\n",
      "system &  &  &  &  &  &  &  &  &  &  &  &  \\\\\n",
      "\\midrule\n",
      "inaspeechsegmenter & 95.4 & 95.5 & 99.8 & 98.1 & 88.4 & 97.8 & 88.3 & 87.7 & 93.6 & 95.0 & 95.1 & 99.5 \\\\\n",
      "liumspkdiarizationcsv & 94.3 & 94.8 & 99.4 & 74.2 & 32.2 & 98.0 & 81.4 & 80.6 & 90.7 & 92.1 & 92.7 & 98.8 \\\\\n",
      "pyannotevad & 96.3 & 96.7 & 99.4 & 75.7 & 33.8 & 99.9 & 89.5 & 86.5 & 97.7 & 96.1 & 96.1 & 99.6 \\\\\n",
      "rvadcsv & 95.1 & 97.9 & 96.8 & 18.8 & 12.9 & 96.6 & 77.5 & 74.4 & 95.1 & 91.8 & 94.6 & 96.2 \\\\\n",
      "silerovad & 96.1 & 97.3 & 98.6 & 61.6 & 23.9 & 95.7 & 86.9 & 83.9 & 96.7 & 95.7 & 97.0 & 98.3 \\\\\n",
      "speechbrainvad & 94.1 & 94.8 & 99.1 & 92.9 & 64.2 & 96.6 & 87.1 & 84.8 & 95.6 & 94.2 & 94.0 & 99.7 \\\\\n",
      "MEAN & 95.2 & 96.2 & 98.8 & 70.2 & 42.6 & 97.4 & 85.1 & 83.0 & 94.9 & 94.2 & 94.9 & 98.7 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(dall.to_latex(float_format='%.1f').replace('_', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f29c2acc",
   "metadata": {},
   "outputs": [],
   "source": [
    "lsystems = ['inaspeechsegmenter', 'liumspkdiarization_csv', 'pyannote_vad', 'rvad_csv', 'silero_vad', 'speechbrain_vad']\n",
    "ldf = []\n",
    "for sys in lsystems:\n",
    "    _, summary = ve.compare_csvset('./automatic_baselines/%s/' % sys, './annotations/filesplit/testset.csv')\n",
    "    ldf.append(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2592901f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ld = []\n",
    "for sys, df in zip(lsystems, ldf):\n",
    "    d = {'system': sys}\n",
    "    for metric in ['accuracy', 'precision', 'recall']:\n",
    "        d[metric] = df[metric]\n",
    "    ld.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bd173201",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrr}\n",
      "\\toprule\n",
      "system & accuracy & precision & recall \\\\\n",
      "\\midrule\n",
      "inaspeechsegmenter & 0.93 & 0.92 & 0.97 \\\\\n",
      "liumspkdiarizationcsv & 0.84 & 0.81 & 0.96 \\\\\n",
      "pyannotevad & 0.89 & 0.85 & 0.99 \\\\\n",
      "rvadcsv & 0.70 & 0.67 & 0.96 \\\\\n",
      "silerovad & 0.84 & 0.80 & 0.98 \\\\\n",
      "speechbrainvad & 0.91 & 0.88 & 0.98 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(pd.DataFrame.from_dict(ld).to_latex(float_format='%.2f', index=False).replace('_', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9012d799",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}