Code/neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a923666",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from skimage import io\n",
    "from skimage.transform import resize\n",
    "from torchvision import transforms\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from tqdm import tqdm\n",
    "# For reproducibility\n",
    "\n",
    "random_state = 42\n",
    "gen = random.Random(random_state)\n",
    "\n",
    "trimmed = True\n",
    "data_dir = '../../allwavs/allvowl/corrected/extracted/' + 'trimmed/'*trimmed + 'img/'\n",
    "files = os.listdir(data_dir)\n",
    "use_mel = True\n",
    "files = [file for file in files if ('mel' in file) == use_mel]\n",
    "#max_w = max(io.imread(data_dir + '/' + file).shape[1] for file in tqdm(files))\n",
    "#print(max_w)\n",
    "max_w = 31 if not trimmed else 20 # Max image width, measured using the code above\n",
    "gen.shuffle(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "70190926",
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):\n",
    "        torch.nn.init.xavier_uniform_(m.weight.data)\n",
    "        m.bias.data.fill_(0.01)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c651a72d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [01:06<00:00, 53.68it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 407/407 [00:07<00:00, 57.97it/s]\n"
     ]
    }
   ],
   "source": [
    "class SpectrumDataset(Dataset):\n",
    "    \"\"\"\n",
    "    A dataset of mel-spectrograms\n",
    "    \"\"\"\n",
    "    def __init__(self, data_dir, files):\n",
    "        self.data_dir = data_dir\n",
    "        self.files = [file for file in files if '~' not in file.split('__')[0]]\n",
    "        gen.shuffle(self.files)\n",
    "        self.idx2lbl = ['2', '9', 'a', 'E', 'e', 'i', 'o', 'O', 'u', 'y']\n",
    "        self.lbl2idx = {label: i for i, label in enumerate(self.idx2lbl)}\n",
    "        all_phonemes = ['l', 'm', 'p', 's', 't', 't1']\n",
    "        self.data = []\n",
    "        for file in tqdm(self.files):\n",
    "            split = file.split('__')\n",
    "        \n",
    "            gender = split[1]\n",
    "            phonemes = split[4].split('_')\n",
    "            \n",
    "            image = io.imread(self.data_dir + '/' + file, as_gray=True)\n",
    "            image = resize(image, (image.shape[0], max_w), anti_aliasing=False) # Resize to the same width\n",
    "            \n",
    "            label = file.split('__')[0].replace('E+', 'e').replace('@', '2') # Standardize vowel names\n",
    "            \n",
    "            # Image, features (gender, one-hot encoded previous phoneme)\n",
    "            self.data.append((\n",
    "            (torch.tensor(image).float(),\n",
    "             torch.tensor([gender == 'f', *[phonemes[0] == x for x in all_phonemes]])\n",
    "            ), self.lbl2idx[label]))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx]\n",
    "\n",
    "l = len(files)\n",
    "train_size = 0.9\n",
    "cutoff = int(l*train_size)\n",
    "\n",
    "train_set = SpectrumDataset(data_dir, files[:cutoff])\n",
    "test_set = SpectrumDataset(data_dir, files[cutoff:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "928e3ca7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.8013, 0.8707, 0.9931,  ..., 0.9990, 0.8168, 0.7121],\n",
      "        [0.7211, 0.7740, 0.8721,  ..., 0.9196, 0.7754, 0.6919],\n",
      "        [0.6715, 0.7268, 0.8221,  ..., 0.8039, 0.7133, 0.6616],\n",
      "        ...,\n",
      "        [0.2827, 0.3439, 0.4436,  ..., 0.5985, 0.4833, 0.4010],\n",
      "        [0.3435, 0.3812, 0.4397,  ..., 0.4725, 0.4434, 0.4234],\n",
      "        [0.3710, 0.4086, 0.4672,  ..., 0.5005, 0.4729, 0.4541]])\n",
      "tensor([ True, False,  True, False, False, False, False])\n",
      "6\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '6')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEoAAAEICAYAAAD4N2KiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkQklEQVR4nO2da4xs23WVv1XV1fXo7vPy9T22j23uxbIBY4mHrBCBApFMJGMw5keAGAWZxChCCg4BJGITpPxAkRyBLCzCHyt24ggTY0IkrCgiuQRFEQI7dkikECzbNzbEN7m+1z6P7uqud9XiR/VYNfaqXdXV3afP2RXOlFrdVbVrV+9Rc8015pxjrR1ijDyxs632uP+BbbEnQG1oT4Da0J4AtaE9AWpDewLUhvYEqA3tDwRQIYTvCiF8IYRwEkL4nRDCtz3sz9h52Cd81BZC+A7gx4C/Cfwa8Oor+ZxtZ+YhhP8OfDTG+NGr/JytHnohhDrwVuCVIYTnQwgvhBB+PITQftiftdVAAbeBBvCdwLcBfxL4U8A/e9gftO1A9U9//+sY44sxxm8CHwLe8bA/aKuBijHeB14ArjzQbjVQp/aTwPtCCE+HEG4CPwj8/MP+kK2nB8A/B54CvgQMgE8BP/qwP2Tr6cGjsj8IQ++R2BOgNrQrAyqE8PYQwhdPieD7r+pzHpVdSYw6ZcxfAr6D+fT9OeDdMcb//dA/7BHZVc163wI8H2P8CkAI4ZPAu4BSoG7cuBHv3LlDrVZjZ2eHer1+rg/Tlx1CONf7JpMJ4/GYyWTCbDbj+eef/2aM8ZVlx14VUHeAr9njF4A/4weEEL4P+D6A27dv8/GPf5y9vT1u3brF3t4e9XqdEELh4mezGTFGZrNZ6YfWajVqteVo4qNG56vVaty9e5cXX3yRe/fuMRgMeOc73/l/V13QVQFV9tUWxniM8SPARwDe9KY3xcPDQ0ajEdPplH6/z+7uLiGEdOExRsbjMbPZrABUvV6nVqslYOv1egJmOp0SY2QymSTQd3Z2EqAnJycMBgMmkwnT6XTtBV0VUC8Ar7PHrwV+f9XBs9mMfr9PrVZjNBoxGAwYDodLHjUcDpnNZkyn0wRio9GgXq8noHQ+eZ6AqtVq6Xz6OT4+pt/vMxwOGY/Hay/oqoD6HPDGEMKzwO8B3wX8rVUH62JGoxGj0ajgFW7D4ZDpdMpsNkte0Ww2qdfr7OzML8VB0t+rJiyB9Ng8KsY4CSH8feAXgTrwsRjjb685nuFwCMxjx3g8JoSQLtK9Q8co4Pf7/TTkNBnIY2q1GtPpNA1ZB2NnZ4fxeJyAXxX30vGXg2S1xRh/AfiFDY9lOp2mWSgPyA6YA+g2nU6TJwo0f697mL8nf26VVSIpVqCG+aw0mUwSKPIeeYcPu/F4nF7T+waDAY1GI9GMHFQBMx6P07DbBKxKAAUUhoUAyYHy4K7f8hZ/TRcvy8FyirEp4a4EUBp6uiAFV128ZrXd3d0CV9IwdHOP9Jimc+m5TYK9WyWAgoVH5bFFFmNMHMifyz0tN3mlxzmdS1/KYw3m5zUfZprqPYD7j6zME/RcGRvPZ86yc66ySgAVQmB3d5dms8n+/j6tVovRaJSGpC5QbF3UQLHGPcaHk4ZpvV5PHqPfOUk9yyoBFCw8Sj+NRiMFaQ/YPvXrPR57VgVu8ag8Rm1qlQEKimCFEJK3yEv0fP4bKJ3iBdp0Ol0iljr/1gVzffO6KKcLOSj+2C9WYOZlF/dGMXdxrDKCW2aVAQqKDNyHhc+Ezp/KZkgPzj7MfFjnk8PWBHOZ/9M+XGR+sfmPArMHdk39SrTz5FlFu/xzyqwSQOWelP/oGP8t0/DT3+5d6+LPeauhlQAKSPFpMplQr9eXcjABkg8XgaQZTcPMj/Finc7lM6jHtVVWGaByj8qnfViQRB9meTyC4uyp1/w4nw398TqrDFCwXE5ZBZRKut6EyL2vrNSSJ8NlpZdVVhmgFJB3d3fZ3d1Nia1fiJeAQwiJlArYer1eKA3DImjnuV4eDx9XzfzCJsCcjeeeJss5VJ4HejzLJ4OtDeZQjEHK7nMgBKAKb86N9Jq8SuRSHjqZTIA5WN692SSVqRRQsrLgWhavgEJRT2A6awdWpjD5F7HOKgvUWa/lRFMXnueK0+mU4XC4NEkA2xnMnUdpiLh3eF4HC6DyOAWLIqAH65wy6L3++eusMkApvggofev50AIKQRqWKwd5Xco9qSzX28QqAVSMkdFoBMxBUOsprxi02+30vJdPckaemzNw7/8BZ3aIZZUASuZe5THHUxHNZK5Ckc1ms8SfVpVOnHdtGp+gQkCVJcQCSGA1m012d3cTmKIHGlIwj08C00GW9yn+iWK4V66zCwMVQngd8NPAq4AZ8JEY44dDCLeAfw88A/wf4G/EuR78rPMlZr27u5su1omilCfyBvGlvOqZy4OcOnhQ99LLWcH8MtLECfCPY4x/DPhW4PtDCG8G3g/8cozxjcAvnz4+0/RP6x/X37nsZzQapfTGj9Ow9JZW2fTvwMJy9XSVXdijYowvAi+e/t0NIXyBuYDsXcC3nx72ceBXgB8663z6htUOl7DCuZECto7Pqwcakjmx9GKenpMXaSieZQ8lRoUQnmG+WOezwO1TEIkxvhhCeHrFe5Li7ubNm/lrpQU4D/Q6zslmHudgWYVXVm55JMw8hLAP/EfgB2OMR5vykmiKu9e//vUx74iUJbOSA0kY5j+wGGq5dqGsOeGNjE1aV5eST4cQGsxB+kSM8edOn34phPDq09dfDbx8kXPnhTexbQcwHzY+9Pxxv99nMBgwGo2WPM4D/Nr/5yIXcfoBAfgo8IUY44fspU8D7zn9+z3Af9rkfLn7uxd4ycWBymkEUJjqBXCv10uTQBl32oShX2bo/TngbwO/FUL4zdPn/inwQeBTIYT3Ar8L/PVNT+jxotFopHKI9/l0kd5R8V6gA1oW52BBRXTOK+3CxBj/G+XqX4C3nfd8Hl88YMv8sZeDVwXiMkrgr3k+uDVqFsUbXcBwOKTRaCzNWC7SyFUvytlU9oWiQsaLdIpdSoOcnK6yygHlM1KuthNb95wuJ5XyMijSh3wW1eduQg2gIkABCSggKXnVQBDjVnMBigIMf69imyyPRw6qK1y2osKpura6MPIWeVFeWvEVDD7Nu+We458Fy4BtxdADlmIJFBuZ+coEWT6112q1pRkyP16vbTrsoEJAeVyC5ZkKWEp4XT5dphyGoirP31sWv9ZZJYDKSyJOJJW86mK9opAnxZ6O5LxJVtY93qpg7hfp9SMvu6gop/UvjUYDKILkNMP5lv729MZB3YoY5RfmMQoW3uC5ni7MOVNZQi3LNeuwoBE+Y66zSgAFxX5d2QrQXLbj7Nrfn8ecsiHmsaosrpVZJYDSN91sNul0OjSbzfS8uJSGmifK8giPXxJuuOX1KJ0TlpeDrLJKAAULj9nZ2WF3d7cAikye5uv58hzQOZee8wrnqgC+FR4FxW+60+nwqle9inq9zmg0otvtcnx8XIhjDqw8bDqdMhqNCsfovB7sYcH+t46Z7+zs0Gq12N/fZ39/PyXAZQ1KAdRoNJZqTK1WKwlYxcu02EhlG58lPZats0oABRRi1P7+fiG5heWZUUPUKwCiDt6aF0kVIN5UKCvnrLLKACXVydHREdPplMPDQ2KMhcqkholKupPJhJOTk0QT6vU64/G4MKTkNbkcW4Lax75U9jzmlQAJxKRFcIB0MQLCV3EKKI9H8kCd21eQtlqtjWY7WSWAgkVwlXgeisRRHqAiXe4hCt5a8u8VTA/WohqtVmvjYQcVAgoW3WJvcTuB9LxPXpL3+PQ3kAK4p0V6/jwgQYWAEmGUEEM1p5wbeQ6XJ9NloEFRFJv392D9CgdZZYCCYhVBs543HLwCqot2EprX2D0P9NlS7/MccGsIp6zMa2CRFK+qDJRtepN3Ylwp7LZVQMl7xG+Uiyn4SgLtU7mnOWWyH68YlEkSXU69FfSgzLwZIFsXR9wr8txOs6OvaDivVQYoD6gilT7MYAGGT/++ciqPPQJoPB6nlMe9T+f036vs0nvchRDqIYTfCCH8/OnjWyGE50IIXz79ffOsczgh9E6xQBmNRvT7/UJQhmIxT/HMhRt+nLZb0g4/mzYV0rnOdXS5/QPgC/b4Qoo779G50MIZuMeeXLkCy2oWmc6rbZfkreexSw29EMJrgb/MfCfVf3T69IUUd8rXZrMZvV6PVquVArMC/NHRUZrdcp2C6uqeBOdVAj2OMdLr9VImkC/2LrPLxqh/BfwT4MCeO7fi7vr168Air7Njli7W6095M0Ie562u/Hw6Pk+Ir4xwhhD+CvByjPHXQwjfft73R1Pc3blzJ0KxlXR4eFgotIUQuHbtGrPZXA0szxAgzuJdBKv35v1CeWPe2lpll9VH/dUQwjuAFnAthPBvOVXcnXrTRoo7HxLK7vWPe6spHz7yHm8W+AIhectwOEz6Bc18eUXhLKAuHMxjjB+IMb42xvgM8z3s/muM8bu5gOJOs5VihkosXqBTtu9DUaD6slmVaHSufr/P0dFR2iFRhT4d9zg32jq34k7lWh82eTc4T2cElDRPOl4eAwuy6kMxb0b4cF9nDwWoGOOvMJ/diDHe5QKKO12IZD7OsmGxmAgW8ej089I5yioNwBIp9c6MU451Vhlm7g2D3d3dpXaTPCrGmHp30+mUdrtd0HKqTKwVWo1Gg/39/QJzB1Lu6Csh1lllgBIg2kcKlru8PhzVTHAJY75poI7Nl4w46P54nVUKKBXutC5PZV8F57zmJO/K2/C+RE3cy3dShEX80m4/WwMUUIgfikf1ep1Op1Mgm7CQTOdrimOMdDqdJcqgv10PKnXMlS5De9jmwVupC1CYCdUUVUVA3uCBfTqd0mg00uwn2uHHCNi8RrXOKgGUDzs1QMv4TafTYWdnh8lkQq/XS0RSw0lsu9/v0+/3Sz9DnjccDtOPujrrrBJAuXltWzEqF33lLajc/LW8qpnniZukL1AhoPKL05JY7/ep1KKhpziVl4JzXlRWrPMlslsDlNfLlcJIL67hNB6PGQwGhV6eS61lPsSgWJHwrUxgsfdBXrUos0oABSx5j+vKvfCm6gEsEmYP/s6p5FlSxLhMSOf3lvw6qwRQikWDwYCTk5PCENTF+g5A/u0LzLJzyks16/lsp/Pkq9dXWSWAguKKJ/3jHm98cbYowqoWkzcgVJlQ08EbE2XahFVWGaCAwnJ+eYFrM/f395NYYzAYMBgMli4yhPn+CIpFuQpPnuMr3rdKwwlFr8pLJbA8e+minTjmuRywVLops61rgLpSJb9oB0PH+BYjuabTO8r5Ft85oz/LKgOUuJF3R7yKqSGnSqcWU5ftpJEnxY1GI5WIBWiz2SzsrbAVMcqDuNpWJycnSxryvb29pMtUfMkJZ71ep9/vpzZUrVaj0+kUZkvVzz2wn2WVAcpb6KPRiMPDw8RvFODv3buXyiouf1ZrfXd3l06nQ7fbTR3hEAKdTid9VqPRoNlscu3atcJQ3hrCucoUd5QMy7wrrPglUuoA6vm8SuD54+NqLpzbykodij2SRDebzdQ9yeOK19PFm/yxFnNLQqQ45u2qrZn1vNCWt8zH4zH9fj/NcL4pqWKTLvjk5CSdz2e2nKh6bX1rgrnM0xFn1Ro6WqcHxVq4vNFLxnl7Km+UOkvfGnqg4eHtI2+Zyzva7XYK7F4S9hmt3W4XNAmNRoNr164tNRzU6fGNu9ZZZYDK69bei5N3aNWV76CRs3Vg6QY8UNx9Uef0DGArSsH+zXpep0CuC3MmLS/KVXQS5StYq8wiWqDzaTZ0Hec6u6w+6gbwE8BbgAh8L/BFzrnHnZd/tfu00hA1EMpSGmfaKp0omdbQypsWuTh2U4+6rOLuw8B/jjH+UeBPMFfeXUhxV6vN7wXTarVotVrAIq1x4UV+T7y8ounCslarVbjflbP3MtDW2WX0UdeAPw/8HYAY4wgYhRAupLgDCimMLsa3HFGgVwxS5VLDVTPj6f+TkmYxc7F+L7lsyqMu41F/GPgG8JNhLnb9iRDCHpniDlipuAshfD6E8HlxpLw8clb3RBfv7/EausDKi3VOCx4FM98B/jTwvhjjZ0MIH2bDYXZ6EUlx95rXvCa22+1CN1h1cXV6leh6kqwCnbfZXSksj3nw4EEa0r6Nrk8OV5nrvQC8EGP87Onjn2UO1LkVd7C84mkwGADFNpZikoak4o8zbpV9XWPgQ8zLynqcp0+l/995kHGLMX4d+FoI4Y+cPvU25ne2vtAed/IKfbu9Xo9ut8vR0RHdbrdQdnEVntIZpSXdbjfdfdarEYp98sZcXHbVPOp9wCdCCLvAV4DvYQ7+uRR3McbkQf6chqCCtVrq0kMNBoOCJ0mB50W7GGMCVKBBUaAmL11nlwIqxvibwFtLXjqX4k58KQ/eLi4TdXCdQVmFUrOjxyrVsLyhummOJ6sUM/fNIRR0RT5Vj9KF6m8NWXGmdrudxK0qFTs90BDUmmLv7a2zSgAlywOts29xpjzHy4VkvkI9jzueYLsYbevKLAqyjUaDXq9XIJUhBPb29tIwFI3QcJS3dLvdpVq6z4Tekclzx3VWCaB8SIhNOwnUBSkPdLGr75qhdEfJtCv0vAQ8nU7T3ugey9ZZJYCCYl3Jhas+XNybFKeU/asSqkqBqgV6LWf3+a0ItgYoL5fU6/Uki/ayb6fTSdP/eDzm5ORk6QLlcSKkeq/ikL4QJ6FbU+HUrKZZT79zYuhpSn6BeVUA1i/T37S8IqsEUPV6nZs3b9LpdLh+/TqtVoter5fAyDsnimdeElaA9i3ZNIzzlemeU8KW7XvggVv8Kc/0Ne37kCybsXwIw7KIQ+CJsG6NfDrnMvrGgcKsJtPQbLfbSzFHskYfgqo4uHc1m81Cc/QsqwRQULxvSy5nVmD28q6nMr5IERbVUlUV/NYDGqJ5PWoreBSU7zEOxeRVQ02Wt8Vh0ej0wp7Lg7xS4J68NUPPeZQuWkE5J57+PsUZWAxJ51T5kHa6cB4JdSWAkukf95XjfhHyLlf2ek0pr2kpjcmphYZvXiJeZ5UBSt6k4r/EGO5hXpvyhmguO3R6oJKxAyXasJVbt7k+yrXfZcNOMyKwRB9cxOq1dW+a6jO3Unvgy/bdi2QKxh58/RjNfFrdIKKqoafnyjo4+nudVQYoXxzkKhXfIUO5njrHGqbAEo9yPZRq6Dkf2zqPguKmylKb1Ov1VO/2W8mJM0l6qPe7+s7LKvnsp+fznt86qwxQiidaU3xwcECz2eT69et0Oh1arVbSZT548IDxeMzx8XGhTZ631100pvKMSGu/3080YmuCuUQV2gK33W6naV5y6l6vl/4+Pj5OzVAvwTSbTfb391Mwd46lz1DRz71xE6sMUKpKdjod2u12AmIwGKQ1wfICxRxYKO/a7TZ7e3vcunWrsHOGN03lUcr9NunnySoBFJCGiLa0dUG+T/9A4XYC165do9VqpST38PBwCSRvlAqgvb29RDP0WeusMkB5iRYWDNoDrmudFM/0491ib4B6xcB5lMrJAm8r6IGsTPbjmX5eO/dyr7iYtKBeHVWXWMNYpWb3siv1qBDCPwT+LnO13W8xb6l3uMBdZe2cyQtyc82UgPCGgQMNFDjYeRLgMrvMjQjvAD8AvDXG+Bagznx7pAsp7k7PWagbqZOi4SUNpud4sKg+qGLgM51q8V6bypfGPgqRxg7QDiGMmXvS7wMf4AKKO12UAFFj0/VPzo/yXE2BXxUDmYPjzNw3HdzEyy4j+/k94F8yV6y8CBzGGH+JSyjunHC6qH5VN6as1ZRfdD4hyJyZX6niLsz313wX8CzwAPgPIYTv3vT90RR3t2/fjjs7O+zv7/PUU09xcHBAt9tNSzt08QJLHjSZTFInRkTSt8WVJ+mL0Dq9EEJhq8mrpgd/EfhqjPEbACGEnwP+LBdU3PkQ0Yp1rZY6PX9hA2WfBWXKEV06JFrhWnU9/6iqB78LfGsIoQP0mWuiPg+cMFfafZBzKO4UjHu9HjFGjo+PU9s77x678lcmtq0Vop4s+yqGPA+88pp5nAtcfxb4n8zvq/4bzIfSPhe8q+zpedOPe4Logi4wbz/5rOflYQ/wDlSZvmGdXVZx9yPAj2RPD7nAHnen51sqeazSM+m1HBQo0gw9luWt9E1Agooxcze/kHxtnh/jWinNYg7gunOfJzGuDFA+RSsoa7g5TXBqoHUyPgFo6OmYsiqmzu/ra86ySgDl5FEpiqZxvwgNMQGj8otKyaqZ+9JYKAKm57WTom+Qus4qAZSGkCvqNOupWaDWk5JcF2G4J6qmpd2rx+Mxh4eHBdmj9kHItefrrBJAAUsNgX6/T6/X4/DwMNWnFI8ExN7eXmksa7fbdDqdQvtLwOn8vv/mlc96D9PkPUdHR2k3+7KmgKxs9pJpCKrk60NbwR4222pEVgmgFGMGg0HqEqs+rmao+n5KU1R4Uz3KW+kCWJ6o84p3iVf5HglboTOfTqfcv3+/QAr7/X7q0ykYa2j2+33a7TbD4ZDbt2+nUvBkMr872vHxMcPhMP3udruF1VWKcXnLap1VAigo3hUIigU8EUkFel20WuVqx+caK82EAj8np6voQ5lVBihYvjNQnvSqSbC3t5e6LgrWrl3wfdFFCVwtHEIoxL+t6hTne8yVZfWuQpHX5J1heZYaDPIy3X5ASbZmvK3SR3kSLCubzcSiPWir2+IqO4FUVqzLSyveBltnlQAKiqVgX12VNxnUFD06OuKll15a8gTFNo9J3teTKfi7V62zSgCVq1ZWicN8zYuLwGChxvPfSpbzvp7aWV7G2Qp6IF2Ad4A1lLwpIMmi71RWtpLTFXl67PHKN4x3Lfo6qwRQAsQVKeJR7j35XYHKZkkN1bwg5+VlBXPZVvGonNMoaVVS7FvgegCX+XbdUNwzwbvIqji4PGgTqxRQ/X4/bYHryy8EoLxAfb+9vb1CzNHQ3d/fT7Xz4XDI3bt3U/lFUiH99hv0rLNKAZXHJG9slpWIvcyioedDUMPJ8znREE+ON7FKAOXagVzS7CApNuUrP3NSqtZXntb4sd5w2KTKWQmgVJrNV2X6N+55WtlOra5KOTo6SuUaJcpe4fSltPkStVVWCaBED/Jh4RzIm5neUPCGp+z4+LgQ72CxSalAkrbhrMqmrDJAieuouekdkjxF8Xu8yFN8LY2ki04VHCRfXZp75iqrBFBuXhP3/pyr4lxb4Mpf/+2BG4qLHeW1XtLZihil/A0WS/Z9eYaeA1K7/eTkhHv37qUZTWTU1/rJ61QvV2cnxki32037vZTduyG3M4EKIXwM0N0Z33L63C1WqOpCCB8A3gtMgR+IMf7ipmDlFQDVjXwlg/hWr9dLKmHNbPIkDSd5p/+4Ss9nu4fhUT8F/Djw0/acVHUfDCG8//TxD4UQ3sxcdffHgdcA/yWE8KYY49qMs0xZ4m2q/BivRznfgsXqT1VCFZ9UEVUM9AkjT8DL7MwoFmP8VeBe9vS7mKvpOP391+z5T8YYhzHGrwLPA99y1mcAhQ0f3EvyfE+caN2Pr2b38zifKvusdXbRGLXqzrF3gM/YcS+cPrdkwe4q2+l00lDzcm6+GlS/c2liLgLzicBlPkqIgfS3QHvUSXHZQC/9D6Ip7q5duxbv3buX9AS6qHRSaw54ydd5kHeR1SBVvler1ej1ejx48IB+v5/yvHa7nQT9ZSpkt4sCtUpV9wLwOjvutcwFsJeyPODm8cQ7NvmtMYGlgC+gYPMm6EWB0j52uaru08C/CyF8iHkwfyPwa2edzAknkGYkWRlL13F5K0ocDEjVAhFQ1bxENL2S+jDowc8wl0M/FUJ4gblwrPTOsTHG3w4hfIr5poAT4PvPmvFgUWLxJNZBgmL7SsPES8jypEajkW5c0ev1ktjVz6eZUY8lALkUUDHGd694qVRVF2P8Ueb3Vt/YVKjT31AcZl5P9+TYS8j+W7oFKYvVUpcn6f1+M/utULPUajWuX79e2N9XCxsVbFutFnfv3qXf73Pv3r20/WS32y10UnI6AYvbY+7v73Pt2rXCDVk3tUoA5XmYRGTtdjvNSu12O60AFVH0GbKMW8k01OSRAknVCgX3s6wSQHkT0kmg6uQaLiKTOSF1cplvBui8ST+5+mWrOsV+4xzdA10FPV3Y4eEhg8GAbreb5Dx5CdkbDyrOtdvtFMMU8K9fv54aDlt1D1CvAgCptSR9lEoozsTL2uWz2ayw+kp6zWazWXju+vXrafu3Xq+3HWUW8SiZ8yPfR8pF+W7+WED5fsKKSb71yPHxcRqqsCUNUFneZPD1MfmSM8/jfCGRB+6yHx2rYZs3HVZZJYCazWYpNmnIaUbL7wgEi7zONx51xq587+DgoLAgUu366XTKyy+/vFGHWFYJoOr1OgcHB4l0qtqpwOytKwVpUQgoyoGApaRYm8YrFsmjvMy8FUOv0Wjw9NNP0+v10j/tBDIvzB0cHHBwcMCNGzcKCl8NW812ajTcvXu3MISBVGZWKfgsqwRQusi8mOaE0C/Qj/PZSkApHSkjoj5j6lzujausMkDlexXA8v0/vasiYBWbXKcQwuLO1mqC5kDBYtMtF9SuskoANZnMbyLfarV4xSteQbPZXGo7eZtc4tYHDx6k5xT0XZeuVMYDfl6BaDabCdx1VgmggEI6Ulbqdb2lZkXXSzmdcKG9Sipl3RYv3WxFMAcSHRgOh6WqXYHglMFr3UpBPAnW32q7u1DNFxCplrXOKgHUZDLh/v37hSHiXWD95MPMqwRORvVepSt+WxSvZel9m1QQKgHUbDaj2+0Ci/xtnUd5dcEvWOfSUFXNKZf3eB9vq4BykUU+hfuPM2kF5Hy5q/9ut9s0m01u3LiR8j0XZYi1e519lVUCKCgSRom8vJeXB/Wc9+TKF49FZfkeLFKhrQnmedwpW+6RD0M9lzc7/Ueg5Eth9UXA6hZYbpUACopxZJUWIK9G5rzIPchXLGjpmV6TCfytWVMMxf0HfHZywORJOajK28pqUDIBKg/yZqhqUuusEkCJ6zgTd7WczIGCxX1kvG3lIPkqdaVJPoxXfSFlVgmggMIQgoUIzIehcyuZjhNQ+QZcnkd6Pqjk2SWO66wSQIUQUj+v0+nQ6XTY29uj0+lw8+bN9PfXv/51+v0+9+/fZzAYMBgM0nTvMUoNg6Ojo9TkVPfGOzjeOL20R61Q3P0L4J3ACPgd4HtijA9OX7uQ4s49R3mcnh8Oh5ycnNDtdhNAns/BYo9zX6rvDYoyDbsXBa9Kcfcc8IEY4ySE8GPMt2u7sOJOQVmVx8FgQK/XS6/n3Ravq7s5OwcKTD7nXz5jbrIuZhPtwa+GEJ7Jnvsle/gZ4DtP/06KO+CrIQQp7v7HGZ9RqC3pYsvSE/12oHLy6UDlQTw/V577rbKHEaO+l7nwFS6ouNN639PngeV7gkIxLytb3pEfl+dxeUrk2cCVVg9CCD/MXN7zCT1VctiZirtWqxW140XeDs+/bc8LHShPeO3/S/Wo/HXXVF2pID+E8B7mQf5tcfG1XlhxJ+HFBf6Ppb/zrsyqeJRLiNbZhYAKIbyd+d6afyHG2LOXLqy4E+FU380vKCed4j2rVCv5Rasx6j95J/ph0IMyxd0HgCbw3Ok/9ZkY49+LF1TcAUuB2b91v3BVP3MwdJzneq7ldMYeQig0HTapR4VNO6VXae12Oz7zzDOFwOy8qMxDXA7txTgnkdIaaI9PXxl6fHzMYDBIHerZbMZzzz336zHGt5b9j5Vg5o1Gg2effTYtf221WoUt/vVtay0LLLajXCeAlXe12+00y6lG/oY3vCFJqrXj/jqrBFC1Wi2BpHssiFV7VVPaAc182uPOl//7TCigOp1O2tRGAB4cHFCr1VJqc5ZVAqgYY9pxTOlLWadF8QmKHuUVUQdWXqTz6VxKc/I8cJ1VAijNevnKhJxNq0+Xz4Ye25xH5ev6dOxkMkl5ozdJ11klgNLQU1upjEHrAp0g5uWRvOEAC28FCsPz3r17hTRoK4AKp4o7F9W793jMKatUwnI9S7HMj/HZ0dm686lVVgmgoLidrXdF3KOcVMoEpLPv3KNEOPU5+tE5RXLXWSV4VAjhG8x3rf7mJU/11CXP8YdijK8se6ESQAGEED6/iuw9ynOssgvfSuD/N3sC1IZWJaA+UpFzlFplYlTVrUoeVWl7AtSGVgmgQghvDyF8MYTwfJhvOLHJez4WQng5hPC/7LlbIYTnQghfPv1982H9j48dqBBCHfg3wF8C3gy8+7Q/eJb9FPD27LkL3zfrLHvsQDHv+z0fY/xKjHEEfJJ5f3CtxfPt8HFpqwJQd4Cv2eOVvcANbKP7Zl3EqgDUxr3Ax2lVAOph7r7xUpjv7EE4x32zNrEqAPU54I0hhGdDCLvMRR6fvuC5tMMHnOO+WRvZqmrio/wB3gF8ibmE6Ic3fM/PML+v35i5V74XeAXz2e7Lp79vPaz/8UkKs6FVYehthT0BakN7AtSG9gSoDe0JUBvaE6A2tCdAbWj/D7WPqotxOQq5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Display a sample image\n",
    "(im, features), label = train_set[0]\n",
    "print(im)\n",
    "print(features)\n",
    "print(label)\n",
    "plt.imshow(im.numpy(), cmap='gray')\n",
    "plt.title(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1c0c96c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNNClassifier(nn.Module):\n",
    "    def __init__(self, num_classes=10):\n",
    "        super(CNNClassifier, self).__init__()\n",
    "        # Neural model architecture. Refer to the model for more information\n",
    "        self.cnn_layer1 = nn.Sequential(nn.Conv2d(1, 16, kernel_size=3, padding='valid'),\n",
    "                                        nn.ReLU(),\n",
    "                                        nn.Dropout(0.1),\n",
    "                                        nn.BatchNorm2d(16),\n",
    "                                        nn.MaxPool2d(kernel_size=2))\n",
    "        self.cnn_layer2 = nn.Sequential(nn.Conv2d(16, 32, kernel_size=3, padding='valid'),\n",
    "                                        nn.ReLU(),\n",
    "                                        nn.Dropout(0.2),\n",
    "                                        nn.BatchNorm2d(32),\n",
    "                                        nn.MaxPool2d(kernel_size=2))\n",
    "        self.linear_layer1 = nn.Linear(32*30*3 + 7, 128)\n",
    "        self.dropout1 = nn.Dropout(0.5)\n",
    "        self.activ1 = nn.ReLU()\n",
    "        self.linear_layer_p = nn.Linear(128, 64)\n",
    "        self.dropout_p = nn.Dropout(0.5)\n",
    "        self.activ_p = nn.ReLU()\n",
    "        self.linear_layer2 = nn.Linear(64, num_classes)\n",
    "        \n",
    "    def forward(self, images, features):\n",
    "        images = images.unsqueeze(1)\n",
    "        \n",
    "        cnn1 = self.cnn_layer1(images)\n",
    "        cnn2 = self.cnn_layer2(cnn1)\n",
    "        cnn_vec = cnn2.reshape(cnn2.shape[0], -1)\n",
    "        cnn_cb = torch.cat((cnn_vec, features), dim=1)\n",
    "        out = self.linear_layer1(cnn_cb)\n",
    "        out = self.activ1(out)\n",
    "        out = self.dropout1(out)\n",
    "        out = self.linear_layer_p(out)\n",
    "        out = self.dropout_p(out)\n",
    "        out = self.activ_p(out)\n",
    "        out2 = self.linear_layer2(out)\n",
    "        return out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4a27de62",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_cnn_classifier(model, eval_dataloader):\n",
    "    # Model evaluation: compute the average loss for every image of the dataset\n",
    "    model.eval() \n",
    "    with torch.no_grad():\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for (images, features), labels in eval_dataloader:\n",
    "            images = images.cuda()\n",
    "            features = features.cuda()\n",
    "            labels = labels.cuda()\n",
    "            y_predicted = model(images, features)\n",
    "            _, label_predicted = torch.max(y_predicted.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (label_predicted == labels).sum().item()\n",
    "    accuracy = 100 * correct / total\n",
    "    return accuracy\n",
    "\n",
    "def training_cnn_classifier(model, train_dataloader, num_epochs, loss_fn, learning_rate, verbose=True):\n",
    "    model_tr = copy.deepcopy(model)\n",
    "    model_tr.train()\n",
    "    \n",
    "    # Optimizer and scheduler\n",
    "    optimizer = torch.optim.SGD(model_tr.parameters(), lr=learning_rate)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, 'min')\n",
    "    \n",
    "    loss_all_epochs = []\n",
    "    test_acc_all_epochs = []\n",
    "\n",
    "    if verbose:\n",
    "        print(f'Epoch [0/{num_epochs}], Loss: N/A, Test acc: {eval_cnn_classifier(model_tr, test_dataloader):.4f}%')\n",
    "    \n",
    "    total = len(train_dataloader)\n",
    "    for epoch in range(num_epochs):\n",
    "        loss_current_epoch = 0\n",
    "        for (images, features), labels in tqdm(train_dataloader, total=total):\n",
    "            images = images.cuda()\n",
    "            features = features.cuda()\n",
    "            labels = labels.cuda()\n",
    "            \n",
    "            # Get prediction and loss\n",
    "            y_predicted = model_tr(images, features)\n",
    "            loss = loss_fn(y_predicted, labels)\n",
    "            \n",
    "            # Backpropagation\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            loss_current_epoch += loss.item()\n",
    "        \n",
    "        loss_all_epochs.append(loss_current_epoch)\n",
    "        test_acc = eval_cnn_classifier(model_tr, test_dataloader)\n",
    "        test_acc_all_epochs.append(test_acc)\n",
    "        if verbose:\n",
    "            print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss_current_epoch / total:.4f}, Test acc: {test_acc:.4f}%')\n",
    "        if test_acc > 94:\n",
    "            break\n",
    "        \n",
    "    return model_tr, loss_all_epochs, test_acc_all_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8dbf874c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of parameters:  383466\n",
      "Epoch [0/15], Loss: N/A, Test acc: 6.8796%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 520.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/15], Loss: 1.2090, Test acc: 76.1671%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 522.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [2/15], Loss: 0.5273, Test acc: 85.7494%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 516.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [3/15], Loss: 0.3953, Test acc: 88.6978%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:07<00:00, 507.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [4/15], Loss: 0.3130, Test acc: 92.1376%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 515.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/15], Loss: 0.2395, Test acc: 93.1204%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 517.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [6/15], Loss: 0.1814, Test acc: 92.3833%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 517.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [7/15], Loss: 0.1548, Test acc: 89.6806%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 521.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [8/15], Loss: 0.1328, Test acc: 91.6462%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 520.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [9/15], Loss: 0.1135, Test acc: 88.9435%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 509.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [10/15], Loss: 0.0958, Test acc: 93.8575%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 3562/3562 [00:06<00:00, 510.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [11/15], Loss: 0.0783, Test acc: 94.5946%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxUElEQVR4nO3deXxV9Zn48c+TfSP7AiRAFkBlEdSIBCuiVqXqFNR2yswobj9pLS5tnVpt+xs77Tj1pdPlZ0dtqXWEaaulVitV1FGrokMQgyKrVCAsIYGEhOx78vz+OCfkBkJyyXZz733er9d93Xu/95x7n6Pk+X7Pc77nHFFVjDHGBIcQXwdgjDFm5FjSN8aYIGJJ3xhjgoglfWOMCSKW9I0xJohY0jfGmCDiddIXkVAR+VhEXnbf/0BEDonIZvdxlceyD4jIbhHZJSJXerSfJyJb3c8eExEZ2s0xxhjTl9MZ6d8D7Dyh7WeqOtt9rAUQkWnAEmA6sBB4QkRC3eWfBJYBU9zHwsEEb4wx5vR4lfRFJAu4GnjKi8UXAc+paouqFgO7gTkiMg6IV9VCdc4IWwUsHljYxhhjBiLMy+V+DtwHjDmh/U4RWQoUAfeq6jEgE9jgsUyJ29bmvj6xvU+pqamanZ3tZZjGGGMANm3adFRV005s7zfpi8g1QLmqbhKRBR4fPQn8CFD3+SfArUBvdXrto72331yGUwZi4sSJFBUV9RemMcYYDyKyv7d2b8o7FwJfFJF9wHPApSLyW1U9oqodqtoJ/BqY4y5fAkzwWD8LKHXbs3ppP4mqrlDVfFXNT0s7qaMyxhgzQP0mfVV9QFWzVDUb5wDtX1X1BrdG3+VaYJv7eg2wREQiRSQH54DtRlUtA+pEZK47a2cp8NJQbowxxpi+eVvT780jIjIbp0SzD/gqgKpuF5HVwA6gHViuqh3uOncAzwDRwKvuwxhjzAiR0X5p5fz8fLWavjHGnB4R2aSq+Se22xm5xhgTRCzpG2NMELGkb4wxQSRgk/5/b9jPXz7pdUaoMcYErcHM3hnV/lh0kKiwUP5u1nhfh2KMMaNGwI70C/JS+PjgMZpaO/pf2BhjgkTgJv3cFNo6lKL9Vb4OxRhjRo2ATfrnZycTFiIU7qn0dSjGGDNqBGzSj40MY9aERNZb0jfGmOMCNumDU+LZeqiG+pZ2X4dijDGjQkAn/Xl5KXR0Kh8WW13fGGMgwJP+uZOSiAgNYf2eo74OxRhjRoWATvpR4aGcMzGRwr1W1zfGGAjwpA8wLy+V7aW1VDe2+joUY4zxuYBP+gV5KajCB1bXN8b4A1VobYC6I8Py9QF7GYYusyYkEBUeQuGeSq6cPtbX4RhjAlF7K7TWQ0ud+1wPrXXus/u+pe7ktpPeuw/tdL73+xUQFjGkoXqd9EUkFCgCDqnqNSKSDPwByMa5c9bfq+oxd9kHgNuADuBuVX3dbT+P7jtnrQXu0WG+i0tkWCjnZyfbSVrGmNNTfRB2/gWqD/SenFtqu193eFk+Do2EyDiIiIPIMc5zTDIkTnTbx/T8fBiczkj/HmAnEO++vx94S1UfFpH73fffEZFpOPfSnQ6MB94UkanuLROfBJYBG3CS/kJG4JaJc3NTePT1XVTWt5ASFzncP2eM8Vd1h2H7n2H7C3DwA6ctMt5NwnHdzzGTeibnkxJ2Lwk8Im7IR+0D4VXSF5Es4GrgIeBbbvMiYIH7eiXwDvAdt/05VW0BikVkNzBHRPYB8apa6H7nKmAxI5D05+WlALBhbxVXnz2un6WNT3R2QHMNNB2Dpmpodp+bjkFztfva833XcjXOH9JJf3Aef2g9/jjHnOKP0n0OCfXhfwTjEw1HYcdLsP1F2Pc+oJAxAy79vzDjOkjO9XWEQ8rbkf7PgfsAz/2NDFUtA1DVMhFJd9szcUbyXUrctjb39YntJxGRZTh7BEycONHLEE9tZmYCcZFhrN9z1JL+cOo6AHViYu73fTW01PT93eExEJ0EUYnOc3IuRCdCZIKza+1ZT22uhdpS531X3bSrRtqf8Ng+Rmtdz/HO67gMGDsTUiZbZ+Fvmo7BzpedEf3ed0E7IHUqXPwdJ9GnneHrCIdNv0lfRK4BylV1k4gs8OI7pZc27aP95EbVFcAKcG6M7sVv9iksNITzs5Nsvv5Qam2Akg9hfyEcWA/lO50/pM4+LnkREtYzccdlQNqZ3e+jE3t+Hp3ovk6EsEGU5VShramXA23u+97aPA++1R7qWc9tb+r5/WHRkDHN6QDGnu08MqZBROzAYzZDr6UOPl3rJPrdb0FnGyRlw4X3OIk+YwZIb2kqsHgz0r8Q+KKIXAVEAfEi8lvgiIiMc0f544Byd/kSYILH+llAqdue1Uv7iJiXl8rbu3ZypLaZjPiokfrZwNFYBQc2OAl+/3oo+8RJ8BLi/LGc8QWISfVI1L0k8YhY3/xRiUBEjPOIS+9/+f50tDsdQM1BOLwNDm+Fw1ucWvCmZ7p+1NkDGDvTozOYCWMyBv/7w62zE+pKoaoYjhV3Px/b5xyIPL5NMyF9GoSP4r+n1kb47HXY9if47A1ob4b4LLjgq06iH39uUCR6T3I6k2fckf4/u7N3HgUqPQ7kJqvqfSIyHfg9MAfnQO5bwBRV7RCRD4G7gA9wDuT+QlXX9vWb+fn5WlRUNIBN62nboRqu+cX7/Pwrs1l8Tq9VJeOppqR7FL+/ECp2Ou2hEZB5HkwsgEnzYMIciErwbayjharz3+3w1u6O4PBWqN7fvUxses+kOfZsSMkb+fJQewsc298zqR9P7vuho6V72ZAwSJgAyTnOHtPhbU7JDEBCnVKI5zZlzITYlJHdHk/tLbD7TSfR73oN2hqc/+7Tr3USfdYcCAn4U5QQkU2qmn9i+2Dm6T8MrBaR24ADwJcBVHW7iKwGdgDtwHJ35g7AHXRP2XyVETiI2+WscfHER4VRuKfSkv6JVOHoZ90J/sB6Z5oaODXtiRfAzC85SX78uaN7ZOdLIpA4wXmceVV3e1M1HNnWszMofNwpL4BbHpresyPImO7smQxG07GTE3rVPue5tpQe1dWIOEjKcRL41IVOgk/KccofCRMg1CNVdHZC9T6P7dnqHADd8ofuZeIzT+jcZkJi9vAl24422PuOk+g/fcWZThmdDGf/vZPoJ11ox11cpzXS94WhGukDLFtVxM7Dtbx336VD8n1+q6PdSTwHCp1SzYEN0OhelC42rXsUP7HA+WO1P5ah194KR3f1TJyHtzizkcApm/UoD7mdgWd5qrMT6sp6H61XFTsHyz3Fpncn8xOfY1MHX+ZoqIQj7raUuXs5R//mHCQFZwAxdkbPbUo7a+CDiM4O2PcebHsBdq5xOrnIBDjrGifR51wMoeGD2yY/Nhwjfb9TkJfC/+w4QsmxRrKSBjmK8idtTXBoU/co/uBGpyYNkDgJplwBkwpg4jyn1BBkNU6fCIvoTnxdVN3jBB4dQcmHzui1S1wGpEyBhgqnbNTe3P2ZhDon+STnwIxze47Wk7KdGUfDKTYFchc4jy5tTc5Bfs9t2vz77n9/vZWHxp7tnLDUm85OOLjBSfQ7/uz8dwiPdfasZlwPeZcO7qB/EAiqpD8vLxWAwj2VfDk/gJN+U7WT2Pf/rzOaL/24+4zB9Okwa0n3aD5+vE9DNR5EnKSdOBHOvLq7vekYHNnenTSPfgapU2DqFT1H6yeWYUaD8GjIPNd5dOnsdPZGPDuC4vf6Lg/FpLgzb150DjKHRcHUK2H6dc6gZbClsCAyyv6FDK+pGXGkxEa4SX9C/yv4g84OZxf60EdQ+hEc+MCpH6POAbjx58AFX3MPul5w6hGUGb2ikyD7c84jEISEOHuUKXkwfXF3e8PRk8tdn/1P9zkWIeEw5XKY/kM4Y+GwXaYg0AVV0hcR5uamULi3ElVF/K2M0TVCOvSRM3ov/diZOtnW4HweEQdZ+bDgAadck5lvIyDjP2JTIe8S59GlrQnKd0BtmdPpRSf6LLxAEVRJH5y6/itby9hX2UhO6ig+eaarvlv6sUeS39x95mpYlFP7POcGZ9d5/DlOrTcIpqKZIBIe7UwRtgl3QyYokz44df1RlfTrDp+Q4D/unlETEu5M4ZtxXXeCTzszqGcmGGMGJuiSfm5qLOljIincW8k/XjD46/oMSGOVU38/9HF3gq9zT06WEGca29SFMH62k+TTp9vceGPMkAi6pC8izMtL4f3dI1TXb65x6u6eo3jPMzRTJju1yvHnOAl+7Ey7ZosxZtgEXdIHp8Tz582l7C6vZ0rGEM8AaKyCLaudefGlH0PlZ92fJU50zmjNv9VJ8ONm2SUMjDEjKiiT/vH5+nsrhzbpf/oK/OUb0FAOY8Y5Cf7srzij+PHn+PZ6JMYYQ5Am/QnJMWQmRrN+dyVLC7IH/4WNVfDqfbD1j87Fpv5ptZPkjTFmlAnKpA9OiefNnUfo7FRCQgZR19/5Mrz8TWiqcubHf+5bo+KWaMYY05ugndQ9Ly+F6sY2dh6uHdgXNFbB87fBH/7JuR7K7W/Dgvst4RtjRrWgHumDM19/+vjTPJi68y/u6P6YM7q/6F6bM2+M8QtBO9IflxBNTmosG07nFooNle7o/gYYMxaWveOM7i3hG2P8RL9JX0SiRGSjiHwiIttF5F/d9h+IyCER2ew+rvJY5wER2S0iu0TkSo/280Rkq/vZY+Lji9/MzU3hg71VtHd4cdPsHWvgiQtgx0twyfecco7nZXGNMcYPeDPSbwEuVdVZwGxgoYjMdT/7marOdh9rAURkGrAEmA4sBJ4Qka67cDwJLAOmuI+FQ7YlA1CQl0JdSzvbS/uo6zdUwvO3wuobnWmYy96Bi++z0b0xxi/1m/TV4d7xgHD30dftthYBz6lqi6oWA7uBOe7N0+NVtVCd23WtAhYPKvpBKsh16/qnKvEcH92vcUf3f3Xu/GOMMX7Kq5q+iISKyGagHHhDVT9wP7pTRLaIyNMikuS2ZQIHPVYvcdsy3dcntvtM2phIpqTHsX7PCUm/oRL+eIszuo8fb6N7Y0zA8Crpq2qHqs4GsnBG7TNwSjV5OCWfMuAn7uK91em1j/aTiMgyESkSkaKKigpvQhywgrwUivZV0dru1vV3vASPz3Fm6Fz6ffg/b9no3hgTME5r9o6qVgPvAAtV9YjbGXQCvwbmuIuVAJ63pcoCSt32rF7ae/udFaqar6r5aWlppxPiaZuXl0Jjawc7du+BP94Mq5dCQiZ89V2Y/20b3RtjAoo3s3fSRCTRfR0NfB741K3Rd7kW2Oa+XgMsEZFIEcnBOWC7UVXLgDoRmevO2lkKvDR0mzIwF+SkcFXoB0z90+XO2bVdo/uM6b4OzRhjhpw3J2eNA1a6M3BCgNWq+rKI/LeIzMYp0ewDvgqgqttFZDWwA2gHlqtqh/tddwDPANHAq+7DdxqOkvTKvTwR/mf2MoXcr66FjGk+DckYY4ZTv0lfVbcAJ109TFVv7GOdh4CHemkvAkZHgXz7i/DKvdBSx18zv8by/RfxcfIZ2K1KjDGBLPjOyK2vcOr2f7zZub79snfpvPBbNLULHx+o9nV0xhgzrILr2jvbXoC1/wwtdXDZv8C8eyA0jDkJbYSIM1+/65o8xhgTiIIj6ddXwNp7nemY48+FxU9A+lnHP46PCmdmZgKFe47C5VN9GKgxxgyvwE76qk7tvmt0//kfQMFdEHryZs/NS+Hp94tpbG0nJiKw/7MYY4JX4Nb068ud2v3zt0DiJPjqe/C5b/aa8MG5hWJbh1K079gIB2qMMSMnMIe0HW3w1OehrqzP0b2n/ElJhIUIhXsrmT91eE8IM8YYXwnMpB8aDgt/DMl5kH6mV6vERoYxe0IihSdeh8cYYwJI4JZ3zrza64TfpSAvha2HaqhrbhumoIwxxrcCN+kPQEFuCh2dyof7qnwdijHGDAtL+h7OnZRERFiIlXiMMQHLkr6HqPBQzp2YePL19Y0xJkBY0j9BQW4qO8pqqW5s9XUoxhgz5Czpn2De5BRUYcNeq+sbYwKPJf0TzMpKJDo8lA2num+uMcb4MUv6J4gICyE/O8kO5hpjApIl/V4U5KWw60gdR+tbfB2KMcYMKW9ulxglIhtF5BMR2S4i/+q2J4vIGyLymfuc5LHOAyKyW0R2iciVHu3nichW97PH3NsmjjoFuc7lla3EY4wJNN6M9FuAS1V1FjAbWCgic4H7gbdUdQrwlvseEZkGLAGmAwuBJ9xbLQI8CSzDuW/uFPfzUWdmZgJxkWFW4jHGBJx+k7466t234e5DgUXASrd9JbDYfb0IeE5VW1S1GNgNzHFvpB6vqoWqqsAqj3VGlbDQEObkJFvSN8YEHK9q+iISKiKbgXLgDVX9AMhQ1TIA9zndXTwTOOixeonblum+PrG9t99bJiJFIlJUUVFxGpszdApyU9h7tIHDNc0++X1jjBkOXiV9Ve1Q1dlAFs6ova+bm/dWp9c+2nv7vRWqmq+q+WlpvrnMcddtEwv3HvXJ7xtjzHA4rdk7qloNvINTiz/ilmxwn8vdxUqACR6rZQGlbntWL+2j0rRx8SREh1uJxxgTULyZvZMmIonu62jg88CnwBrgJnexm4CX3NdrgCUiEikiOTgHbDe6JaA6EZnrztpZ6rHOqBMSIszNTabQZvAYYwKINzdRGQesdGfghACrVfVlESkEVovIbcAB4MsAqrpdRFYDO4B2YLmqdrjfdQfwDBANvOo+Rq2C3BRe336Eg1WNTEiO8XU4xhgzaP0mfVXdApzTS3slcNkp1nkIeKiX9iKgr+MBo0pBXioAhXsrLekbYwKCnZHbh6kZcaTERrDB6vrGmABhSb8PIsLcvBTW76nEObXAGGP8myX9fszLS+FwbTP7Kht9HYoxxgyaJf1+dF2HZ/0em69vjPF/lvT7kZMaS0Z8pM3XN8YEBEv6/RAR5uWlsmGv1fWNMf7Pkr4XCnJTOFrfymfl9f0vbIwxo5glfS8cvw6PlXiMMX7Okr4XJiTHkJUUbUnfGOP3LOl7qSA3hQ3FlXR2Wl3fGOO/LOl7ad7kFKob29h5uNbXoRhjzIBZ0vdSQa57HR4r8Rhj/JglfS+NTYgiJzXWkr4xxq9Z0j8NBXkpbCyuor2j09ehGGPMgFjSPw0FuSnUtbSzrdTq+sYY/+TNnbMmiMjbIrJTRLaLyD1u+w9E5JCIbHYfV3ms84CI7BaRXSJypUf7eSKy1f3sMfcOWn5jbq7N1zfG+DdvRvrtwL2qehYwF1guItPcz36mqrPdx1oA97MlwHSce+k+4d51C+BJYBnOLRSnuJ/7jbQxkUzNiLNbKBpj/Fa/SV9Vy1T1I/d1HbATyOxjlUXAc6raoqrFwG5gjnvz9HhVLVTnIjargMWD3YCRVpCbwofFVbS2W13fGON/TqumLyLZOLdO/MBtulNEtojI0yKS5LZlAgc9Vitx2zLd1ye2+5WCvFSa2jrYUlLt61CMMea0eZ30RSQO+BPwDVWtxSnV5AGzgTLgJ12L9rK69tHe228tE5EiESmqqKjwNsQRMTc3GRFYb3V9Y4wf8irpi0g4TsL/naq+AKCqR1S1Q1U7gV8Dc9zFS4AJHqtnAaVue1Yv7SdR1RWqmq+q+WlpaaezPcMuMSaCs8bG28FcY4xf8mb2jgC/AXaq6k892sd5LHYtsM19vQZYIiKRIpKDc8B2o6qWAXUiMtf9zqXAS0O0HSNqXl4Kmw4co7mtw9ehGGPMafFmpH8hcCNw6QnTMx9xp19uAS4BvgmgqtuB1cAO4DVguap2Zcc7gKdwDu7uAV4d0q0ZIQV5KbS2d/LRgWO+DsUYY05LWH8LqOr79F6PX9vHOg8BD/XSXgTMOJ0AR6Pzc5IJEdiwp5J5eam+DscYY7xmZ+QOQHxUODOzEm2+vjHG71jSH6CC3BQ2H6ymsbXd16EYY4zXLOkP0Ly8FNo6lKJ9Vtc3xvgPS/oDlJ+dRHio2Hx9Y4xfsaQ/QDERYcyyur4xxs9Y0h+EeXkpbC2ppra5zdehGGOMVyzpD8LcvBQ6FT4srvJ1KMYY4xVL+oNw7sQkIsJC7JIMxhi/YUl/EKLCQzlvYpLV9Y0xfsOS/iAV5KWwo6yW6sZWX4dijDH9sqQ/SPPyUlCFDXutrm+MGf0s6Q/S2VmJRIeHUrjnqK9DMcaYflnSH6SIsBDys62ub4zxD5b0h8C8vFT+dqSeiroWX4dijDF9sqQ/BAryUgDYYKN9Y8woZ0l/CMwYH8+YyDAr8RhjRj1vbpc4QUTeFpGdIrJdRO5x25NF5A0R+cx9TvJY5wER2S0iu0TkSo/289y7be0Wkcfc2yb6vbDQEObkJLPBTtIyxoxy3oz024F7VfUsYC6wXESmAfcDb6nqFOAt9z3uZ0uA6cBC4AkRCXW/60lgGc59c6e4nweEgrwU9h5t4HBNs69DMcaYU+o36atqmap+5L6uA3YCmcAiYKW72Epgsft6EfCcqraoajHO/XDnuDdSj1fVQlVVYJXHOn6vq65fuNembhpjRq/TqumLSDZwDvABkKGqZeB0DEC6u1gmcNBjtRK3LdN9fWJ7b7+zTESKRKSooqLidEL0mbPGxpMYE8763VbiMcaMXl4nfRGJA/4EfENVa/tatJc27aP95EbVFaqar6r5aWlp3oboUyEhwgU5yXYw1xgzqnmV9EUkHCfh/05VX3Cbj7glG9zncre9BJjgsXoWUOq2Z/XSHjDm5aVScqyJg1WNvg7FGGN65c3sHQF+A+xU1Z96fLQGuMl9fRPwkkf7EhGJFJEcnAO2G90SUJ2IzHW/c6nHOgHheF3fZvEYY0Ypb0b6FwI3ApeKyGb3cRXwMHC5iHwGXO6+R1W3A6uBHcBrwHJV7XC/6w7gKZyDu3uAV4dyY3xtSnocqXERVuIxxoxaYf0toKrv03s9HuCyU6zzEPBQL+1FwIzTCdCfiAhzc1NYv+coqkqAnIZgjAkgdkbuELvkjHSO1LZw93ObqW9p93U4xhjTQ78jfXN6rj0nkyN1zfzH67vYfqiGJ244lzPHxvs6LGOMAWykP+RCQoSvL5jM72+fS31LO4sf/19WFx3sf0VjjBkBlvSHydzcFF65+yLOm5TEfc9v4Z//+AlNrR39r2iMMcPIkv4wShsTyapbL+Duy6bwp49KWPz4/7Knot7XYRljgpgl/WEWGiJ86/KprLxlDhX1LXzxF++z5pOAOifNGONHLOmPkPlT01h790WcNS6eu5/9mO//eSvNbVbuMcaMLEv6I2hsQhTPLpvLVy/O5bcbDvClX67nQKVdssEYM3Is6Y+w8NAQHvjCWfx6aT4HKhu5+hfv8dq2w74OyxgTJCzp+8jl0zJ45e6LyE2N5Wu/3cS/vbyDto5OX4dljAlwlvR9aEJyDKu/VsDN87J56v1ivvKrQkqrm3wdljEmgFnS97HIsFB+8MXp/Oc/nsPfjtRz9WPv8fau8v5XNMaYAbCkP0pcc/Z41tx5IRnxUdzyXx/y6Ouf0m7lHmPMELOkP4rkpsXx5+UXsuT8CTz+9h5u+M0HlNfajdaNMUPHkv4oExUeysPXn81PvjyLTw7WcNVj77N+j91s3RgzNLy5c9bTIlIuIts82n4gIodOuKlK12cPiMhuEdklIld6tJ8nIlvdzx4Tu9h8n64/L4uX7ryQhOgwbnjqA37x1md0dvZ6S2FjjPGaNyP9Z4CFvbT/TFVnu4+1ACIyDVgCTHfXeUJEQt3lnwSW4dw+ccopvtN4mJoxhjV3fo6/mzWen7zxN25+5kMq61t8HZYxxo/1m/RVdR1Q5eX3LQKeU9UWVS3GuS3iHPfG6fGqWqiqCqwCFg8w5qASGxnGz78ym3+/diYb9lZy9WPvU7TP2/8dxhjT02Bq+neKyBa3/JPktmUCnhePL3HbMt3XJ7YbL4gI/3jBRF64Yx6R4SF8ZcUGVqzbg9N/GmOM9waa9J8E8oDZQBnwE7e9tzq99tHeKxFZJiJFIlJUUVExwBADz4zMBP5y1+e4/KwM/n3tp9y+ahM1jW2+DssY40cGlPRV9YiqdqhqJ/BrYI77UQkwwWPRLKDUbc/qpf1U379CVfNVNT8tLW0gIQas+KhwnrzhXP7lmmm8s6ucq3/xHltKqn0dljHGTwwo6bs1+i7XAl0ze9YAS0QkUkRycA7YblTVMqBOROa6s3aWAi8NIu6gJiLc+rkcVn+tgM5O5UtPFrKqcJ+Ve4wx/fJmyuazQCFwhoiUiMhtwCPu9MstwCXANwFUdTuwGtgBvAYsV9Wui8bfATyFc3B3D/DqUG9MsDl3YhKv3H0RF05O4V9e2s6dz35MTZOVe4wxpyajfXSYn5+vRUVFvg5jVOvsVH65bg//8fouYiPDWFowiVsuzCE1LtLXoRljfERENqlq/kntlvQDx/bSGh5/ezevbjtMRGgIS86fwO3zc8lKivF1aMaYEWZJP4jsqajnV+/u4cWPD6EKX5w9njsuzmNKxhhfh2aMGSGW9INQaXUTT71XzLMbD9DU1sEV0zL4+iWTmT0h0dehGWOGmSX9IFbV0Moz6/excv0+apramJeXwtcXTObCySnYJZCMCUyW9A31Le08+8EBfv3eXsrrWjg7K4GvL8jjimljCQmx5G9MILGkb45rae/gT5sO8at1e9hf2UheWixfuziPRbMziQizq20bEwgs6ZuTdHQqa7eW8cQ7e9hZVsv4hChun5/LkvMnEh0R2v8XGGNGLUv65pRUlXd2VfDEO7v5cN8xkmMjuGVeNksLskmICfd1eMaYAbCkb7zy4b4qnnh7N2/vqiA2IpQb5k7its/lkB4f5evQjDGnwZK+OS07Smt58t09vLKllLDQEL50XhZfnZ/LpJRYX4dmjPGCJX0zIPuONvCrdXv506YS2js7uebs8dyxII+zxsX7OjRjTB8s6ZtBOVLbzG/eL+Z3G/bT0NrBpWem8/UFeeRnJ/s6NGNMLyzpmyFR09jGysJ9/Nf/FnOssY052cnccUkeC6am2YlexowilvTNkGpsbee5jQf59Xt7Katp5syxY7jlwmy+OCvTpnsaMwpY0jfDorW9kz9vPsRv3itm15E6EqLD+cr5E7jhgklMTLGrexrjK5b0zbBSVTYWV7GqcD+vbT9MpyqXnpHO0nnZXDQ51S7zYMwIO1XS9+bOWU+LSLmIbPNoSxaRN0TkM/c5yeOzB0Rkt4jsEpErPdrPc++2tVtEHhMrAAcUEeGC3BQe/6dzef87l3DXJZP5pKSam57eyGU/fZen3y+mttnu6mWMr/U70heR+UA9sEpVZ7htjwBVqvqwiNwPJKnqd0RkGvAszo3SxwNvAlNVtUNENgL3ABuAtcBjqtrvLRNtpO+/Wto7eG3bYVau38dHB6qJiQjl2nMyWVqQzRlj7dr+xgynU430w/pbUVXXiUj2Cc2LgAXu65XAO8B33PbnVLUFKBaR3cAcEdkHxKtqoRvMKmAxdp/cgBYZFsqi2Zksmp3J1pIaVhXu44+bSvjdBweYm5vMTQXZXD4tg7BQu8ibMSNloH9tGapaBuA+p7vtmcBBj+VK3LZM9/WJ7SZIzMxK4NEvz2LDA5dx/xfO5GBVE3f87iMueuRtHn97N0frW3wdojFBYaiHWL3V6bWP9t6/RGSZiBSJSFFFRcWQBWd8Lzk2gq9dnMe6+y7h10vzmZwex6Ov72Lej//Kt/6wmc0Hq30dojEBrd/yzikcEZFxqlomIuOAcre9BJjgsVwWUOq2Z/XS3itVXQGsAKemP8AYzSgWGiJcPi2Dy6dlsLu8nv8u3Mfzm0p44eNDzMpKYGlBNlefPY6ocJvzb8xQGuhIfw1wk/v6JuAlj/YlIhIpIjnAFGCjWwKqE5G57qydpR7rmCA3OT2Of100gw3fvYwfLppOfUs79/7xE+Y9/Fceee1TDlU3+TpEYwKGN7N3nsU5aJsKHAEeBP4MrAYmAgeAL6tqlbv894BbgXbgG10zdEQkH3gGiMY5gHuXenGSgM3eCT6qyvo9laxcv483dx4B4PJpGdxUkE1Bnt3X1xhv2MlZxi8dqm7idxv289yHB6lqaGVyehw3FUzi2nOziIscaHXSmMBnSd/4tea2Dl7eUsbK9fvYeqiGuMgwvnReFjcWTCIvLc7X4Rkz6ljSNwFBVdl8sJpVhft5ZUsZrR2dnDMxkatnjuMLM8eRmRjt6xCNGRUCKum3tbVRUlJCc3Ozj6IaHlFRUWRlZREebvel9UZFXQvPbyrh5S2lbC+tBTjeAVw1cxzjrQMwQSygkn5xcTFjxowhJSVwDuqpKpWVldTV1ZGTk+PrcPzOvqMNvLK1jLVby453AOdOTOQq6wBMkAqopL9z507OPPPMgEn4XVSVTz/9lLPOOsvXofi1rg7glS1l7CizDsAEpwFfe2e0CrSED4G5Tb6QnRrL8ksms/ySyT06gH97ZSf/9spO6wBMUPPbpO9rcXFx1NfX+zoM0w/PDqD4aANre+kArj57PFfNHMu4BOsATOCzpG+CRs4pOoAfvbyDH728g/MmJbl7ANYBmMBl17QdJFXl29/+NjNmzGDmzJn84Q9/AKCsrIz58+cze/ZsZsyYwXvvvUdHRwc333zz8WV/9rOf+Tj64NXVAay95yLe/ucFfPvKM2hq7eBHL++g4Md/5fon1/Ob94spq7FLQJjA4vcj/X/9y3Z2uLM1hsq08fE8+HfTvVr2hRdeYPPmzXzyySccPXqU888/n/nz5/P73/+eK6+8ku9973t0dHTQ2NjI5s2bOXToENu2OTchq66uHtK4zcD0tgfwsu0BmADl90nf195//33+4R/+gdDQUDIyMrj44ov58MMPOf/887n11ltpa2tj8eLFzJ49m9zcXPbu3ctdd93F1VdfzRVXXOHr8M0JPDuAvRX1Tglo6+EeHYBzIph1AMY/+X3S93ZEPlxONeV1/vz5rFu3jldeeYUbb7yRb3/72yxdupRPPvmE119/nccff5zVq1fz9NNPj3DExlu5aXHceekU7rx0So8O4Icv7+CHL+/gzLFjmJweR25aHHlpseSlxZGTGkusXRPIjGL2r3OQ5s+fz69+9StuuukmqqqqWLduHY8++ij79+8nMzOT22+/nYaGBj766COuuuoqIiIiuP7668nLy+Pmm2/2dfjGS711AEX7j7GlpIa1W8vo9Oj7x8ZHkZsWS67bEeSmxZGbGktmYjQhITYt1/iWJf1BuvbaayksLGTWrFmICI888ghjx45l5cqVPProo4SHhxMXF8eqVas4dOgQt9xyC52dnQD8+Mc/9nH0ZiC6OoAuzW0d7K9sZG9FPXuPNrCnop69FQ28tLmUuub248tFhoWQk9rVETidQm6q83pMlF16w4wMvz0jN1DPWg3kbQs2qsrR+tbjncFetzPYU1HPwWNNdHjsHqSPiXQ7AmevIC8tjry0ODKTogm1vQMzAAF3Rq4xo52IkDYmkrQxkVyQm9Ljs9b2Tg5UNbCnouF4R9BVNqpubDu+XERYCNkpMcf3CHLdvYSJyTGkxEbYWdzmtA0q6YvIPqAO6ADaVTVfRJKBPwDZwD7g71X1mLv8A8Bt7vJ3q+rrg/l9Y/xVRFgIk9PHMDl9zEmfVTW09tgr2FPRwN/K63hz5xHaPfYOosNDyUqKdh8xTEh2nrveJ8WEW6dgTjIUI/1LVPWox/v7gbdU9WERud99/x0RmQYsAaYD44E3RWSqqnYMQQzGBIzk2AiSY5PJz07u0d7W0cnBqkb2VjRQcqyRkmNNHHSfPzpQTU1TW4/lYyNCj3cCE5JjenYQSTHER4dZpxCEhqO8swjnnroAK4F3gO+47c+pagtQLCK7gTlA4TDEYEzACQ8Nccs7vd8prLa5jZKqppM6hJJjTWwsrqKupb3H8mMiw8jsZS9hQlIMWcnRxNvB5YA02KSvwP+IiAK/UtUVQIaqlgGoapmIpLvLZgIbPNYtcdtOIiLLgGUAEydOHGSIxgSH+Khwpo0PZ9r4+JM+U1Vqm9o9OgLP50YK9xylobXjhO8LO6lDyEyMZlxCNGMTokiJjbApqH5osEn/QlUtdRP7GyLyaR/L9vavo9epQ27nsQKc2TuDjNGYoCciJMSEkxCTwIzMhJM+V1WqG9s89hC69xL2VjSw7m9HaWrr2SlEhIaQkRDJuPhoxiVGMTYhinHxUYxLjGZcgvM+NTbSOoZRZlBJX1VL3edyEXkRp1xzRETGuaP8cUC5u3gJMMFj9SygdDC/b4wZGiJCUmwESbERzMzqvVOobGilrLqZ0pomDtc0U1bTTFlNE2U1zXx8oJrDNc20dnT2WC88VMiIj2JcQhTjEro7g67X4xKiSI2zjmEkDTjpi0gsEKKqde7rK4AfAmuAm4CH3eeX3FXWAL8XkZ/iHMidAmwcROzGmBEiIqTGRZIaF9lrpwDQ2alUNbZyuKaZ0uomDte6HUO10zF8UlLNa9ubaW3v2TGEhXh0DF17CfFRjE+MYqzbOaTGRdr5CkNkMCP9DOBF9+h/GPB7VX1NRD4EVovIbcAB4MsAqrpdRFYDO4B2YLk/z9xZvHgxBw8epLm5mXvuuYdly5bx2muv8d3vfpeOjg5SU1N56623qK+v56677qKoqAgR4cEHH+T666/3dfjGDLmQkO6OobcSEjh7DFUNre5eQjOHa5oorWl29xya2FJSzeun6BjSxkSSHh9FxphIMuKdPYZ097XziCQh2qap9mfASV9V9wKzemmvBC47xToPAQ8N9Dd79er9cHjrkH4lY2fCFx7uc5Gnn36a5ORkmpqaOP/881m0aBG3334769atIycnh6qqKgB+9KMfkZCQwNatTozHjh0b2liN8SMiQkpcJCn9dAzHGtucvYWaZspqnb2F8roWjtQ2s6+ygQ+Kq06aogrO+Q8Z8ZGMjY9yOwinM8iIjyLdbc+Ijwrqi+IF75YP0mOPPcaLL74IwMGDB1mxYgXz588nJycHgORkZ471m2++yXPPPXd8vaSkpJEP1hg/IiLuuQoRp+wYwLnmUXltC0fqmjlS28yR2hb32XnsLK3l7dpyGltPLijERYaRHh/Z3SkkdHUQ3Z1E2phIosJDh3NTfcL/k34/I/Lh8M477/Dmm29SWFhITEwMCxYsYNasWezateukZVXVdjeNGQZR4aFMTIlhYkpMn8vVt7RzuKaZ8tpmt4NwOody97lo/zHKa1tOOggNkBgTTsYYZy8hI96znBRJ2piu50giw/ync/D/pO8DNTU1JCUlERMTw6effsqGDRtoaWnh3Xffpbi4+Hh5Jzk5mSuuuIL//M//5Oc//znglHdstG/MyImLDGNyehyT03s/qQ26p6z27BSaOdzVOdS1sLv8KBV1LT0uhdElKSb8+N5BV6eQ7nYK6W5nMVo6B0v6A7Bw4UJ++ctfcvbZZ3PGGWcwd+5c0tLSWLFiBddddx2dnZ2kp6fzxhtv8P3vf5/ly5czY8YMQkNDefDBB7nuuut8vQnGGA+eU1bPHHvq5bpmKB2pbaa8rsXZe6htodztLMprm9ldXk95XUuPq6h2SY6NIN3jgHT3HkT33kRaXCQRYcN3+3K7tPIoE8jbZkyw6Ox0zmsor+suI3UdiD5S20KF20lU1PfdOTx/xzziBnjQ2S6tbIwxIyQkpPuy2tPHn3q5jk5nCuuR2mYqPDqF8jrnfWzE0JeDLOkbY4yPhHp0DiNl+ApHxhhjRh2/Tfqj/VjEQATiNhljRhe/TPpRUVFUVlYGVJJUVSorK4mKivJ1KMaYAOaXNf2srCxKSkqoqKjwdShDKioqiqysLF+HYYwJYH6Z9MPDw49f7sAYY4z3/LK8Y4wxZmAs6RtjTBCxpG+MMUFk1F+GQUQqgP0DXD0VODqE4fgD2+bgEGzbHGzbC4Pf5kmqmnZi46hP+oMhIkW9XXsikNk2B4dg2+Zg214Yvm228o4xxgQRS/rGGBNEAj3pr/B1AD5g2xwcgm2bg217YZi2OaBr+sYYY3oK9JG+McYYDwGZ9EVkoYjsEpHdInK/r+MZbiIyQUTeFpGdIrJdRO7xdUwjRURCReRjEXnZ17GMBBFJFJHnReRT9/93ga9jGm4i8k333/U2EXlWRALuqoQi8rSIlIvINo+2ZBF5Q0Q+c5+H5ObaAZf0RSQUeBz4AjAN+AcRmebbqIZdO3Cvqp4FzAWWB8E2d7kH2OnrIEbQ/wNeU9UzgVkE+LaLSCZwN5CvqjOAUGCJb6MaFs8AC09oux94S1WnAG+57wct4JI+MAfYrap7VbUVeA5Y5OOYhpWqlqnqR+7rOpxEkOnbqIafiGQBVwNP+TqWkSAi8cB84DcAqtqqqtU+DWpkhAHRIhIGxAClPo5nyKnqOqDqhOZFwEr39Upg8VD8ViAm/UzgoMf7EoIgAXYRkWzgHOADH4cyEn4O3Ad0+jiOkZILVAD/5Za0nhKRWF8HNZxU9RDwH8ABoAyoUdX/8W1UIyZDVcvAGdgB6UPxpYGY9KWXtqCYoiQiccCfgG+oaq2v4xlOInINUK6qm3wdywgKA84FnlTVc4AGhmiXf7Ry69iLgBxgPBArIjf4Nir/FohJvwSY4PE+iwDcHTyRiITjJPzfqeoLvo5nBFwIfFFE9uGU8C4Vkd/6NqRhVwKUqGrXXtzzOJ1AIPs8UKyqFaraBrwAzPNxTCPliIiMA3Cfy4fiSwMx6X8ITBGRHBGJwDnos8bHMQ0rERGcOu9OVf2pr+MZCar6gKpmqWo2zv/jv6pqQI8AVfUwcFBEznCbLgN2+DCkkXAAmCsiMe6/88sI8IPXHtYAN7mvbwJeGoov9cs7Z/VFVdtF5E7gdZwj/U+r6nYfhzXcLgRuBLaKyGa37buqutZ3IZlhchfwO3dAsxe4xcfxDCtV/UBEngc+wpml9jEBeHauiDwLLABSRaQEeBB4GFgtIrfhdH5fHpLfsjNyjTEmeARieccYY8wpWNI3xpggYknfGGOCiCV9Y4wJIpb0jTEmiFjSN8aYIGJJ3xhjgoglfWOMCSL/H5Y69qXM0qiIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Model training\n",
    "\n",
    "batch_size = 1\n",
    "train_dataloader = DataLoader(train_set, shuffle=True, batch_size=batch_size)\n",
    "test_dataloader = DataLoader(test_set, batch_size=batch_size)\n",
    "\n",
    "num_classes = 10\n",
    "model = CNNClassifier(num_classes)\n",
    "\n",
    "print('Total number of parameters: ', \n",
    "      sum(p.numel() for p in model.parameters()))\n",
    "\n",
    "torch.manual_seed(0)\n",
    "model.apply(init_weights)\n",
    "\n",
    "num_epochs = 15\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "learning_rate = 0.01\n",
    "\n",
    "model.cuda()\n",
    "model, loss_total, test_acc_total = training_cnn_classifier(model, train_dataloader, num_epochs, loss_fn, learning_rate, verbose=True)\n",
    "\n",
    "torch.save(model, 'model_cnn_classif.pt')\n",
    "plt.plot(loss_total, label='loss')\n",
    "plt.plot([max(loss_total)*x/100 for x in test_acc_total], label='acc')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f3090437",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94.5945945945946\n"
     ]
    }
   ],
   "source": [
    "# See final accuracy\n",
    "acc = eval_cnn_classifier(model, test_dataloader)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9b001be9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           2     0.9412    0.9697    0.9552        33\n",
      "           9     0.9000    0.9375    0.9184        48\n",
      "           a     1.0000    0.9130    0.9545        46\n",
      "           E     0.7500    1.0000    0.8571        24\n",
      "           e     1.0000    0.8667    0.9286        60\n",
      "           i     0.9730    0.9730    0.9730        37\n",
      "           O     1.0000    0.8974    0.9459        39\n",
      "           o     0.9706    0.9706    0.9706        34\n",
      "           u     0.9038    1.0000    0.9495        47\n",
      "           y     1.0000    1.0000    1.0000        39\n",
      "\n",
      "    accuracy                         0.9459       407\n",
      "   macro avg     0.9439    0.9528    0.9453       407\n",
      "weighted avg     0.9527    0.9459    0.9467       407\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f4bec91a260>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATMAAAEGCAYAAAAJ73JAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtJUlEQVR4nO2deXxU9dX/3yeTEJKQACGAgCCIoKJVFFSQqkhrta0t2ro+2mJtS2m1Wpe2av09tvrUarXaqrWVahWXutUFrAsiFcVaFRBUZBEVRCASVkkCZJk5vz/ujQQkMzcz907mm5z363VfmXvnzucehnD4rucjqophGIbr5LV1AIZhGGFgycwwjHaBJTPDMNoFlswMw2gXWDIzDKNdkN/WAexKrKxEC3p2C1230/LtoWsCYLPBhmNsp5Z6rZNMNI4/tkQ3bIwHunfe23XTVfWETJ4XhJxLZgU9uzHg+h+FrjvwrKWhawJoQ30kuoYRFa/rzIw11m+M8/r0PQPdW9Dng4qMHxiAnEtmhmG4gBLXRFsHsROWzAzDaDUKJMitIRZLZoZhpEUCa5kZhuE4itJg3UzDMFxHgXiI3UwRWQFUA3GgUVVHikg58DAwEFgBnKaqm1rScCOZ1SfY46oPkUaFuLJ1VFc+Pa033e6rpHheNZovNPbuxPqf7ImWxNJ+zEU3LOeIcZvZvKGASV85MLTwR47dwqRr1hDLU559sJxHbuudk5qmG52mi7qpiGDM7FhVXd/s/DJgpqpeJyKX+ee/bOnDkS6aFZH+IvKiiCwWkXdF5MK0hAqEtVcNovKGIVT+fghFC6rp9N5Wth/UhTV/GELljUNo6NOJrk9UZRTvjEcruHLC0Iw0diUvTznv2tVcedYgfjh2X44dv5kBQzJb8xaFpum6F2uUuqlQIK4a6MiA8cAU//UU4KRkN0e9A6ARuERV9wdGAeeJyLBWq4ignb0Wl8S91hkC2w8uhZi39q9uaDH5GxsyCnbhG6VUbw63sbrvIVtZs6ITn6wspLEhj1lTuzH6+E9zTtN03Ys1St0gJAIeQIWIzG12TNyNnALPi8i8Zu/3VtVKAP9nr2TxRNrN9ANoCqZaRBYD/YBFrRZLKH1++T75n9RTfXw59UOKd3q7y783sfXIriFEHS499mhg3ZpOn52vryxgv0O35pym6Uan6aJuKhRtzZjZelUdmeKeMaq6RkR6ATNEZElrY8ramJmIDAQOAV7fzXsTgYkA+RUtJKQ8ofKGIUhtnF43fkTByu00DOgMQNnjVRATao/qFk3wGSC72TSS6Q6oKDRNNzpNF3VToQoNIT5HVdf4P6tE5AngcGCtiPRR1UoR6QMkHUfKykZzEekCPAb8TFW37Pq+qk5W1ZGqOjJWVpJUS0tibB9WQtGCagBKZm2ieN4W1l/Qf/d/s23M+soCevbdseWpok8DGz4pyDlN041O00Xd1AjxgEdKJZESESlteg18BVgITAMm+LdNAKYm04k8mYlIAV4ie0BVH09HI29LI1LrbWqV+gSd36mhoV8hnRdUUzZ1HVW/HIgW5mYBkKULiuk3qJ7e/evIL0gwdvxmXns+s+5wFJqm616sUeqmQoGEBjsC0Bt4RUTeAt4AnlbV54DrgONEZBlwnH/eIpF2M0VEgLuAxap6U7o6sU0NVPx5lTeaqMrW0V3ZNqKMvj9dijQqva9ZDkDdkGI2TuyXdryX3fIBB42upqx7I/e9toD7b+7H9Id7pq0HkIgLf/5VP679x4fkxeD5h8r56L3OOadpuu7FGqVuEIK0uoKgqh8CB+/m+gbgS0F1JEpDExH5IjAbeIfPJja4QlWfaekznQf3U6uaYRjR8brOZItuzCgTHXBQJ33o6aSTi59x0IDV8wJMAGRM1LOZr0BI6dswjJxBgQbNraEdN3YAGIaRUyhCPMcKVVsyMwwjLRKaW50uS2aGYbQab6O5JTPDMJxHiNuYmWEYruNVmrVklpTCFXUM+v6K0HUnLFwWuibA3fvuFbqmFHRKfVMa2DISIyxUhXpNv9xWFORcMjMMww0SNmZmGIbreBMA1s00DMN5bALAMIx2gE0AGIbRbojbolnDMFxHERo0t9JHbkUTgIo96rj09+/RvaIeTQjPPtKbqfemX/YHIBGHp77dh+LejRx3xzrm39qV9x7pQudyr9DHoRdvov8x6ZtEROGe45KTlGu6LsUapW4ycnECIBvFGS8UkYW+O9PPMtWLx4W/XTeIH31tBBedfhAn/k8lAwZnVvN80b2ldBu8sxnKsHOqGT+1kvFTKzNKZFG557jiJOWarkuxRqmbCkWIa7AjW0RtNXcg8EO8et4HAyeKyJBMNDet68QHi7oAsK02n48/LKZH77q09Wo/ibFqVhFDTqnJJKwWico9xxUnKdd0XYo1St0gJMgLdGSLqJ+0P/Caqm5V1UbgJeDksMR79dvO4P1rWfpWadoar1/bnZE/34zk7VykcskDpTz5jT68cnkP6j5N/2vanXtORZ/MLPGiIqpYXdJ1KdYodVOhCnHNC3Rki6iftBA4WkR6iEgx8DWg/643icjEJk+9eg3WRO5cHOfKWxZzx7WD2FqbXgvl4xeLKCpPUHHgztt89juzmm/PWM34qZUU9Yoz57ruaelD27nnpINrDkLmztSG7kwIDRoLdGSLqCvNLhaR64EZQA3wFp4x8K73TQYmA3SNVaT8q4jlJ7jylsW8+FQvXp1RkXZ8a98sZOW/i1j1cj/idUJ9jfDSpT045sYNn90z9NRqXpgUrDzw7mg795zW45qDkLkzte3vV4ebAFDVu1T1UFU9GtgIZLjjW/nZb5fx8YfFPHFPZrOYIy/ZzOkvr+bUf6/mmJvW0WfUdo65cQNbq3b8b7LyhWK6D0m/2d5W7jnp4JqDkLkztaU7k5DQYEe2iHxphoj08o09BwDfAkZnonfAiC18+aR1LF9azG1Pzgdgyk17Mefl8hCi9Zh7Qzc2LOmEAF36NXLk1RvT1orKPccVJynXdF2KNUrdIORayyxSdyYAEZkN9AAagItVdWay+7vGKnRUl2+GHseEuQtD1wQrAWS4RxjuTP0PLNOLHx0V6N6Lh81w350JQFWPivoZhmFkm2Bu5dnEuR0AhmG0PZ7VnBVnNAzDcVSFhJUAMgyjPWD1zAzDcB6vnpmNmRmG4TxWaTYlmkiQqK4OXffeMYeGrglw5FuVoWu+enDokkYzolj60tGWvSjmaG4YRjugaW9mLmHJzDCMtMg1D4DcisYwDCfwSgCFV5xRRGIiMl9E/uWfl4vIDBFZ5v9MWbrGkplhGGkR8kbzC4HFzc4vA2aq6hBgpn+eFEtmhmG0Gq9qRl6gIxUisifwdeDOZpfHA1P811OAk1Lp2JiZYRitxtvOFFpb6I/AL4DmJaN7q2olgKpWikjKooJOJrMo3GgKOsX5/d1vUtBJicWUV17oyQO37522nsbh7TPz6dQL9r+tkRU3xdj0Uh5SoHTeE/a5upH8svTjdc3pxyVdc74KQqu2M1WIyNxm55P9gqyIyIlAlarOE5GxmUTkXDczKjeahvo8Lv/BIZx/6uGcf9phjByzkX0PSt8YovKBPIr23lFeqduoBMMfa2D4PxvpvJey6q70p7Vdc/pxTdecr4KRQAIdwHpVHdnsmNxMZgzwTRFZATwEjBOR+4G1ItIHwP9ZlSoe55JZdG40wvZtXkM1P1+J5Se8tnQa1K2FTbPz6H1y4rNr3Y5UxG8Hlx6UoD7lX03LuOb045quOV+lJqzZTFW9XFX3VNWBwBnAv1X1bGAaMMG/bQIwNVVM2fDNfFJE5vm+mRMz1YvSjSYvT7n1kTf4x6xXmP/fcpa+k1754eW/z2evi+ItfrtVT8boPib9opiuOf24phsF7fE7CGsCoAWuA44TkWXAcf55UrIxZnauqm4UkSJgjog8pqobmt/gJ7mJAJ0pTioWpRtNIiH89LTDKSlt4Mqb32GvfWr46P0urdLY+JJQUK50GaZ8Oufzwa76Wx4Sg4qvJ3bz6WC45vTjmm4UtLfvoMkDIFRN1VnALP/1BuBLrfl8NpLZBSLS5JXZHxgC7JTMmrszlUl50r+KbLjR1FYX8M7c7owYs7HVyax6QR6bZuUx75U8EnUQr4X3Lo8x9HdxqqblsfHlPA6Y3LjbX8KguOb045puFLS370CBxhzbaB61o/lY4MvAaFU9GJgPZOS2EJUbTVn3ekpKveZ5p8I4w0dtZNXy5K3E3bHXhXFGzmhgxLMNDL2+ka6HKUN/F2fTf4TVd8fY/0+NxIoyi9U1px/XdKOgPX4HEXczW03ULbOuwCZV3Soi+wHBHBCSEJUbTXlFPZf83yLyYorkwezpvXjj5fQ9OXdl+e/ySdTDu5O8r7z0C8rg/xdPS8s1px/XdM35KgBZtpELQqTuTCJSCDwJ9AOWAj2BX/t9491SJuV6hLSqqxyIWEWP0DUBjpgZRQmgaNyZDI+OXgIoDHem7vv10nF/PyXQvY+P+Yv77kyqWgd8NcpnGIbRNuRay8zJHQCGYbQtVpzRMIx2gSI0JnJrNtOSmWEYaWGGJoZhuI9aN7PNiK/fkPqmNIhi5nH5taND1wQYdMV/I9F1DZdmHnMVGzMzDKPdYMnMMAznUYS4TQAYhtEesAkAwzCcR20CwDCM9oJaMjMMw31yb6N5bo3gBWTk2C3cOXsJd/9nMaedv7ZD6O5RXMO9x0/j2ZMe4unxD/Pd/d/e6f1zD1jAe+f8le6F29o8Vtd1XYo1St1UqEqgI1tko2x2XEQWNDtSmnkmwzVjiLB04ypcN2c0X33yDE57+mTO2u9dBnfdCHiJbkzfVayuaV0hyahidVnXpVij1E2FKsQTEujIFtlomW1T1eHNjpS1vJPhmjFEWLrrtpWwaKNXU6u2sRMffNqd3sW1AFxx+KvcMHdUuv4rocfqsq5LsUapG4RWuDNlBee6ma4ZQ0Sh26/LFoaVr+et9b0Z138Fa7cWs2RT5oUkXfoOotJ1KdYodVOhdMBuJlC0Szfz9F1vEJGJIjJXROY2UJdUzDVjiLB1i/MbuHXs81z7xpHEE8KPD3qTP80/LH3BZrjyHUSp61KsUeoGeDIJDXZki2zMZm5T1eHJbsgFQxMXdPMlzq3HTuepD4fw/Mq9GdptA3t22cK08Y8CsEdxLU984zFOefpbrN/Wev8CF76DqHVdijVK3SDkmhOWc91M14whwtNVrh3zEh982p27Fx0MwHubezD64XMY98+zGffPs/lkawknP/XttBJZuLG6q+tSrFHqBiHXupnOrTNzzRgiLN0RvT7hpH3eY8nGcqZ+02uJ3TTvcF5avVfGMYYdq8u6LsUapW4qvNnM3GoLRWpoAt7SDOCdZpeeU9UWl2dEZWjiElYCyIiSMAxNivbpq4NunBjo3sUn/8Z9QxMAVY1F/QzDMLKPbWcyDMN5lOyOhwXBkplhGGmRY5OZlswMw0gDBc3iVqUgWDIzDCMtrJtpGEa7INcWzbaYzETkVpJ0i1X1gkgiMhj80KZIdJ9dsyAS3RP2OjwSXXNRyl2a9mbmEslaZnOzFoVhGG6hgCvJTFWnND8XkRJVrY0+JMMwXCCsbqaIdAZeBgrxctI/VfUqESkHHgYGAiuA01S1xW5Lyv0IIjJaRBYBi/3zg0Xk9oz/BIZhOIygiWBHAOqAcap6MDAcOEFERgGXATNVdQgw0z9vkSCbq/4IHA9sAFDVt4Cjg0RoGEY7RgMeqWQ8avzTAv9QYDzQ1EOcApyUTCfQTlFV/XiXS/EgnzMMo52iraqaUdFUr9A/PrepU0RiIrIAqAJmqOrrQG9VrQTwf/ZKFlKQpRkfi8iRgIpIJ+AC/C6nYRgdmOBjZutTbTRX1TgwXES6AU+IyIGtDSdIMpsE/AnoB6wGpgPntfZBYTJy7BYmXbOGWJ7y7IPlPHJb7w6n22/PLVx++Y4qGH32qOG++w7kySf3bbXWdw8fRlGXOHl5EMtXbnvuPf52dV9em1FGQSelz151XHLzx3Tpml6D/KIblnPEuM1s3lDApK+0+nc0KVF8ty79HkSpm5rwZzNVdbOIzAJOANaKSB9VrRSRPnitthZJ2c1U1fWqepaq9lbVnqp6tqpuCCf01uOay01UuqtXlXH+ecdz/nnHc8FPj2N7XT6vvrpn2nq/f/R9/vLCUm577j0ADj26mskvLuGvM5fSb+86Hro1aQs/KTMereDKCUPT/nxLmDtT27kzAZAIeKRARHr6LTJEpAj4MrAEmAZM8G+bAExNphNkNnNvEXlKRNaJSJWITBWRvVOHGA2uudxkwz1n+PAqKitLqKoqCU1zxNhqYn67ff8RW1lfmX4p5oVvlFK9OfzNJubO1IbuTE3rzIIcqekDvCgibwNz8MbM/gVcBxwnIsuA4/zzFgkyAfAP4BH/gX2BR4EHg0QIICJni8gbvpnJHSKSUX0z11xusuGec8wxK3lpVgYVZ0W54szBnHf8UJ65v8fn3p7+YDmHjavOIMJoMHemtnNnAm+dWZAjtY6+raqHqOpBqnqgql7tX9+gql9S1SH+z43JdIIkM1HV+1S10T/uJ+DQn4jsD5wOjPFNTeLAWbu5z9yZ0iQ/P84Ro1Yze3b/tDVunrqMPz//Hr994EOm3VPBO6/taOH940+9ieUr474VzRarTDB3prZ0ZyK0pRlhkWxvZrn/8kXfhfwhvNBOB54OqP8lYAQwR7xvvYjdDOKZO1P6jBz5CR+8353Nm9Ov+95jj0YAulU0MuaET1kyv5gvjKplxiPdeeOFMq57+P3d/qNpa8ydqW3dmXJtO1Oyltk8vP2ZpwM/Al4EZgE/Br4XUF+AKc3czPdV1V+nH657LjdRu+eMHfsRs2YNSPvz27fmsbUm77PX814qZeB+25nzYimP/Lk3v77nQzoX51h5BB9zZ2pbdybRYEe2SLY3c1AI+jOBqSJys6pW+a29UlX9KF1B11xuonTPKSxs5JBD13LLLel7RWxal89vvu/9Vccb4diTN3PYsdWcc+T+NNQJl5++DwD7jajlwutXpfWMy275gINGV1PWvZH7XlvA/Tf3Y/rDPdOOuQlzZ2o7dyZUIMeKMwZyZ/IXsA0DPvuWVPXeQA/wHMwvx2sFNgDnqeprLd1v7kyQd9B+keg++9xDkehaCSC3CMOdqXCv/trn8gsD3fvRj3+eG+5MInIVMBYvmT0DfBV4BQiUzFT1Ybyd74ZhtCdybPQhyGzmKXgD+Z+o6veAg/FKdRiG0ZFxZTazGdtUNSEijSJShjcb2WaLZg3DyAFcKs7YjLn+VoO/4c1w1gBvRBmUYRi5TzZnKoOQMpmp6k/8l38VkeeAMlV9O9qwDMPIeVxJZiJyaLL3VPXNaEIyDMMFXGqZ/SHJewqMCzkWwyfx9pJIdI/vOzwS3cYX9ohEN//LKyPRjQIp6JT6pjTI6eUproyZqeqx2QzEMAyHyPJMZRDMBNgwjPSwZGYYRntAAhRezCaWzAzDSI8ca5kFqTQrfoHF//XPB4hINJvxDMNwgqAVM7I54xlkO9PtwGjgTP+8GvhzZBEZhuEG4ZXNDoUgyewIVT0P2A7g26NHMw8dkJFjt3Dn7CXc/Z/FnHb+2g6pm/Ox1iux8z4hNrGS2PcryZuy+bO35IlqYues8a5PzqyCrUvf7UU3LOehefP56/MLQ9OE6OJNSY7tzQySzBr8uv0KnpMKgTxXPo+IvJrO55rjmstNh3UQKoD4jb2IT+5D/I49kDnbYVEdsmA78upW7/pdfUicWpYb8Uao2UQULlVt6c7kYjfzFuAJoJeI/Bav/M+16TxMVY9M53PNcc3lpsM6CIlAkf/r1ajeISDTatAzukInv/vRPX1/G5e+W4jGpaot3ZkkEezIFkF8Mx8AfgH8DqgETlLVR9N5mIjUpPO55rjmctOhHYTiSuxHlcROWY2O6Az7FyKrG5CF24md/wmxi9fCkuQGNlmNNyLNKGnTeHOsmxmkOOMAYCvwVPNrqhraXhMRmQhMBOhMcYp7P38tl11uOrSDUEyI39EHahLkXbUOltd7/lzVCeK39oal9cT+bz3x+/ru/uHZjjcizShpc3emHCJIm/dpvLAFr2z2IGApcEBYQZg7U9trRqlLlzz04M7InO1oRQz9YrH3r3C/Qu/npwno1vrupkvfbVS0Zby5ttE8SDfzC7455xdUdQhwON64WZvgmstNh3UQ2hyHGn/ApC5B3pvbYUABOqYIWeAPUK9q8MbSugYZuo043gg1o8S1eKOk1aORqvqmiBwWRTBBcM3lpsM6CG2ME7t+gzfvrZA4phgdVQQNity4gdgPKiEfEr/okVYXM/R4I9RsIgqXqjZzZ4Kc62amdGcSkYubneYBhwI9VPX4Vj9MpEZVuyS7x9yZ3KPxhfR9O5NhJYCiKQEUhjtT5779deDEi1PfCCz9zcW54c4ElDZ73Yg3hvZYOg9LlcgMw3CIHGuZJU1m/mLZLqr68yzFYxiGAwi5NwGQrGx2vqo2JiufbRhGB8aVZIbnwHQosEBEpgGPArVNb6rq4xHHZhhGrpLlrUpBCDJmVg5swKv537TeTAFLZobRkXGoOGMvfyZzITuSWBM5lpMNw8g2LrXMYkAXdk5iTeTYH8NoS6JaQlHz3N6R6HY54cPQNWO9KkLXBGhcvSYS3VAIKQuISH/gXmAPvPbeZFX9k4iUAw8DA4EVwGl+CbLdkiyZVarq1eGEaxhGuyLcTeSNwCX+gvxSYJ6IzADOAWaq6nUichlwGfDLlkSS7SPJLVM8wzByirDqmalqZZOpuKpWA4uBfsB4YIp/2xTgpGQ6yVpmtgzfMIyWCd4yqxCRuc3OJ/vFJT6HiAwEDgFeB3qraiV4CU9EeiV7SDIT4I2BQzUMo8PRisKL64NsZxKRLni7i36mqluklXt20ytXYBhGxyZoYcaArTcRKcBLZA80W8O6VkT6+O/3AaqSaVgyMwyj1UgrjpRaXhPsLmCxqt7U7K1pwAT/9QRgajIdJ02AR47dwqRr1hDLU559sJxHbuvd4XRdijVU3foERZdWQoNCXIkfVUL9d8rpdN9G8p+rRrt6RR7rzyknfnjyqsWRx7ob8vKUP977HzZUdeY3F4dTSCLKeJMS3mzmGOA7wDsissC/dgVwHfCIiHwfWAmcmkwk0mQmInvieWwOw2sF/gv4uaqmXdekyY3m8jP2Zn1lAbc+s4zXpndl5bLMaji5pOtSrKHrFgjbru/jmaU0KkWXrCFvpFfsseHkrjSc0i13Yt0N3zxjBR8v70JxSWMoelHHm4ywFs2q6iu03IgLPBEZWTfTbzo+DjzpV6gdircI97eZ6DrhTBSxrkuxhq7bgutTWETpdtSj1zYO++I6pk/tH4oetKE7E+ScoUmUY2bjgO2qejeAqsaBi4BzRSS99j8OORNFqOtSrJHoxpWin6yi5IyPiB9aRGI/rxVSMG0LRZNWUXhTFVTHcyPWZky8eDF337IvGuKexjZzZ3LRai4DDgDmNb+gqlvw+r77NL8uIhNFZK6IzG0gufWYM85EEeq6FGskujFh2+17Unv/APKW1pG3op6GE8vYend/tt3eDy3Pp/BvG3IjVp/DvljFp5sKeX9JuPX529ydKYdaZlGOmTVV10h53dyZ2l7TRV26xIgfVERs7tadxsoaTiil81WfpCUZVazDDt7EEUetZeSR6+hUGKeopJFLr36LG//34Ix0zZ1pB1G2zN4FdpquEZEyoD/wQbqiTjgTRazrUqyh626OQ43fhaxLkD9/G4n+BciGHQPq+a/WkhiYXk3+qL6DKX/elwknjuPc8WO5/orhvD2nR8aJLMp4A9GBWmYzgetE5Luqeq9fgvsPwD2qujVdUSeciSLWdSnWsHXzNjZS+Id1npmwKo1HdyF+RAmFv68i78M6QNDe+dRdkF4VizZ1O0qDtow311pmKd2ZMhL3SnvcDuyH1wp8BrhUVVscGDN3JqMJl0oA5ffrG7omRFMCKAx3puKe/XW/bwVzZ5o/OXfcmdJGVT8GvhHlMwzDyD5OGZoYhmEkxZKZYRjtAcnaGpBgWDIzDKP1ZHmmMgiWzAzDSAsbMzMMo12Qza1KQegwyUwK0ltEmQptSLsAiJGCKJZQAKx5Yljomn1PXhS6Zs5jLTPDMJzHUUdzwzCMz2PJzDAM17FFs4ZhtBskkVvZzJKZYRitx9aZhUNUBg4X3bCcI8ZtZvOGAiZ95cBQNMEMTXJetz5Bxa9WII0Kcdg+upTqM3tR+o8qOr9RDQLxrvlsvqAvifL0a4Xl9HeQBrm2NMM5q7kmA4crzxrED8fuy7HjNzNgyPZQtGc8WsGVE4aGotVEFPFG9R10WN0CYcPVA1l382DW3bQ3hfNrKFi6lZqTerDuj4NZd/Ng6kZ2ofTh9W0fa5Z0A5Fj9cycS2ZRGjgsfKOU6s3hNlbN0MQBXRHUN0mRuNc6Q0CLYztuqcvMOCXnv4M0EA12ZIvIk5mIXCwiC/3jZ5nqtZmBQ5qYoYkjunGl50Uf0PucpdQdXELDUM9zp/T+Knr/4D2KXvqU6jN75kasWdBNieKZDQQ5skSkyUxERgDfA44ARgE/FJFDdnNfmxuaRIUZmjiiGxPW3TyYtXcOpdOybeR/5HXVqs/uxdo7h7LtmK6UPLMxN2LNgm6gZ3cgdyaALwJPqGqtqtbg+WgetetNqjpZVUeq6sgCCpMKtqWBQzqYoYlbuloSo+7AEgrn1+x0fdtRZXT+b3Xaui59B0FoWmfWkbqZIdqzerSpgUMamKFJ7uvmfdqI1O4wSSl8q4bGfoXE1uzoJXSeU0Pjnunv783176DVBO1iZrHbFPXSjJeBe0TkOrzEdjLwnUwEozRwuOyWDzhodDVl3Ru577UF3H9zP6Y/nP44SVTxumA84pJu3qZGut+yBhIKCdg2poy6w0rpfv3H5K+uhzyI9yxg86Q+bR5rtnSDkGs7ACI1NAFvAgA41z+9U1X/mOz+qAxNrGqG0URHr5oRhqFJabc99ZCjLwx07+ynfuG+oQmAqt4E3BT1cwzDyC651jJzcgeAYRhtjALx3MpmlswMw0gLa5kZhtE+yLEFnpbMDMNIi1xrmTm3N9MwjBwg6CbzAAlPRP4uIlUisrDZtXIRmSEiy/yf3VPpdJiWmS2hMJqIYhnFrz5cELomwG/3Hh6JbqYI/qb8cLgHuA24t9m1y4CZqnqdiFzmn/8ymYi1zAzDSAtRDXSkQlVfBnbd+DoemOK/ngKclEqnw7TMDMMIkdbVKqsQkbnNzier6uQUn+mtqpUAqlopIr1SPcSSmWEYadCqfZfrs7EDwLqZhmGkRcRVM9aKSB8A/2dVqg9YMjMMIz2irZoxDZjgv54ATE31AetmGobRejS82UwReRAYize2tgq4CrgOeEREvg+sBE5NpeNkMnPN5cbcmdzSDVszEYe/jx9Kae8GTr9rOY//dC82fOiV6anbEqOwLM4Pn16aM/EGJqSVGap6Zgtvtap8jnPJrMmN5vIz9mZ9ZQG3PrOM16Z3ZeWyzGo4uaTrUqyu6UahOefunlQMrqOuxhvV+datH3323gu/7UthaTyn4g1KkGUX2cS5MTPXXG7Mnckt3bA1t1QW8P6LZQw/fcPn3lOFRc9044BvbMqZeFtFjlWajdrQZOAuWxQuFZFfZ6LpmsuNuTO5pRu25oxr+jHusjXIbv6lfTynhJIejZQPSn93Spu6MyUCHlkiJ1pmueDO5JKuS7G6phum5rKZZRT3aKTPF7bt9v13p3XngG+m3yqDtnNnEoKt/s9mVzQnxsz81cCTwSubnexe11xuzJ3JLd0wNVfNK2HZzDI+mDWMxjqhribG1IsGMP7mlSQaYen0rpw77b2cibfVJLLY7ApA1C2zxl2ekfGopGsuN+bO5JZumJrH/qKSC15dxPmzF3HyLR8xcHQ1429eCcDy/5TSY3AdZRl2CdvOnYmc62ZG3TJbC/QSkR5ADXAi8Fwmgq653Jg7k1u62XI7WvSv7gzLYOC/ibZ1Z8qt2cxsuDNdAFwALAdWAytU9dct3R+VO5NhRIlLJYDCcGfqWtxXRw/5fqB7p7/9f+3GnekW4Jaon2MYRjbJ7rKLIOTEBIBhGI5h7kyGYbQXcm3MzJKZYRjpYcnMMAznUSBhycwwDOexCQDDaJdE5aLU97XS0DU7TQhprbwlM8MwnEeBeG5tZ7JkZhhGGiioJTPDMNoD1s00DMN5bDbTMIx2g7XMDMNoF+RYMsuJSrOtZeTYLdw5ewl3/2cxp52/tkPquhSra7q5HqvWKevOraXq7Fqqzqxly9+86swNy+Ks+0EtVWfVsuGSrSRqI0w2qhCPBzuyhHPJrMmN5sqzBvHDsfty7PjNDBiyvUPpuhSra7pOxNoJetxWTK/7S+h5XzF1/22kfmGczddup+wnhfR6oISisQXU3J++t0AgOpKhSRS45PQTla5Lsbqm60KsIkJesVeOTBvx6jkDjR8l6HRIDIDCw2Nsf7Ex47iT0pGSmYhcIyIXNjv/rV+sMW1ccvqJStelWF3TdSVWjStV36ll7VdrKDw8n04HxsgfnMf22V4C2zazkXhVlOvA1JvNDHJkiahbZncBEwBEJA84A3hg15vMnantNU03Os0odCUm9LqvhN7TulC/KE7DB3G6/aozW//ZwLoJtehWjXZ6T0E1EejIFpHOZqrqChHZICKHAL2B+ar6OTdUc2dqe03TjU4zSt28UqHw0Bh1r8XpclYnetxSDEDjygTbX424m5lj25myMWZ2J3AO8D3g75mKueT0E5WuS7G6putCrPFNCRLV3v/5ul2pmxMnf6884hu95KIJpfruOkpO7pRMJjNUPau5IEeWyMY6syeAq4EC4H8yFXPJ6ScqXZdidU3XhVgT65VN12yDOKBQ9KV8On8xn5qH66n9p9f6KxpbQNGJEf/zzrF1ZpG7MwGIyF+Bzap6Wap7zZ3JMHYQRQmgpydMY/3i9Zm5M8UqdFTR1wPd+3ztve3Dnckf+B8FnBr1swzDyBa5V5wx6qUZw4D3gZmquizKZxmGkUWaNprn0NKMqGczFwF7R/kMwzCyjwKaxa1KQXBuB4BhGDmA+sUZgxwBEJETRGSpiLwvIinH1neHVc0wDCMtNKQupIjEgD8DxwGrgDkiMs3v2QXGWmaGYaRHeC2zw4H3VfVDVa0HHgLGtzacrCzNaA0isg74KODtFcD6CMKIQtelWF3TdSnWXNDdS1V7ZvIgEXnOf14QOgPNS4RM9nf9NGmdApygqj/wz78DHKGq57cmppzrZrbmSxaRuVGsX4lC16VYXdN1KVYXdXeHqp4Qotzu1ry1upVl3UzDMNqaVUD/Zud7AmtaK2LJzDCMtmYOMEREBolIJ7zqOtNaK5Jz3cxWMjn1LTmj61Ksrum6FKuLupGiqo0icj4wHYgBf1fVd1urk3MTAIZhGOlg3UzDMNoFlswMw2gXOJfMRKS/iLwoIotF5N3mHgMhaF8oIgt93Z+FpRsFIhIXkQXNjrS2gGQTEXk1ZL09RWSqiCwTkQ9E5E/+ALLRAXFuzExE+gB9VPVNESkF5gEntXbrw250D8RbeXw4UA88B/w4V6t9iEiNqnZp6zjaChER4HXgL6p6t78lZjKwUVV/3rbRGW2Bcy0zVa1U1Tf919XAYqBfCNL7A6+p6lZVbQReAk4OQRcReVJE5vktvolhaEaFiJwtIm/4rb07/CQRlnZNWFrAOGC7qt4NoKpx4CLgXBEpzlRcRC72W+kLw2ili8hAEVnY7PxSEfl1CLqhO6C5inPJrDkiMhA4BO9/6ExZCBwtIj38fwxfY+eFfJlwrqqOAEYCF4hIjxA0i3bpZp6eqaCI7A+cDoxR1eF4hZnPylQ3Ig7Aa5V/hqpuAVYC+2QiLCIj8DwrjsArLPpD35QnFwnkgNYRcHadmYh0AR4Dfub/EmeEqi4WkeuBGUAN8Baf2atmzAUi0tTK6w8MAT7nUtVKtvkJJ0y+BIzAq1oAUARUhfyMsBB2v+Wlpeut4YvAE6paCyAijwNHAfMz1A2doA5oHQEnk5mIFOAlsgdU9fGwdFX1Lrz/6RCRa/G2WWSEiIwFvgyMVtWtIjILb+NtLiLAFFW9vK0DCcC7wLebXxCRMrz/LD7IUDuj+vgt0MjOPaEwfweaHND2IAQHNFdxrpvpD/zeBSxW1ZtC1u7l/xwAfAt4MATZrsAmP5Hth9dtyVVmAqc0+x7KRWSvNo6pJWYCxSLyXfisJtYfgHtUdWuG2i8DJ4lIsYiU4I2dzs5Qcy3Qyx/GKAROzFCvOU8AJwCH4a2i75C42DIbA3wHeEdEFvjXrlDVZ0LQfswfz2oAzlPVTSFoPgdMEpG3gaXAayFogj9m1vw5QdyvkqGqi0TkSuB5f/ylATiP4CWZsoaqqt91v11E/h/ef8zPAFeEoP2miNwDvOFfulNVM+piqmqDiFyNN767HFiSWZQ7adeLyIt4Dmi5Vcs6izi3NMMwjJ3x/+N5Ezg1V5cSZQPnupmGYezAHNB2YC0zwzDaBdYyMwyjXWDJzDCMdoElM8Mw2gWWzByjWbWMhSLyaCb7EEXkHt8ZBxG50x9MbunesSJyZBrPWCEin3Pxaen6Lve0ai+niPxaRC5tbYxG+8CSmXtsU9XhqnogXnWPSc3fTHdjuKr+IEXlkbFAq5OZYWQLS2ZuMxvYx281vSgi/8BbTBwTkRtEZI6IvC0iPwJv94SI3CYii0TkaaBXk5CIzBKRkf7rE0TkTRF5S0Rm+hv6JwEX+a3Co0Skp4g85j9jjoiM8T/bQ0SeF5H5InIHAbYGJasqIiJ/8GOZKSI9/WuDReQ5/zOz/Z0VRgfHxR0ABiAi+cBX8XYYgFeH7UBVXe4nhE9V9TB/68x/ROR5vAoj+wJfwNuUvIhd9vL5CeNvwNG+VrmqbhSRvwI1qnqjf98/gJtV9RV/+9d0vDJKVwGvqOrVIvJ1IEjJo3P9ZxThbXJ/zN8sXQK8qaqXiMj/+trn49Utm6Sqy0TkCOB2vJJARgfGkpl7NN/GNBtvn+qRwBuquty//hXgoKbxMLz9oUOAo4EH/S0va0Tk37vRHwW83KSlqhtbiOPLwDC/ugZAmXjFMo/G29eKqj4tIkG2hLVUVSQBPOxfvx943K+WciTwaLNnFwZ4htHOsWTmHp8r/eP/o65tfgn4qapO3+W+r5G6PE7QEjp5eJVAtu0mlsArsVtZVUT9526OoPyR4Tg2ZtY+mQ782C+VhIgM9as/vAyc4Y+p9QGO3c1n/wscIyKD/M+W+9ergdJm9z2P1+XDv2+4//Jl/IKOIvJVoHuKWJNVFckDmlqX/4PXfd0CLBeRU/1niIgcnOIZRgfAkln75E688bA3xSvVfAdeK/wJYBnwDvAXvNLgO6Gq6/DGuR4XkbfY0c17Cji5aQIAuAAY6U8wLGLHrOpv8Cr2vonX3V2ZItbngHzxqopcw85VRWqBA0RkHt6Y2NX+9bOA7/vxvQuMD/CdGO0c25tpGEa7wFpmhmG0CyyZGYbRLrBkZhhGu8CSmWEY7QJLZoZhtAssmRmG0S6wZGYYRrvg/wOjOL2xoLRPPgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Generate list of predictions for the test set, and displays the results as a confusion matrix\n",
    "\n",
    "model.eval()\n",
    "y_pred = []\n",
    "y_true = []\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for (images, features), labels in test_dataloader:\n",
    "        images = images.cuda()\n",
    "        features = features.cuda()\n",
    "        labels = labels.cuda()\n",
    "        y_predicted = model(images, features)\n",
    "        _, label_predicted = torch.max(y_predicted.data, 1)\n",
    "        y_pred.append(label_predicted.item())\n",
    "        y_true.append(labels.item())\n",
    "\n",
    "        \n",
    "from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\n",
    "target_names = ['2', '9', 'a', 'E', 'e', 'i', 'O', 'o', 'u', 'y']\n",
    "print(classification_report(y_true, y_pred, zero_division=0, digits=4, target_names=target_names))\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=target_names)\n",
    "disp.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3bf2d1fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([359., 402., 402., 402., 402., 257., 402., 402., 539., 402.]),\n",
       " array([0. , 0.9, 1.8, 2.7, 3.6, 4.5, 5.4, 6.3, 7.2, 8.1, 9. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPN0lEQVR4nO3df6zdd13H8eeLDsf4oW7stjZttUtsIh0JW7hWzNQgI64CsfPHQolglcVGU8MQf7VE46/UjD8k+oczNgxoIrI0gbkGyKSpTDQC5XYMWFuaFja2ps16GRCY6KD17R/nC5629/ae9p4f7afPR3Lz/X4/5/M931ebe1/3e77nx01VIUlqy3MmHUCSNHyWuyQ1yHKXpAZZ7pLUIMtdkhp01aQDAFx//fW1evXqSceQpMvK/v37v1JVU3PddkmU++rVq5mZmZl0DEm6rCT58ny3eVlGkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIadEm8Q1WSAFZv/fBEjvv43a+dyHFHyTN3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQQOVe5LHk3w+ySNJZrqx65LsSXKkW17bN39bkqNJDie5bVThJUlzu5Az95+tqpuqarrb3grsrao1wN5umyRrgY3AjcB64J4kS4aYWZK0gMVcltkA7OzWdwK3943fV1XPVtVjwFFg3SKOI0m6QIOWewEfTbI/yeZubFlVnQDolku78RXAk337HuvGzpBkc5KZJDOzs7MXl16SNKdBP8/9lqo6nmQpsCfJF84zN3OM1TkDVTuAHQDT09Pn3C5JungDnblX1fFueRK4n95llqeSLAfolie76ceAVX27rwSODyuwJGlhC5Z7khckedF314GfAx4FdgObummbgAe69d3AxiRXJ7kBWAPsG3ZwSdL8Brksswy4P8l35/9TVT2Y5NPAriR3Ak8AdwBU1YEku4CDwClgS1WdHkl6SdKcFiz3qvoS8LI5xp8Gbp1nn+3A9kWnkyRdFN+hKkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUoIHLPcmSJJ9J8qFu+7oke5Ic6ZbX9s3dluRoksNJbhtFcEnS/C7kzP0u4FDf9lZgb1WtAfZ22yRZC2wEbgTWA/ckWTKcuJKkQVw1yKQkK4HXAtuBt3XDG4BXdus7gYeAP+rG76uqZ4HHkhwF1gGfGFrqS8TqrR+edASNweN3v3Yix/X7a3wm+X89qu+vQc/c/wb4Q+B/+8aWVdUJgG65tBtfATzZN+9YN3aGJJuTzCSZmZ2dvdDckqTzWLDck7wOOFlV+we8z8wxVucMVO2oqumqmp6amhrwriVJgxjksswtwC8keQ3wPOD7k/wj8FSS5VV1Isly4GQ3/xiwqm//lcDxYYaWJJ3fgmfuVbWtqlZW1Wp6T5T+a1W9EdgNbOqmbQIe6NZ3AxuTXJ3kBmANsG/oySVJ8xroCdV53A3sSnIn8ARwB0BVHUiyCzgInAK2VNXpRSeVJA3sgsq9qh6i96oYqupp4NZ55m2n98oaSdIE+A5VSWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ1azGfLXDL8owaSdCbP3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNWjBck/yvCT7knw2yYEkf96NX5dkT5Ij3fLavn22JTma5HCS20b5D5AknWuQM/dngVdV1cuAm4D1SV4BbAX2VtUaYG+3TZK1wEbgRmA9cE+SJSPILkmax4LlXj3PdJvP7b4K2ADs7MZ3Ard36xuA+6rq2ap6DDgKrBtmaEnS+Q10zT3JkiSPACeBPVX1KWBZVZ0A6JZLu+krgCf7dj/WjUmSxmSgcq+q01V1E7ASWJfkpeeZnrnu4pxJyeYkM0lmZmdnBworSRrMBb1apqq+DjxE71r6U0mWA3TLk920Y8Cqvt1WAsfnuK8dVTVdVdNTU1MXnlySNK9BXi0zleQHu/VrgFcDXwB2A5u6aZuAB7r13cDGJFcnuQFYA+wbcm5J0nlcNcCc5cDO7hUvzwF2VdWHknwC2JXkTuAJ4A6AqjqQZBdwEDgFbKmq06OJL0may4LlXlWfA26eY/xp4NZ59tkObF90OknSRfEdqpLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMG+eAw6Yq2euuHJx1BumCeuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJatCC5Z5kVZKPJTmU5ECSu7rx65LsSXKkW17bt8+2JEeTHE5y2yj/AZKkcw1y5n4K+L2qegnwCmBLkrXAVmBvVa0B9nbbdLdtBG4E1gP3JFkyivCSpLktWO5VdaKqHu7WvwkcAlYAG4Cd3bSdwO3d+gbgvqp6tqoeA44C64acW5J0Hhd0zT3JauBm4FPAsqo6Ab1fAMDSbtoK4Mm+3Y51Y2ff1+YkM0lmZmdnLyK6JGk+A5d7khcCHwDeWlXfON/UOcbqnIGqHVU1XVXTU1NTg8aQJA1goHJP8lx6xf6+qvpgN/xUkuXd7cuBk934MWBV3+4rgePDiStJGsQgr5YJcC9wqKre2XfTbmBTt74JeKBvfGOSq5PcAKwB9g0vsiRpIVcNMOcW4E3A55M80o29Hbgb2JXkTuAJ4A6AqjqQZBdwkN4rbbZU1elhB5ckzW/Bcq+q/2Du6+gAt86zz3Zg+yJySZIWwXeoSlKDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1aMFyT/LuJCeTPNo3dl2SPUmOdMtr+27bluRoksNJbhtVcEnS/AY5c38vsP6ssa3A3qpaA+zttkmyFtgI3Njtc0+SJUNLK0kayILlXlUfB7561vAGYGe3vhO4vW/8vqp6tqoeA44C64YTVZI0qIu95r6sqk4AdMul3fgK4Mm+ece6sXMk2ZxkJsnM7OzsRcaQJM1l2E+oZo6xmmtiVe2oqumqmp6amhpyDEm6sl1suT+VZDlAtzzZjR8DVvXNWwkcv/h4kqSLcbHlvhvY1K1vAh7oG9+Y5OokNwBrgH2LiyhJulBXLTQhyfuBVwLXJzkG/ClwN7AryZ3AE8AdAFV1IMku4CBwCthSVadHlF2SNI8Fy72q3jDPTbfOM387sH0xoSRJi+M7VCWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDRlbuSdYnOZzkaJKtozqOJOlcIyn3JEuAvwN+HlgLvCHJ2lEcS5J0rlGdua8DjlbVl6rq28B9wIYRHUuSdJarRnS/K4An+7aPAT/RPyHJZmBzt/lMksOLON71wFcWsf8wXAoZwBxnM8ellQHMcYa8Y1E5fmS+G0ZV7pljrM7YqNoB7BjKwZKZqpoexn1dzhnMYY5LPYM5xpdjVJdljgGr+rZXAsdHdCxJ0llGVe6fBtYkuSHJ9wEbgd0jOpYk6SwjuSxTVaeS/A7wL8AS4N1VdWAUx+oM5fLOIl0KGcAcZzPH/7sUMoA5zjaSHKmqhWdJki4rvkNVkhpkuUtSgy7bck+yKsnHkhxKciDJXRPO85+TPL4uTUlOJ3mk72vsH8WRZGWSB5IcSfLFJH/bvdBh7JLcleTR7mf2rZPIcKW4bK+5J1kOLK+qh5O8CNgP3F5VByccTfqeJM9U1QsnePwAnwL+vqre0300yA7gq1X1B2PO8lJ671ZfB3wbeBD47ao6Ms4cV4rL9sy9qk5U1cPd+jeBQ/TeGTsRSZ6Z1LG7469O8mjf9u8n+bMJ5Hhbd2b26KTOzJL8Zf8juSTbk7xlElkuAa8C/qeq3gNQVaeB3wXenOT5Y87yEuCTVfWtqjoF/Bvwi2POAECSNybZ1z2a+oful964M/xzkv3do5jNC+9xYS7bcu+XZDVwM70zFE1IkpcDv0HvoyZeAfxmkpsnEOVeYFOX6Tn03mfxvgnkALjmrMsyrx/z8W+k96j2e6rqG8ATwI+OOcujwM8keXH3i+U1nPlmx7FI8hLg9cAtVXUTcBr41XHnAN5cVS8HpoG3JHnxMO98VB8/MDZJXgh8AHhr902ryfkp4P6q+i+AJB8Efhr4zDhDVNXjSZ7ufrEsAz5TVU+PM0Of/+4KZFLCWR/9scD4yFTVoSTvAPYAzwCfBU6NM0PnVuDlwKd7V624Bjg5gRxvSfLdRy6rgDXA0L5PL+tyT/JcesX+vqr64KTzTNgpznwk9rwJZJjrM4Um5V3ArwM/BLx7slEm6gDwy/0DSb6fXpl8cdxhqupeeo+sSPJX9D6qZNwC7KyqbRM4di9A8krg1cBPVtW3kjzEkH9mL9vLMt0TRfcCh6rqnZPOcwl4CljaPeS9GnjdBDJ8HLg9yfOTvIDe9dR/n0AOgPuB9cCP03un9JVqL/D8JL8G3/tbC38NvLeqvjXuMEmWdssfBn4JeP+4M9D7P/mVvizXJZn30xVH5AeAr3XF/mP0LmMO1eV85n4L8Cbg80ke6cbeXlUfmVykyamq7yT5C3rPOzwGfGECGR5O8l5gXzf0rqoa6yWZvizfTvIx4Ovdk4iTck3f9yfAg1U1tpdDVlV1D/3vSfIn9E7oPgK8fVwZzvKB7tryd4AtVfW1cQeoqoNJ/hj4aPeczHeALcCXxxjjQeC3knwOOAx8ctgHuGxfCimdT/dD+zBwhy+105Xosr0sI82n+5OOR4G9FruuVJ65S1KDPHOXpAZZ7pLUIMtdkhpkuUtSgyx3SWrQ/wGxNM3UNzatFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# See how well the dataset is balanced\n",
    "plt.hist([x.split('__')[0] for x in files if '~' not in x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9143d464",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Lambda Notebook (Python 3)",
   "language": "python",
   "name": "lambda-notebook"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}