test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from data.LSA_Dataset import LSA_Dataset\n",
    "from data.transforms import (get_frames_reduction_transform,\n",
    "    get_roi_selector_transform,\n",
    "    get_keypoint_format_transform,\n",
    "    get_text_to_tensor_transform,\n",
    "    keypoints_norm_to_nose,\n",
    "    interpolate_keypoints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = '/mnt/data/datasets/LSA-T/data/cuts'\n",
    "max_frames = 75\n",
    "keypoints_to_use = [i for i in range(94, 136)]\n",
    "\n",
    "dataset = LSA_Dataset(\n",
    "    root,\n",
    "    mode = 'train',\n",
    "    load_videos = False,\n",
    "    words_min_freq = 5,\n",
    "    signer_confidence_threshold = 0.5,\n",
    "    frame_transform = get_roi_selector_transform(128,128),\n",
    "    video_transform = get_frames_reduction_transform(max_frames),\n",
    "    keypoints_transform = get_frames_reduction_transform(max_frames),\n",
    "    keypoints_transform_each = get_keypoint_format_transform(keypoints_to_use)\n",
    "    )\n",
    "dataset.label_transform = get_text_to_tensor_transform(dataset.vocab.__getitem__(\"<bos>\"), dataset.vocab.__getitem__(\"<eos>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "i = random.randint(0, dataset.__len__())\n",
    "sample = dataset.__getitem__(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'tokens'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m/home/pdalbianco/cn_sordos_model/test.ipynb Cell 4'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2B192.168.0.62/home/pdalbianco/cn_sordos_model/test.ipynb#ch0000003vscode-remote?line=0'>1</a>\u001b[0m \u001b[39mprint\u001b[39m(sample[\u001b[39m3\u001b[39;49m]\u001b[39m.\u001b[39;49mtokens)\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2B192.168.0.62/home/pdalbianco/cn_sordos_model/test.ipynb#ch0000003vscode-remote?line=1'>2</a>\u001b[0m clip \u001b[39m=\u001b[39m sample[\u001b[39m1\u001b[39m]\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2B192.168.0.62/home/pdalbianco/cn_sordos_model/test.ipynb#ch0000003vscode-remote?line=2'>3</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m[F, C, H, W]\u001b[39m\u001b[39m\"\u001b[39m, clip\u001b[39m.\u001b[39msize())\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'tokens'"
     ]
    }
   ],
   "source": [
    "print(sample[3])\n",
    "clip = sample[1]\n",
    "print(\"[F, C, H, W]\", clip.size())\n",
    "plt.figure(figsize=(12, 12))\n",
    "for i in range(20):\n",
    "    plt.subplot(5, 5, i + 1)\n",
    "    iclip = clip[i, ...].permute(1,2,0)\n",
    "    plt.imshow(iclip)\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = list(sample[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([   2,   90,   37,  185, 2202,   11,  621,   10,   39,  135,    4,    3])\n",
      "75\n",
      "torch.Size([3, 42])\n",
      "torch.Size([75, 3, 42])\n"
     ]
    }
   ],
   "source": [
    "from torch import stack\n",
    "\n",
    "print(sample[2])\n",
    "\n",
    "print(len(keys))\n",
    "print(keys[0].size())\n",
    "\n",
    "vid_keys = stack(keys)\n",
    "print(vid_keys.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "75\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from data.collate_fn import get_keypoint_model_collate_fn\n",
    "\n",
    "dl = DataLoader(dataset, 12, collate_fn=get_keypoint_model_collate_fn(dataset.vocab.__getitem__(\"<pad>\")))\n",
    "for b in dl:\n",
    "    print(type(b[0][0]))\n",
    "    print(len(b[0][0]))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "from torch import Tensor, nn\n",
    "\n",
    "from model.modules.KeypointsEmbedding import KeypointsEmbedding\n",
    "from model.modules.PositionalEncoding import PositionalEncoding\n",
    "from model.modules.TokenEmbedding import TokenEmbedding\n",
    "\n",
    "\n",
    "class KeypointModel(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                src_max_len: int,\n",
    "                tgt_max_len: int,\n",
    "                keys_len: int,\n",
    "                tgt_vocab_size: int,\n",
    "                kernel_size: int = 5,\n",
    "                emb_size: int = 64,\n",
    "                keys_initial_emb_size: int = 128,\n",
    "                ):\n",
    "        super(KeypointModel, self).__init__()\n",
    "\n",
    "        # in_features is the result of flattening the input of (x,y,c).(k1, ..., k42)\n",
    "        self.keys_emb = KeypointsEmbedding(keys_len=keys_len, kernel_size=kernel_size, emb_size=emb_size, keys_initial_emb_size=keys_initial_emb_size)\n",
    "        self.src_pe = PositionalEncoding(emb_size=emb_size, max_len=(src_max_len - kernel_size + 1))\n",
    "        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)\n",
    "        self.tgt_pe = PositionalEncoding(emb_size=emb_size, max_len=tgt_max_len)\n",
    "        self.transformer = nn.Transformer(d_model=emb_size)\n",
    "        self.generator = nn.Linear(emb_size, tgt_vocab_size)\n",
    "        \n",
    "\n",
    "    def forward(self,\n",
    "                src: List[Tensor],\n",
    "                tgt: Tensor,\n",
    "                src_mask: Tensor,\n",
    "                tgt_mask: Tensor,\n",
    "                src_padding_mask: Tensor,\n",
    "                tgt_padding_mask: Tensor,\n",
    "                memory_key_padding_mask: Tensor):\n",
    "        src_emb = self.src_pe(self.keys_emb(src))\n",
    "        tgt_emb = self.tgt_pe(self.tgt_tok_emb(tgt))\n",
    "        outs = self.transformer(src_emb, tgt_emb, src_mask, tgt_mask, None,\n",
    "                                src_padding_mask, tgt_padding_mask, memory_key_padding_mask)\n",
    "        return self.generator(outs)\n",
    "\n",
    "    def encode(self, src: Tensor, src_mask: Tensor):\n",
    "        return self.transformer.encoder(self.src_pe(self.keys_emb(src)), src_mask)\n",
    "\n",
    "    def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor):\n",
    "        return self.transformer.decoder(self.tgt_pe(self.tgt_tok_emb(tgt)), memory, tgt_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(0)\n",
    "\n",
    "SRC_MAX_LEN = max_frames\n",
    "TGT_VOCAB_SIZE = len(dataset.vocab)\n",
    "EMB_SIZE = 512\n",
    "NHEAD = 8\n",
    "FFN_HID_DIM = 512\n",
    "BATCH_SIZE = 128\n",
    "NUM_ENCODER_LAYERS = 3\n",
    "NUM_DECODER_LAYERS = 3\n",
    "\n",
    "\n",
    "transformer = KeypointModel(NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, EMB_SIZE,\n",
    "                                 NHEAD, SRC_VOCAB_SIZE, TGT_VOCAB_SIZE, FFN_HID_DIM)\n",
    "\n",
    "for p in transformer.parameters():\n",
    "    if p.dim() > 1:\n",
    "        nn.init.xavier_uniform_(p)\n",
    "\n",
    "transformer = transformer.to(DEVICE)\n",
    "\n",
    "loss_fn = torch.nn.CrossEntropyLoss(ignore_index=PAD_IDX)\n",
    "\n",
    "optimizer = torch.optim.Adam(transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([71, 1, 32])\n"
     ]
    }
   ],
   "source": [
    "print(model(vid_keys, None).size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., -inf, -inf, -inf, -inf],\n",
      "        [0., 0., -inf, -inf, -inf],\n",
      "        [0., 0., 0., -inf, -inf],\n",
      "        [0., 0., 0., 0., -inf],\n",
      "        [0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "def generate_square_subsequent_mask(sz):\n",
    "    mask = (torch.triu(torch.ones((sz, sz))) == 1).transpose(0, 1)\n",
    "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
    "    return mask\n",
    "\n",
    "print(generate_square_subsequent_mask(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEsCAYAAAAsMK9uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABQl0lEQVR4nO3dd3gUVRfA4d9JIAkl9CK9SgcR6UjviPQmSG8KNlARFJVPEQUUUCmCKCC9SweRjtKL9KL0okAooQVS7vfHTMIm2YQkJGzKeZ9nn03ulD2zs7tn5t65d8QYg1JKKQXg5uoAlFJKxR+aFJRSSoXQpKCUUiqEJgWllFIhNCkopZQKoUlBKaVUCE0KDkTExODRJQ7iGGKve0gsrW+jvb4asbG+xExEutjv1dQozPusPa+/iGSNwvzN7PkviEi0v3siktde/kxUyqOwvhr2chujG0tMxPbnOraJyNS4+k4nJMlcHUA8M81JWUGgCvAfsNrJ9L/jNCIVbxljTorIn0BloD0w+jGLdLafpxtjguI0OBcQEQNgjBFXx6JiTpOCA2NMl7Bl9lFDFeCYs+lxZCwwB7gWS+vrBKQEzsXS+tQjU7GSQmciSQoikhF4yWGZ2HQRKAr4x/J6Y1tsf65VHNCkEA8ZY64Ri18cY4wmg7gzF/gWeE5EnjPG/BXBfO2B5MB2Y8zx2AzAGOMPHIvNdcaF2P5cq7ihbQpPwLGuXkTqiMhvInLdLittz1NcRD4XkW0icllEHorIvyKyWESqRLBep3WvjuUikl1Eptjr8hORoyLy1uPijCT+SiKyWkRuisg9EflDROpGsu3lRGSFPf8dEflTRFrEpH5bRJKLSEcRmSsiJ+z13RGRv0TkExFJFcFyJrjKwl5+tx37dRFZKCIFI3nNNiKyXUTuioiPiCwTkTJRjTmYMcYXWGz/2ymSWYOrjqbar+8tIr1FZKmI/CMi90XEV0R2isjbIhLlA7bHveciUktENojIbRG5JSLrRaR2JOuL1v4Qux3G4f9Q7W4O5RG2KYiIm4h0sz93t+z346iIDBORdE7mD2kPERFPEfmfiPwtIg/EarP5VkS8o/QGRoH9Gv1FZI/9XtwVkX0i8oGIpIhgmRYi8rsdzwP7u7pbRL4Wkcxh5n1BRGbbnwU/+zN8RER+jsnn8okYY/QRyQPoAhhgo5NpG+1pPwBBwD5gFrAFKGXPM9me9hewDJgPHLCXCwDaOVnvEHv6kAjKfwYuAxeAX+04Au1pH0cSZ40IykdgVT0cAhYABx3iq+lkfQ2AB/Y8f9nb/Ie9nd/Y5Wei8R7ntJe5CmzGqmL4Dbhll+8GUjhZztiPYXb8fwIL7ffFAP8CmZws96E9PdB+D2YDJwE/YII9bWo04q/r8HrJnEwvbk+/D6Szy160yy4C6+0Y1tvzGPuzImHWk9fZextRuT3tVXu/GGCHva/+srf9O5x8tqO7P+xtmeqwP6Y6PqLwuRZgnsN7tNL+/7JddhLIEWaZGva0P+x9eMuOcQXga0/7Pex7+Jj9GLwNXcKUp8D6Thv7dX61P2c37LKdgHeYZb6wpz0A1tnv+2p7WwxQMcz3KcAu34V19rkE6/ckEBj4VH/znuaLJcQHUUsK4T5IDvNUB3I5KX8ZeAhcB1KGmRbRl2eIw+tNBDwcprWyy+8AqSKIs0YE5UFAV4dyAb63p20Is0wqrEZ3A/R3sk3BH+4z0XiPvbHq25OFKU9nf5GMsy+Gw3vxH1DOoTw1sN2e9mmYZcrYXzQ/oJZDuRtWNVDID1s04nfDaq8xwEtOpo+wp812KMtpfzbC/vBns38MDGEOGIhmUgBy2J8HA7waZtq7Dtu6Mcy0J9ofkbxPEX2u3wiOH8jnUJ4C68fRAKvDLFPDIf5twDMO0woAN+1p1aOxH6fiPCl8bZfvweEgA8iAlWgN8INDuRdWcvMF8jt5neeALA7/b7DX0crJvNmBYlHdhth4PLUXSqgPopYUVsdw3TOd/ZBE8uUJLj8DeDpZX/ARfo0I4oyofJaTdWWypz0Ekjt5P/ZFsE2zcfID9QTv/7P2+nY5mRb8o9DLybSWzvYb1lmWAcY7WcYT68g9WknBXjb4yHBemHJ3h3XWj+K6gs885ocpz+vsvY2k/BO7fGUEr7M7os/2k+yPSJaN6HP9D06SoD0tG4/Onoo6lNfg0dleUSfLfe/stR6zbVMJkxSwElNwYq3oZJky9jQ/Hp0FZrbL9kfxdQ/b86d70u9LbDy0oTl2LI5sooikBRpjHSGkx2pwBChhPxfCOu2NqvXGmAdOyo/b68wWjXWBk0ttjTHXRMQHyIiVIC7bk6raz/MiWNdsoF00Xx+w2imAmkAerKulxH6A9R5FxNmlwsGNuWHfi+r288ywCxhjHojIfODtqMbsYCpWtVQTEUlnjLlpl9fFOtq7CKx1XEBEBKiG9Z5mx/oBEqwjdYh8m6Miwm21zQBeiGjhJ9gfUSYiOYH8WAcf88NON8ZcFpG1WGeh1YGjYWY5a4wJWwYR7//oegHr7PgfY8x2J/HtFZGDQEmgItYB4lUROYt18cEo4McIYgy2GygGzBaRL7AuRgh4wrhjTJNC7Ijw6h4RaY51dJoukuXTRPP1LkRQfsd+9ozF9WUMs74c9vPZCJaJqDxCIpIaq976pUhmi+w9chZ/RO9FcPxnIlhXROWRMqH7LLTFqt6DCPomiMgzWHXTFSJZbXQ/F2HFaFtjYX9ER3CM54wxgRHMcyrMvI5i+7sQVvBrno5knlNYScExvo5Y72E/oJ+IXAW2YrWXzDLG3HOYdyBQBKttoQFwT0R2YB1ETDPGXHrCbYgWvfoodtx3VigiubAamNJhVS+UwDoKdDNWB58vg2eN5uvFdsen2FxfTDoufYX1A3QY64zqGaz2EiEKX2oTs45gJgbLPM5U+7kzhJwhNg0zLdhkrISwBaiFVeWQ3N7mwvY8sdUJLLrb+kT7I5qis43OtiOuOwHGKD5jzBasqrYWWAcIV4DmwI/AURHJ4zDvZaAS1pnQV8B+rMb7YcDfIhJZco51mhTi1ktYjU4LjTGDjTGHjTF3jF2RiNVbOqG5aD/niWB67hiss5X93M4Ys8IY85+xrr2H2H+PguPPG8H0iMqjYi7WAUIlEXkWaINVJRSqb4J9SWdDrPrwl40xG4wx1xyqDGJrm2O6rU9zfwQf6ecWEfcI5slnPz/VI2ZbcHz5IpnHaXzGmHvGmMXGmNeMMSWwvjOrsL4jX4WZN8gYs9kYM8gYUwWryvYrrM/P5FjYjijTpBC3MtjP58NOEJFMWPXNCc0W+7lNBNNficE6I3yfYri+yGyyn9uHnSAiHjz6QYw2E77PQnDV0dQws6bF+u7dNsbccrKq2NrmCLfV1iGC8pjuD3+A6PSxMMZcwKqa8QBah51uV7PVs//dHNX1xqI9wF2ggIhUDDtRrP5IpbDaRMK1OTgyVifSofa/pR4zry9WG9UD4Jmw/RrikiaFuBXcy7SlOAyYZh8pTibydob4agHWqXBpEennOEFEGuPkix0Fwe9TnzDrq4N16WRsGodV5dBdHDrziTVA3Vc4r7eOjqn282tYw6P4YZ1BOPoP65LJdCIS6gdbRF4l4h/r6PoJ6wftJREJ9WMuIu8AZSNYLqb7I/jMpGg04xxtP38pIiFH5HansAlYZ9trjDFHorneJ2aMuc+j9qGx9sFccHzp7fgApgRfXCAieUSkewSd5162n0PaIUXkXbvBPax6WNV1vlifl6dCk0LcWobVUSgXcEJElojIQqwGvkrAFBfGFiPGmDtYR8D+wCi7l+ssEdkKLMUa3wasI6eoCj56GiYie+2enX9iNbR9G1uxAxhj9mBdqukFrBOrp+8srB/CPjz6AYipdVhH2ME/Hr86XIkUHEMgVn0xwEy7F+8sEdkHTAeGP2EMwa9zEXgd+7JjsXpwzxKR/cAorMs2nYnp/gg+S1onInNEZLKIRKXqYxzWlUd5gcMislxE5mJdqtrMfu4ehfXElcFYjcQvAP+INRrBQjuuilgdzt53mD891kHfVbFGMpgtIvNF5DhWo/Id4FOH+T8GzonIIRFZYO+jbVhVTQCDHKrv4pwmhThk78jqWEdCV4D6WI2LS7Gub06QYxIZY1ZjNYStwvoiN8W6kq0t1pkERGOMG2PMPKyqtC1Y9bONsT6bnY0xg2It8Eev9wVWrLux9kdDrNFuX+QxVQBRWHcQ8ItD0dQI5huJdenuLqyqhEZYR4Mv8eSJyfF1pmO9txuxLnRobL9OfWBRBMvEdH98hJVs7mA1sHYnCj/m9nvWDuiB1chaHesz5Yt19lbOTnAuYZ8t1AHew0oE9bA+M+eBQVgd5G47LPIP0B9YA2TBOjuoh9WxcwxQ0hiz02H+N7AOBgSojZUIM2Nd9l3FGDM+jjbNKXnU5qnUkxORj7CONMcZY95wdTxKqejRpKCizW78S2Y3EjqWBx99pgQqhDkaUkolANp5TcVEWWCpiBzAah8JwurhWtye/qUmBKUSJj1TUNFmd7wZhFX3+wzWAHQ3sC7fm2iM+dV10SmlnoQmBaWUUiESfPVRpkyZTN68eV0dhlJKJSh79uy5ZowJ1ykuwSeFvHnzsnv3bleHoZRSCYo9kms42k9BKaVUCE0KSimlQmhSUEopFUKTglJKqRCaFJRSSoXQpKCUUiqEJgWllFIhEnw/hZj6ed/PZE6ZmZcLv/z4mVWCcevWLa5du8bDh9G5nYNSCZ+7uzve3t5kyJABT8+Y30o7SSYF/0B/xu4cy+mbp9nXex950+V1dUgqFvj5+fHff/+RM2dOUqRIgUhs3fdeqfjNGIO/vz++vr6cO3eO3LlzxzgxJMnqo+TuyZnfej7GGFrPb82DgAeuDknFgqtXr5I5c2ZSpkypCUElKSKCh4cHmTJlIn369Fy/fj3G60qSSQGgQIYCTGk6hd2XdvPub7F9G2DlCn5+fqROndrVYSjlUmnSpOH27duPnzECSTYpADQv2pz+Ffszbtc45h4Ke291ldAEBASQLFmSrBFVKkTy5MkJDAyM8fJJOikAfFXnKyrlrESPZT04fu24q8NRT0irjVRS96TfgSSfFJK7J2duq7l4unvSan4r7vnfc3VISinlMkk+KQDkSpuLGS1mcPjKYd5YqfeaV0olXZoUbA0KNuCjqh8xZf8Upuyb4upwlIoXzpw5g4gwZMiQGK+jS5cu8aJaT0To0qWLq8OI95JkUggIgAED4IcfQpcPqTGEWvlq0WdlHw78d8A1wSkVCRGJ8uPMmTOuDlclQEnyUg03Nzh8GMaMgeefhwoVrHJ3N3dmtZhF6YmlaT2/Nbt67iKNZxqXxqqUo+nTp4f6f8uWLUyaNIlevXpRtWrVUNMyZw53p8Voy5MnD/fv33+iq7p+/PFHfgh7BKbirSSbFGbMgLJloVUr2LMHsmSxpmVNnZU5LedQ65da9FzWkzkt58SLU1+lAF599dVQ/wcEBDBp0iQqVaoUblpYt2/fxtvbO1qvJyJ4eXlFO05HyZMnJ3ny5E+0DvX0JMnqI4D06WHhQrh2Ddq1s6qUglXPW50van3BvMPzGL9rvOuCVCqG8ubNS40aNdi3bx/169cnbdq0lCpVCrCSw+DBg6lQoQKZMmXC09OTggULMnDgQO7dC331nbM2Bcey5cuXU65cOby8vMiWLRvvv/8+AY5fJpy3KQSX3bp1i9dff50sWbLg5eVFlSpV2LFjR7jt8fHxoVu3bmTMmJHUqVNTq1Yt9u3bR40aNcibN+8TvVeTJ0+mTJkypEiRgrRp01KvXj22bt0abr4VK1ZQvXp1MmXKRIoUKcidOzctWrTgxIkTIfOcP3+ebt26kSdPHjw9PcmSJQuVK1dm2rRpTxTj05QkzxSClS5ttSt06QKDB8NXXz2aNqDKALae20q/Nf0on6M85XKUc1WYSsXIuXPnqFWrFq1bt6Zly5bcuXMHgIsXLzJ58mRatmxJ+/btSZYsGZs2bWLEiBHs27ePNWvWRGn9K1euZPz48bz22mt069aNJUuW8PXXX5M+fXo+/PDDKK2jfv36ZM6cmU8++QQfHx9GjRpFo0aNOHPmTMhZzcOHD6lTpw779++nS5culC9fngMHDlCnTh0yZMgQszfH9sEHHzBixAjKly/PsGHDuH37NpMmTaJmzZosWbKERo0aAbBp0yaaNGlCyZIlGTRoEOnSpePSpUv8/vvv/P333xQqVIiAgADq1q3LxYsX6dOnD4UKFeLWrVscOHCALVu20Llz5yeK9WlJ0kkBoHNn2L4dhg+32haaN7fK3cSNac2mUWZSGVrPb82+3vtInyK9a4NVMfLO6nfY/+9+V4cRSulnSjOmwZg4fY3Tp0/z448/0qNHj1Dl+fPn5/z586GqdPr27cvHH3/M0KFD2blzJ+XLl3/s+g8fPszhw4dDjtRfe+01SpYsyffffx/lpFCmTBnGj390Nl6sWDHatGnDrFmz6N27N2Adye/fv5+hQ4fy0UcfhcxbsmRJ+vbtS548eaL0WmEdP36ckSNHUqVKFdavX4+HhwcAPXr0oFixYvTp04d//vkHd3d3lixZQlBQEL/99htZguuagY8//jjk7yNHjnD8+HGGDx/OgAEDYhRTfJBkq48cjRkD5ctbCcLhTJCMKTMyv/V8Lt2+ROdfO2OMcVmMSkVXhgwZ6Nq1a7hyDw+PkIQQEBDAjRs3uHbtGnXq1AFwWn3jTLNmzUJV3YgINWvW5N9//w05K3mcfv36hfq/Vq1aAJw8eTKkbNmyZbi7u/P222+Hmrdnz56kTZs2Sq/jzJIlSzDGMGDAgJCEAJA9e3a6dOnC2bNn2bdvH0DI6yxcuDBc9Viw4Hk2bNjAlStXYhyXqyX5MwUAT09YsADKlIEWLawzh+Bx1crnKM/X9b7m7dVv8822b3iv8nuuDVZFW1wfkcdXBQoUwN3d3em08ePH88MPP3D48GGCgoJCTbtx40aU1p8/f/5wZRkzZgSsNoCoDE4Ydh2Oywc7ffo02bNnD7e+5MmTky9fvijHG9bp06cBKF68eLhpJUqUAODUqVOULVuWN954gyVLltCnTx8++OADXnzxRRo0aMArr7wScpVXnjx5+Oijj/jyyy/Jli0bpUuXpnbt2rRu3Zpy5RJO9bOeKdhy5YI5c+DoUejRAxxPCt4s/yYti7Zk4O8D+fP8n64LUqloSJkypdPyUaNG0bdvX7Jly8bEiRNZsWIFa9euZerUqQDhkkREIko4QJTPqiNah+PycXWGHp31ZsyYkV27drFhwwbefPNNbt++Tb9+/ShUqBDbtm0LmW/o0KGcPHmSMWPGUKBAASZPnkz58uX54IMP4mIT4oQmBQe1a8MXX8DcufDdd4/KRYSfmvxEnnR5aLugLdfuXXNdkEo9oenTp5M3b15WrVpFjx49aNSoEXXq1CFr1qyuDs2pfPnycenSpXBVUv7+/iFH+zFRoEABwGobCevIkSNA6DMZd3d3atSowRdffMGWLVvYt28fd+7cYejQoaGWzZ8/P2+++Sbz5s3j0qVLVKtWjREjRiSYKiVNCmF88AE0awbvvQdbtjwqT+uVlvmt53Pl7hU6Le5EkIna0ZRS8Y27uzsiEupIOSAggK8cL7+LR15++WUCAwP59ttvQ5X/+OOP3Lp1K8brbdKkCSLCyJEj8ff3Dym/fPkyU6ZMIU+ePDz//PMAXLsW/kCwSJEipEiRIuSGNrdu3Qq1HgAvLy+KFi0KRL1aztW0TSEMEZg6FcqVgzZtYO9eyJbNmlYmWxlG1x9N35V9GfHHCAa+ONClsSoVE61atWLQoEE0bNiQFi1a4Ovry6xZs+JtB7MePXowceJEBg8ezN9//x1ySeq8efMoWLBghA2/j1O4cGHef/99RowYQbVq1Wjbtm3IJal37txh5syZIdVbPXv25MKFC9SrVy+kl/fcuXO5ffs2nTp1AqwG5l69etGyZUsKFy5M6tSp2bNnD5MnT6ZChQoULlw41t6TuKRJwYm0aa2ObRUrQtu2sG4dBH9fXi/7OpvPbmbw+sFUyVWFqnmqRr4ypeKZ999/H2MMP/30E2+//TbPPPMMbdu2pWvXrhQrVszV4YXj6enJunXreP/991myZAnz5s2jQoUKrFu3jh49eoTrcBcdw4cPp2DBgowfP56BAwfi4eFBhQoVmDVrVqhhQzp27MjUqVOZNm0aV69eJU2aNBQrVowFCxbQsmVLAJ577jlatGjBxo0bmTlzJoGBgeTOnZsPP/yQd99NOHd3lIR+mWXZsmXN7t2742Tds2dD+/bQrx+MGvWo3PeBL2UnleWu/1329d5HllRZIl6JemqOHj0acqquEr/AwEAyZcpEhQoVWL16tavDiVei8l0QkT3GmLJhy7VNIRKvvAJvvQWjR1uNz8HSeKZhXut5+NzzoePijtq+oFQcu3//friyH374gZs3b1K3bl0XRJR4afXRY4wcaQ2Y17271Y/h2Wet8tLPlOa7ht/Re3lvhm0ZxuBqg10bqFKJWM+ePfHz86Ny5cp4enqybds2Zs2aRcGCBenVq5erw0tU9EzhMTw8rGokDw+rKunhw0fTepbpSfuS7fl046dsOL3BdUEqlcjVq1eP8+fP8/nnn/POO++wceNGevTowdatW6M98quKnLYpRNGiRdCypXVznuHDH5XfeXiHspPKcuvBLfb33k/W1PHzWu+kQNsUlLJom8JT0KIF9OoFI0bA778/Kk/tkZr5redzy+8W7Re1JzAo0HVBKqXUE9KkEA2jR0PRotCpE1y9+qi8ZNaSjG00lvWn1/P55s9dF6BSSj0hTQrRkDKl1b7g42M1PDvWvHUt3ZWOpTry2abP+P3U7xGvRCml4jFNCtH03HNWFdKyZTBu3KNyEWHCSxMokqkIHRZ14PLty64LUimlYkiTQgy89RY0amSNj3Tw4KPyVB6pmN96Prcf3KbTrzo+klIq4dGkEAMiMGUKpEtndXBz7FdTPEtxvm3wLb+f+p0Rf4xwWYxKKRUTmhRiKEsW+OUXOHwYwg5r0qNMD1oXa83g9YPZfmG7awJUSqkY0KTwBOrVsxLChAmwZMmjchFh0suTyJkmJ68sfIWbfjddFqNSSkWHJoUnNGyYNfxFt25w8eKj8nRe6Zjdcjbnb53nteWv6f2dVbx35swZRIQhQ4aEKhcRunTpEqV1DBkyBBHhzJkzsR7f1KlTERE2btwY6+uOqojeo8REk8IT8vCAWbPAzw86doRAh75rlXJV4vOanzP38Fx+3vez64JUiUbr1q0REfbv3x/hPMYY8uXLR7p06ZwOJBefbdy4kSFDhnDz5k1Xh5JkaVKIBYULw/ffw4YN1gB6jgZUGUCtfLV4c9WbHL161DUBqkSje/fuAEyZMiXCeTZs2MCZM2do164dKVKkeOLXvH//Pj/++OMTrycqNm7cyP/+9z+nSaFjx47cv3+fatWqPZVYkipNCrGka1frTm0ffww7dz4qd3dzZ3rz6aTySEW7he3wC/BzXZAqwatXrx65cuVi5syZPHQcndFBcMIITiBPysvLK17clc3d3R0vLy/c3PRnKy7puxtLRGDiRMie3bpM1df30bTs3tmZ1mwaB/47wHu/vee6IFWC5+bmRpcuXfDx8WHp0qXhpvv6+rJo0SJKlChBuXLluH37NoMHD6ZChQpkypQJT09PChYsyMCBA6N8xzJnbQpBQUF8+eWX5MuXDy8vL0qWLMnMmTOdLn/s2DH69OlD8eLF8fb2JmXKlLzwwgvhzj66dOnC//73PwDy5cuHiISqv4+oTeHatWv07duXXLly4eHhQa5cuejbty8+Pj6h5gtefv369Xz99dcUKFAAT09PChUqxLRp06L0XkQkICCA4cOHU6xYMby8vMiYMSPNmzfnoGNHJtsvv/xC+fLlSZcuHalSpSJ//vx06NCBqw5j5xw+fJjWrVuTI0cOPD09eeaZZ6hZsyYrVqx4ojijQu+nEIvSpbPaF6pXt84cFiywkgVAo2cb0a9iP0ZvH03d/HVpWqSpS2NVCVfXrl0ZOnQoU6ZMoVWrVqGmzZkzh3v37oWcJVy8eJHJkyfTsmVL2rdvT7Jkydi0aRMjRoxg3759rFmzJkYx9O/fn2+//ZZq1arRr18/rly5Qt++fcmfP3+4eTdu3MjmzZtp3Lgx+fLl4+7du8yfP59evXpx7do1Bg0aBEDv3r3x9fVl8eLFjB49mkyZMgFQqlSpCOO4desWlStX5u+//6Zbt26UKVOGffv2MWHCBNavX8/OnTvDDa394Ycfcv/+fXr37o2npycTJkygS5cuFCxYkCpVqsTo/ejQoQPz5s2jbt26vP766/z777+MGzeOSpUqsWXLFp5//nkAZsyYQefOnalatSqfffYZKVKk4Ny5c6xatYorV66QOXNmfHx8qFWrFgCvvfYaefLk4dq1a+zevZsdO3bw0ksvxSjGKDPGJOjHCy+8YOKbr782BowZMSJ0uZ+/nykzsYzJMDyDOX/rvGuCS8SOHDnitPztt42pXj1+Pd5++8m2tVatWsbd3d1cvHgxVHnFihWNh4eHuXr1qjHGmAcPHpiHDx+GW37w4MEGMDt27AgpO336tAHMp59+GmpewHTu3Dnk/2PHjhkRMbVq1TIBAQEh5Xv27DEiYgBz+vTpkPI7d+6Ee/3AwEBTvXp1kyZNmlDxffrpp+GWDzZlyhQDmA0bNoSUffjhhwYw48aNCzXv2LFjDWAGDx4cbvnSpUubBw8ehJRfuHDBeHh4mHbt2oV7zbCcvUe//fabAUybNm1MUFBQSPlff/1l3N3dzYsvvhhS1rx5c+Pt7W38/f0jfI0lS5YYwMydO/ex8UQkou+CI2C3cfKbqtVHcaB/f2jVCgYOBMczXc9knsxpOYeHgQ/psKiDDrOtYqx79+4EBgYyffr0kLJjx46xfft2mjRpEnKU7eHhEdIeEBAQwI0bN7h27Rp16tQBYMeOHdF+7SVLlmCMoX///ri7u4eUlylTxumtMVOlShXyt5+fHz4+Ply/fp169erh6+vLsWPHoh1DsMWLF5M5c+Zwd1/r3bs3mTJlYvHixeGW6dOnDx4eHiH/58iRg0KFCnHy5MkYxwDw0UcfIcFVA1hnOI0bN2br1q0hVUNp06bl3r17rFixIsLL1NOmTQvAqlWr8HWsh35KtPooDojAzz9b4yK1bQt790KOHNa0ZzM+y/hG4+n0ayeGbh7KpzU+dW2wScCYMa6OIPa1aNGCdOnSMWXKFD744AMAfv7Zuuy5W7duoeYdP348P/zwA4cPHyYoKPR4XDdu3Ij2a586dQqAIkWKhJtWrFgxfvvtt1Bld+7cYciQIcybN4/z58+HWyYmMQQ7ffo0ZcuWJVmy0D9lyZIlo3DhwuzduzfcMs6quDJmzMjZs2djHIObm5vTm9qUKFGCJUuWcPr0aTJnzsyHH37I5s2badasGRkzZqR69eo0bNiQtm3bhlRzVa9enU6dOjF16lRmzpxJuXLlqFOnDm3btqVYsWIxijE69Ewhjnh7W3dru3vXuirJ8UKRjs91tIbZ3vwZm89udl2QKsHy8vKiffv2HD9+nD///DPkrCFnzpzUq1cvZL5Ro0bRt29fsmXLxsSJE1mxYgVr165l6tSpAOGSRFQEH+E6HhWHneaoffv2jBo1ikaNGjFz5kxWrVrF2rVr6devX4xjeBKOZzeOIjpyf5zoLPfss89y5MgRVqxYQefOnTl79iw9e/akSJEi/PPPPyHzTZs2jYMHDzJ06FAyZszIN998Q6lSpRg7dmyMYowOTQpxqFgx+Okn+PNPeP/90NPGNRpH/vT56bCoAz73fJyvQKlIOPZZWLVqFf/++y+dO3cO9aM3ffp08ubNy6pVq+jRoweNGjWiTp06ZM0a89vGFihQALBu+RhW2LKbN2+yfPlyOnbsyA8//ED79u1p0KABderUCVWFE8xZoolM/vz5OX78OAEBAaHKAwICOHHihNOzgthWoEABgoKCnL4fR44cAayrqYJ5enrSqFEjvvnmG3bv3s2KFSu4dOkSo0aNCrVsiRIlGDBgAEuXLuXChQsUKFCAgQMHxvnoCJoU4ljbtvD22/Ddd9YNeoJ5e3ozp+Uc/rvzH68uflXbF1S0lSlThtKlSzN37lzGjh2LiNC1a9dQ87i7uyMioX5IAgIC+Oqrr2L8uk2aNEFEGDVqFIEOXfj37t3L77+HvsFUcIIK+0N2+fJlJk+eHG7dqVOnBuD69etRiqVZs2ZcvXo13Lp+/PFHrl69SvPmzaO0nifRrFkzAL788stQ23no0CGWLl3Kiy++SObMmQHr8tmwypQpAzza5uvXr4c7e0qXLh358uXj3r17+PnFbV8nbVN4CkaOhN27oUcPKFUKihe3yl/I/gLfN/ye11a8xscbPmZY7WGuDVQlON27d+fNN99kzZo11KhRI+QoPlirVq0YNGgQDRs2pEWLFvj6+jJr1qwn6oxWpEgR+vbty9ixY6lVqxYtW7bkypUrjB07lueee459+/aFzOvt7U29evWYMWMGKVKkoFy5cpw9e5aJEyeSL1++cH0JKlasCMAHH3xAhw4d8PLyokSJEpQoUcJpLAMGDGD+/Pn07duXvXv38vzzz7Nv3z5++uknChcuzIABA2K8nVFVt25d2rRpw5w5c7hx4waNGzcOuSTVy8uL7777LmTeevXqkTZtWqpVq0auXLm4efNmSP+Jjh07AlY/htGjR9O8eXMKFixI8uTJ2bRpE2vWrKFNmzax0ks9Us4uSUpIj/h4SaozFy8akzWrMYUKGXPrVuhpvZb2MgzBzDs0zzXBJRJRuQwvsbl+/brx8vIygPnll1/CTQ8ICDDDhg0zBQoUMB4eHiZ37tzm/fffN0eOHAl3aWVUL0k1xrqkdOjQoSZ37tzGw8PDFC9e3MyYMcPpJaVXr1413bt3N9myZTOenp6mRIkSZtKkSU4vMTXGmOHDh5t8+fKZZMmShYonovmvXLliXn/9dZMjRw6TLFkykyNHDtOnT5+Qy3KDRbS8McZUr17d5MmTx8k7HFpE75G/v7/56quvTJEiRYyHh4dJnz69adq0qTlw4ECo+SZNmmTq1KljsmbNapInT26eeeYZ07BhQ7N+/fqQefbt22c6depkChQoYFKmTGm8vb1NqVKlzNdff238/PweG6MxT3ZJqpg4rp+KjIgMASK7/CbAGBPpIU3ZsmXN7t27YzWuuLJpE9SuDU2bhu7Y9iDgAbV+qcX+f/ezrfs2SmWNuLOOitjRo0edXgGiVFITle+CiOwxxpQNW+7qNoVFQEcnj+Bh5Za5KK44Ub06DB9uXZX09dePyj2TebKg9QLSeaWj2ZxmXL8ftfpUpZSKbS5NCsaYA8aYGWEfQFp7lp9cGV9ciKhjWzbvbCxqs4iLty/SbkE7AoICIlyHUkrFFVefKYQjIimBdsBFYLWLw4l1wR3bnn3WujLJ8cY8FXJWYMJLE1h7ai2Dfh/kuiCVUklWvEsKQBsgDTDFGJMor9OMrGNbt+e70bdcX77e9jWzD86OeCVKKRUH4mNS6A4YIFHfqsyxY5vdsTPE6PqjqZq7Kt2Xdmff5X3OV6CUUnEgXiUFESkMvAisN8acjmS+XiKyW0R2O45BntC0bQvvvgvjx8OkSY/Kk7snZ37r+WRMmZHmc5tz7V74Di9KKRUX4lVSwDpLAAjf1dGBMWaSMaasMaZscE/BhGr4cKhfH/r2hS1bHpVnTZ2VxW0X8++df2kzv402PEeRKy+xVio+eNLvQLxJCiKSDOgEXAfCj3ebSLm7w5w5kD8/tGwJjgM1ls1elkkvT2LDmQ28/9v7Ea9EAdbImGHHwFEqqfH3949w0L+oiDdJAXgZyApMN8Y8cHUwT1O6dLB0KTx4YHVsu3v30bROz3Xi7QpvM2bHGH756xeXxZgQeHl5cefOHVeHoZRL+fr6hrvbXHTEp6QQXHWU6PomREXhwtaAeQcOQJcu4HgGOLLuSGrmrUmvZb3YfSlh9N52hcyZM3P16lXu3bun1UgqSTHG8PDhQ65du8aNGzfIkCFDjNfl0mEuQoIQyQ6cA/YYYypEZ9mENMxFVIwcCQMGwGefwccfPyq/du8aZSeV5WHgQ7Z130aedHlcF2Q8duvWLXx8fHjwIEmdbCqFu7s73t7eZMiQAU9Pz8fOH9EwF/FllNQugDuPaWBOCt57zzpb+OQTKFkS7FF5yZQyEyvar+DFKS9Sf0Z9/uj2BxlTZnRprPFR2rRpQ25nqJSKvnhRfWSMGWaMEWPMj66OxdVErMtTy5WDjh2tW3oGK56lOEvbLeXMzTM0nt2Yuw/vRrwipZSKgXiRFFRoKVLA4sVWz+emTcHxvhxV81RldsvZ7Ly4k7YL2uqlqkqpWKVJIZ7KkcNKDJcuWUNh+Ps/mta8aHPGNRrHipMr6L2stzaqKqVijSaFeKxCBasqacMGa3RVR6+VfY1Pqn3Cz/t/5pMNn7gmQKVUohNfGppVBDp1shqev/nGupVnz56Ppg2pMYRLty8xdMtQsnlno0+5Pq4LVCmVKGhSSACGD4dDh6yhMIoUgapVrXIRYULjCVy5d4U3Vr5BllRZaFWslWuDVUolaFp9lAAED4WRL5/V8HzgwKNpydySMbvlbCrlqkSHRR3YdGaT6wJVSiV4mhQSiHTpYPVqSJUK6tSBY8ceTUuZPCXLXllGgfQFaDqnKQf/OxjhepRSKjKaFBKQfPng99+tvgx16sBph8HFM6TIwOpXV5PKIxUNZjbg7M2zEa9IKaUioEkhgSlc2EoM9+5BrVpw4cKjabnT5mZ1h9XcfXiXBjMb4HPPx3WBKqUSJE0KCVDJkrBmDfj4QO3a8N9/DtOylmTpK0s5feM0jWc35s5DHTVUKRV1mhQSqHLlYOVK60yhbl0rQQSrlqcas1rOYufFnTSd05T7/vddF6hSKkHRpJCAvfgiLFkCJ05AgwZw69ajaS2KtmBas2lsOL2BVvNb8TDwoesCVUolGJoUErg6dWDBAti/Hxo3Dn2DnldLvcoPjX9g5cmVtF/YXsdJUko9liaFRKBxY5g1C/780xpq28/v0bReL/RidP3RLDy6kC6/diEwKNBlcSql4j/t0ZxItG5tXZHUpYs1gN7ChZA8uTXtnYrvcN//Ph+u/5AUyVIw6eVJiIhL41VKxU+aFBKRzp2txNCnD7z6qnX2EHz/7kFVB3HX/y5fbPmClMlTMqbBGE0MSqlwNCkkMq+/biWG994DLy/46SdIZu/lz2t+zj3/e4zePpqUyVMyrPYwTQxKqVA0KSRC774L9+9b93j+91+YO9caJkNE+KbeN9z3v89Xf3xFKo9UDK422NXhKqXiEU0KidTgwZA9O7z2GlSsCMuXQ8GCVmIY99I47gXc4+MNH5MyeUr6V+r/+BUqpZIEvfooEevWzRoS49o1KF/eulkPgJu48VOTn2hdrDXv/vYuE3ZNcG2gSql4Q5NCIletGuzcCdmyQb16MHGiVZ7MLRkzWszg5UIv02dlH6btn+baQJVS8YImhSQgf37Yts0aDuO11+CddyAgADzcPZjXeh5189el29JuTP9ruqtDVUq5WKwkBRFJJiItRaSniDwTG+tUsStNGli2DPr1g2+/tTq83boFXsm8+LXdr9TMW5NOv3Zi/K7xrg5VKeVC0U4KIjJCRHY5/C/A78A8YCJwUEQKxF6IKra4u8OoUfDjj7BuHVSqBP/8Y92kZ3n75TQt3JS+K/vy1davXB2qUspFYnKm0ADY4vD/y0A1YCTQ3i4b+IRxqTjUowesXWsNuV2+PGzaZJ0xzG89n/Yl2zNo3SA+XPchxhhXh6qUespicklqLuCkw/8vA6eNMQMBRKQ40CEWYlNxqEYN2LEDXn7ZGlRvwgTo0SM505tPx9vDmy+3fonvA1++a/gdbqJNT0olFTFJCh6A46hqNbGqj4KdArI9SVDq6ShYELZvh7ZtoWdPuHgRPvnEjQkvTSCNZxpG/jkS3we+/Nz0Z5K5aZcWpZKCmBwCngcqQshZQX5gk8P0LIDe7iuBSJvW6tjWtSsMGQKDBgEIw+sMZ2jNoUw/MJ0289vwIOCBiyNVSj0NMTn8mwN8LCJZgOKAL7DSYfrzwD+xEJt6SpIlg8mTrbGShg+3hsgYM0b4qNpHeHt68/bqt2kypwmL2iwilUcqV4erlIpDMUkKX2K1KzQDbgGdjDE3AUQkLdAEGB1L8amnxM0Nxo2zEsPo0fDgAYwfD29VeAtvD296LOtB/Rn1WdF+BWm90ro6XKVUHIl2UjDGPAC624+wbmO1J9x7wriUC4jAN99AihQwbJh1s56ffoKuz3cltUdqOizqQK1farG6w2oyp8rs6nCVUnEgtlsPkxtjbj1+NhVficAXX1iJ4eOPrcQwfTq0Lt6a1B6paTGvBdWnVmdtx7XkSJPD1eEqpWJZTDqvNRSRIWHK+oiIL3BXRGaJSPLYClC5xuDBMHKkNex2mzZWdVLDZxuyusNqLvheoNJPlTh05ZCrw1RKxbKYXH30PlAk+B8RKQp8C1wC1gJtgb6xEp1yqffeg++/h19/te79fP8+VM9bnU1dNhEQFECVn6uw7tQ6V4eplIpFMUkKRYHdDv+3Be4D5Y0xDYG5QOdYiE3FA2+8YV2ZtGaNNV7S3bvwfLbn2d5jO7nS5KLBzAb88tcvrg5TKRVLYpIU0gPXHP6vA6w3xvja/28E8j1hXCoe6d4dfvkFNm6E+vXB1xdyp83N1m5bqZanGp1/7cxnmz7TYTGUSgRikhSuAXkARMQbKAdsdZieHHB/8tBUfPLqq1b7wo4d1rAY169DOq90rOqwik7PdeLTjZ/SfWl3/AP9XR2qUuoJxOTqo23AayJyGGhor8Ox81pB4HIsxKbimVatwNPTeq5UCRYvhmLFPJjadCp50+bls82fccH3AgvaLCCNZxpXh6uUioGYnCl8ai83D+gK/GKMOQIhw2g3B/6ItQhVvPLyy9YtPm/ehAoVYOFC677P/6v5P35u8jMbzmyg6pSqXPC94OpQlVIxEO2kYCeAokBToIYxpqvD5HRYvZnHxEZwKn6qWhX27oXixa2zhkGDIDDQ6uS2sv1KTt84TcXJFfnr379cHapSKppiNCayMea6MWaZMWZzmPIbxphvjTH6a5DI5chh3YehVy/46ito1Ah8fKBugbps7WY1MVWdUpXf/vnNxZEqpaIjxgPli0gBEekvImPtR3+941rS4ukJEydad3LbuBHKloX9+6FU1lJs77GdfOnz0WhmIybvnezqUJVSURSjpCAinwPHgK+BPvbja+C4iHwWe+GphKBHD9i8Gfz9oXJlmDkTcqbJyZauW6idvzY9l/Wkz4o+PAx86OpQlVKPEZNhLroBHwE7sBqVn7UfzbCuTPpIRLpGuAKVKFWoAHv2QLly1uWr77wDKdzSsKL9Ct6v/D4Tdk+gxtQaXLp9ydWhKqUiIdHtcCQie4CHQFVjTECYacmw7t/sYYx5IdaijETZsmXN7t27Hz+jeir8/eH99+Hbb6F6datvQ9asMP/wfLousUZbnd96PlXzVHV1qEolaSKyxxhTNmx5TIe5mBM2IQDYZXPseVQSlDw5jBljjay6cye88ILV4a118dbs6LGDNJ5pqPVLLb7f8b32gFYqHopJUngIpI5kurc9j0rCXn0V/vzTShJVqlhnD/lSF2dnz500LNiQt1a/RedfO3PPX2+9oVR8EpOksAvoLSJZw06wb9HZC6u9QSVxpUtb/Rm6dYOvv4YSJWDn5nT82u5XPqvxGTMOzKDKz1U4feO0q0NVStlikhQ+x7q72lERGSkiXe3H18BR4BlgaGwGqRKu9Olh0iTrktXkya0B9Tp3cuO1Yh+zvP1yztw8Q9kfy2p/BqXiiZj0aN4MtMC69ea7wE/2o79d1twYsyU2g1QJX/Xq8Ndf1t3c5s6FokXh2rZG7OyxixzeOWgwowFfbvlS2xmUcrGY9mhehjU8dgWgHfAKUB7ID+QUkSOxFqFKNLy84LPPYN8+KFwYOneGPu0KMrPGdtqWaMuH6z+k5byW3PS76epQlUqyYtyj2RgTZIzZZYyZZ4yZa4zZbYwJAjIBhWMvRJXYFC8OW7bA+PHWlUkVXkhJ6b9nMbLWGJadWMbzE59n+4Xtrg5TqSQpxklBqSfh5gavvw5Hj1rtDAMHCjPefJtJZfYC8OLPLzJ863CCTJCLI1UqadGkoFwqRw7rvgyLFsHVq9CzaUm63j1Ci6ItGbhuIA1mNODfO/+6OkylkgxNCipeaN4cjhyBFi3g049SEDhnDt/VnMrWc1t57ofnWPP3GleHqFSSoElBxRtp01pXJn3zDSxZIozt2Zk51Q6SJVUWGsxswIC1A3RQPaXiWJRuxyki/aOxzioxjEUpRKB/f2tgvTZt4JUGBRj/wx6253qbkX+OZOOZjcxpNYf86fO7OlSlEqUoDYgnItFt7TPGGPcoBSCSAfgQa5TVnFh9HQ4Bn0Slv4MOiJd4Xb4MbdtaVyq98QZU6b6I19d0J8gEMbHxRNqVaOfqEJVKsCIaEC9KZwpAzViOBwARyQNsxBpL6SfgBJAWKAXkiIvXVAlHtmywbh188AGMHg179rRg1Y/leXdHW15Z+Apr/1nLtw2/JbVHZENxKaWiI9pDZ8fqi4tsAfIC5Y0xl2OyDj1TSBrmz7fGUEqRAmbOCmST26cM2zKMXGlzMb7ReF4q9JKrQ1QqQYnNobNjhYhUA14ERhhjLotIchFJ6ap4VPzWurU1FHemTNCgvjveu4aypetWvD28aTy7MW0XtNVLV5WKBa68+qiR/XxORJYB94G7InJCRF51YVwqnipa1OoB3bIlDBwII9+qzKpmexlacyhLji2hyNgiTNozSTu8KfUEXJkUgofC+BHIAHQGumPdi2F6ZLf0FJFeIrJbRHZfvXo17iNV8Ya3t3XZ6ujRsGIFPJvfg/OzPmJpvaOUyVaG3st7U21KNY5c1eG3lIoJl7UpiMjvQG3gFFDUGPPQLk9vl/kBOezxlCKkbQpJ14kTVp+GqVOt24C2bGko1mw5Yy924faD2wx8cSAfVv0Qr2Rerg5VqXgn3rUpYFUXAcwOTggAxpgbwFKs+zLowHoqQoUKwcSJcPasVZ20dq3w2asvU2zFZar6D+XzTZ/z3A/PsfHMRleHqlSC4cqkcMF+dtY6GHwlUvqnFItKwJ55BoYNg3PnrDu8nf7Hg/WfDyD/PF9u7mxIzZ/r0G1JN3zu+bg6VKXiPVcmhZ32c04n04LLrjylWFQikCYNvPsunDoFP/8MnnhzZfoY0v54hakTvSk0qjQ/7vlRG6KVioQrk8KvWL2XXxWRkN5HIpINq3fzSWPM364JTSVkHh7QtSscOgRLlkCJAhkwq77l7ujd9Pp6GRUnV2TXxV2uDlOpeMllScFuO3gPq+fydhHpLyIDge2AB/CGq2JTiYObGzRpAlu3WveIfjZ7FpizlAPf/Y/y37Sk17JeXLt3zdVhKhWvuHSUVGPMJKAlcAf4HPgIOA7UNMbondxVrKleHfbuFYYPB7d/GpB8wkkmj0vDs2OK8sPuHwgMCnR1iErFCy4d5iI26CWpKrrOnLEG2FuxAlLnPsmdeh0oUy6QcY3GUTFnRVeHp9RTER8vSVXKJfLmhWXLYOFCSBtYEPlpB8emvUmlcfXptqQbV+7q9Q0q6dKkoJIkEesub0ePCm+9Jfht70yqSeeZNvMBhb4vzHc7vtMb+qgkSZOCStK8vWHMGNi1SyiaPw1B82fiPnMtb8/6lkLfF2LKvikEBAW4OkylnhpNCkoBZcrA9u3w/ffgf/YFkk04yb1fv6bb9I8pPr44cw7N0f4NKknQpKCUzd3daoA+flzo2cONm3+2JPm4s/gs/JhXpvaj9A+lWXJsCQn94gylIqNJQakwsmWD8ePh5EmhS2d3bm3tQPKx5zk//x2a/dSTij9VZO0/azU5qERJk4JSEciTByZNss4cOrySDN9NXfEYe5Gjs7tQb+Ir1JhWg63ntro6TKVilSYFpR4jf36YMsW6Uql1y+Tc2fAanuMusWdGM6qOa0LDmQ3549wfrg5TqVihSUGpKCpUCGbMgMOHhWYve3B3XT88x11m05RavPhdK6pOqcqKEyu0WkklaNqjWakYOngQ/vc/qxOcm3sQHkV+x6/kWEpUOc/Aau/RtkRbkrklc3WYSjmlPZqVimUlS8KCBXD8OLz/nhvpfOrCnKUcG7yGV/ueI+8ndRm/azz3/e8/fmVKxRN6pqBULPH3h5UrYfJPhpUrIChIIM9GvCvOp3/3PLxTtRfpvNK5OkylgIjPFDQpKBUHLl2CqVMN4yf5cfFsCvC8RfLn5tOu012+7NiaHGmyuzpElcRpUlDKBYyBzZth5PfXWb0sNYEPPSDtWZ594QI9W+WnQ9NsZNf8oFxAk4JSLnbzJoybcoWfF1zg1L48cD8jAHkL3qdRvRTUrg01akCGDC4NUyURmhSUikcu+/7H4FnzmbX0X/xOVsb9fE0CH6RAxPD880KtWlC7NlSrBilTujpalRhpUlAqHvJ94MukPZMYtXUsl4/l5JlrHUh/qSV/H8iMv7+QIwdMngwNGrg6UpXY6CWpSsVDaTzT8F7l9zjd/ziT3+hK2nrfcvTlrOQcVpI+o1eSJm0QDRtC9+5w65aro1VJgSYFpeIBz2SedC/TnSN9j7CozSIyp0vF+FsvcaVDbiq23cjUqYYSJWDNGldHqhI7TQpKxSNu4kbzos3Z3n07GzpvoFqB8uwsVhvTvTI3g87ToAH06GHw9XV1pCqx0qSgVDwkItTIW4NFbRdx+u3TDGpbC6++laHKcH76OYg8hW6zaPkdV4epEiFNCkrFc7nT5uaL2l9wYcDfzBifkxKDenMz8AItX05N0fpb+PPvQ64OUSUimhSUSiA8k3nSoVQHDn4xmT93+lGy2SqOra1MlbLelHyvP/MOz8M/0N/VYaoETpOCUglQpXzPc2BxQ1ZvuEPmtKk59M0o2na4T7bXu/LZ7yO5du+aq0NUCZT2U1Aqgbt/Hz7+2PDd90H4P3QHdz/c8m6lbLXrDOj0HC2qF0bE1VGq+EY7rymVyN27Z42zNPtXH5atfMCN89agSh7pr/JizXt0b5mL+vXdyJjRxYGqeEGTglJJzMGTN/ns5+2sXBXAveOVwS+DNYxGmUBeapSM9u2hSBFXR6lcRZOCUklUQFAAiw4vYdjc3/jrj6y4nWqIuVABE+RGlSpWb+nWrSF1aldHqp4mTQpKKfZd3sd3O79j5rbf8d/bDs8DfXjwXz5SpQ6i/StudO8O5cujbRBJgCYFpVQIn3s+zDk0hyn7p7JnuyeyvyduR9oS+MCLYsUNPboLHTtCpkyujlTFFU0KSimnDl05xLT905i2czFXd9Qi2V+9CTj/AsmSB9G8mXX2ULs2JEvm6khVbNKkoJSKVEBQAKv/Xs3U/VNZsvkUAbs74X6oC4F305EylaHsC0KFClChglXFlDOnVjMlZJoUlFJR5nPPh9mHZvPzrpns25wdOVuTtNfqcudcQQL83QHIlo1QSaJcOfD2dnHgKso0KSilYuTgfweZfmA6Mw/O5NKNa6S6XoVS/j1JfbUWZw5n4eRJ63RBBIoVgzp14M03oUABFweuIqVJQSn1RAKDAtlwZgO//PULi44u4q7/XfKmy0vLPD0p8rATl47lZPt2+P13CAiAFi3g3XehUiVXR66c0aSglIo1dx/eZfGxxUw/MJ3fT/1OkAmifI7ydCzVkeoZXmH2zxmZMAFu3oTKleG996BJE3B3d3XkKpgmBaVUnLh0+xKzD87mlwO/cOC/A7iLO/UK1KN5/o7c3N6cCd97cfo0FCwI/fpBly6QMqWro1aaFJRSce7AfweYdXAWsw/N5tytc3gl86JRgSbku9yfTbPLsXuXNfZSnz7Qty9kzerqiJMuTQpKqacmyASx7fw2Zh+azbzD87h67yreHml4kQ+4ub4H29dlxsNDeOUVq1qpZk1Il87VUSctmhSUUi4REBTA+tPrmX1oNouOLsL3gS8Z7lUk26ERnNpYmfv33HFzsy5prVvXunqpUiXw8HB15ImbJgWllMv5Bfix8uRKZh+azfITy/HzCyTLzSYUvPE6d45X5PC+lAQGCilTQvXqj5JEiRLaUS62aVJQSsUrvg98WXJsCbMPzea3f34j0ARSKFU5XnjwLsnO1GfnlnQcP27NmzWrlRy6dbOqmjRBPDlNCkqpeOvq3assOLKA2Ydms+XcFgDKZi9Lw8y9SP9vK3ZvSc+aNeDjAy+8AAMGWP0gdDymmNOkoJRKEM7fOs/cw3OZfWg2ey/vRRCq561Oq2df5cHe1kz8Pg0nTkD+/FbnOL3ENWY0KSilEpwTPieYfXA2sw/N5riPVZdULlsFCl7pz6FfX+LgnlRkygRvvGFd4qpDfUedJgWlVIJljOHAfwdYfmI5y08uZ8eFHRhjyHStGSl3DeHczudIkcLQvbvQvz/ky+fqiOM/TQpKqUTjyt0rrDy5kuUnlrPmnzXcuZAL9+0fEPRXezDuNG52n2YvpcLdnZCHmxuh/ncsS5vWul912rSu3rKnR5OCUipRehj4kM1nN7P8xHIW79zJuTXNYPdr8DBNtNeVIwcUL249ihV79JwYk4UmBaVUomeM4bjPcRb+tYq5Ozdw8N9DYNx5Pms5GhV4mTr56pHOMyOBgRAYCEFB1rOPDxw9CocPW4+jR+H+/UfrDU4WxYpZfSYqVoSiRa0zjYRKk4JSKsn5+/rfzD00lzmH53DoyiHcxI1a+WrxSolXaF6kOelTpHe6XFAQnDljJYgjR5wni/TprZ7XVarAiy9aPbJTpHh62/akNCkopZK0Q1cOMefQHOYcmsM/N/4huVtyGhRsQLsS7WhSuAmpPVI/dh1BQfD33/Dnn/DHH9bj6FFrWvLkUKaMlSSCH/F5wD9NCkophVXFtOfynpAEcfH2RbySedGwYENaFWtF40KNSeMZ9fYIHx/Ytg22brWSxK5d8OCBNa1AAavaKV++8I/Uj89BcUqTglJKhRFkgth6bisLjixg4dGFXLp9CQ93D+oVqEeroq1oUrhJhFVMEXnwAPbutRLEtm1w4gScPg1374aeL1MmyJs3dKJ48UWrzeJp0KSglFKRCDJBbL+wnQVHFrDgyALO+54nmVsy6uSvQ6uirWhapCmZUsasd5wxcO2alRycPc6eBX9/a95GjWDgQCtBxOUYT5oUlFIqiowx7Lq0i4VHFrLg6AJO3TiFu7hTI28NWhZtSb0C9cifPj8SS7/agYFw4QLMmAHffgtXr1q3MR04EF56KW6uctKkoJRSMWCMYf+/+60ziKMLOOFzAoA8afNQO19tauevTa18tXgm9TOx8nr37sGUKTBypHUGUbw4fPABtGtnNWbHFk0KSin1hIL7Qaw7tY51p9ex4cwGbvrdBKBY5mJWkshXm+p5q5POK90TvZa/P8ybB199BYcOQe7c8N570L177AwAGG+TgohEFMBdY8xj2+c1KSilXCUwKJB9/+4LSRJbz23lfsB93MSNstnLUitvLV4q9BKVclbC3c09Rq9hDKxcaSWHrVutBuq33rIGAMyQIeaxx/eksAWYFGaSvzFm7uOW16SglIovHgQ8YNuFbaw7tY71Z9az48IOAk0gWVJloWnhpjQr0oza+WrjmcwzRuvfuhWGD4fly61LWo8fh+zZYxZrfE8K04wxXWKyvCYFpVR8dcvvFqv+XsXiY4tZeXIldx7ewdvDm0bPNqJ5keY0fLZhtPpEBDt40EoMgwbFPLZ4nxSAXoCHMeZOdJbXpKCUSgj8AvxYf3o9i48uZsnxJVy9dxUPdw/q5K9Ds8LNaFK4CVlTP70u0PE9KdwFvAB34CowFxhsjLn1uOU1KSilEprAoEC2XdjG4qOLWXxsMadvnkYQKuSsQL389ahboC4VclQguXssXm4URnxOCjuA+cDfQBqgEdAWOAhUdnbmICK9sM4syJ079wtnz559egErpVQsMsZw8MpBFh9dzKq/V7Hr0i6CTBDeHt7UyFuDuvnrUrdAXQpnLBxr/SIgHicFZ0TkQ+ALrLOFLyKbV88UlFKJyY37N9hwZgNr/1nL2lNr+efGPwDkSpOLOvnrUDd/Xerkr0PmVJmf6HUSWlJIDtwB9hhjKkc2ryYFpVRidurGqZAEse70upB+EaWfKc1vr/4W4+QQUVJI9kTRxhFjjL+IXAL0NtxKqSQtf/r89C7bm95lexMYFMiey3tY+89adl/eHeOxmCITL5OCiHgBOYHtro5FKaXiC3c3d8rnKE/5HOXj7DVcejM5EckYwaTPsRLWsqcYjlJKJXmuPlMYLCIVgQ3AOSA11tVHNYEdwPcujE0ppZIcVyeFjUAxoDOQEQgETgIfAaOMMX6uC00ppZIelyYFY8wSYIkrY1BKKfWIS9sUlFJKxS+aFJRSSoXQpKCUUiqEJgWllFIh4uUwF9EhIleBmI6Ilwm4FovhxDeJffsg8W+jbl/CF1+3MY8xJtwYGQk+KTwJEdntbOyPxCKxbx8k/m3U7Uv4Eto2avWRUkqpEJoUlFJKhUjqSWGSqwOIY4l9+yDxb6NuX8KXoLYxSbcpKKWUCi2pnykopZRyoElBKaVUCE0KSimlQiSppCAibiLST0SOiYifiJwXkW9EJJWrY4stImIieNxxdWzRISKDRGS+iJyy4z/zmPkLi8ivInJDRO6KyBYRqfWUwo2R6GyjiAyJZN++9xTDjhIRKSQin4nIdhG5KiK3RWS/iHzk7PuWQPdflLcxIe0/V99P4WkbDbwFLAa+AYra/z8vInWMMUGuDC4WbSH8FQ/+rgjkCQwDrgN7gXSRzSgiBYA/gQBgBHAL6AmsEZGGxpjf4zbUGIvyNjroR/jesXtiMabY0g3oCywFZmJ9/moCQ4E2IlLRGHMfEvT+i/I2Ooj/+88YkyQeQHEgCFgYpvxNwADtXR1jLG2nAaa6Oo5Y2I78Dn8fAs5EMu88rBs0lXYoS401/Mlx7Kvs4tsjmts4xN63eV0ddxS3rSyQ1kn5UHs73kgE+y8625hg9l9Sqj56BRBgTJjyH4F7wKtPO6C4JCIeIpLa1XHElDHmVFTms0/TmwAbjTH7HZa/A0wGCgHl4iLGJxXVbQxLRNKISLw+yzfG7DbG3HIyaa79XAIS/P6L0jaGFd/3X1JKCuWwzhR2OhYa65af+4mnH7wYaoWV6G6LyBUR+V5E0ro6qDhSCvAEtjmZtt1+Tkz79gBW9YqfiPwpIg1dHVA05bSf/7OfE+P+C7uNjuL9/ou32SoOZAeuGWMeOJl2EagsIh7GmIdPOa7YthOYD/wNpAEaAW8A1UWksn0Elphkt58vOpkWXJbjKcUSl25itRP9CdwACgPvACtEpJsxZqrLIosiEXEHPsFqO5hlFyeq/RfBNkIC2n9JKSmkBJwlBAA/h3kSdFIwxlQIU/SLiBwAvgDetp8Tk5T2s7N96xdmngTLGDMmbJmI/IzVFjFaRBYkgIQ/BqgIfGiMOW6XJbb9N4bw25ig9l9Sqj66h3Wa6oyXwzyJ0UisZPeSqwOJA8H7zNm+TdT71RjjA/yAdeVSZddGEzkR+RzrjHWSMeZLh0mJZv9Fso1Oxdf9l5SSwiUgk4g4+/DlwKpaStBnCRExxvhjb7+rY4kDl+xnZ1UMwWXOqiYSizP2c7zdtyIyBBgMTAFeCzM5Uey/x2xjZM7Yz/Fm/yWlpLALa3vLOxaKiBdQGtjtgpieCnsbc+K84SuhO4hV9VDJybSK9nOi3bfAs/ZzvNy3IvIp8CnwC9DD2NdnOkjw+y8K2xiZ+Lf/XH1N7NN6ACWJvJ/Cq66OMRa2MWME5SPtbRzg6hhjuF2Pu4Z/PtZ17s85lAVf536CeHqde1S3EavtL62T8lyAD1ZnqBSu3gYn8X1if+5+AdwS4/6LyjYmtP2XpIbOFpHvser8FgMredSj+Q+glkngPZpFZDTW0dUG4BzWF6sRVi/LHUBNE76HZbwkIh2BPPa/bwIeWL3QAc4aY6Y7zFsQ66orf6xe675YPWJLAi8ZY9Y8rbijI6rbKCLpgNPAr8BRHl290gNrH79ijJn/1AKPAhHpC4zF+hx+jHVA5ug/Y8xae96Euv+itI0Jbv+5Ois95azuDryL1UvyAVZd5Sggtatji6XtawqssbfLD7iL1QfjQ8DL1fFFc1s2Yh2BOXtsdDJ/UWAJ1qV/94CtQB1Xb0dsbCNWI+xkrKqWG1g/npeBBUB5V29HBNs2NZJtC7cPE+j+i9I2JrT9l6TOFJRSSkUuKTU0K6WUegxNCkoppUJoUlBKKRVCk4JSSqkQmhSUUkqF0KSglFIqhCYFpZRSITQpKBWPiMhGETnj6jhU0qVJQSV6IlJDREwkjwBXx6hUfJGUbrKj1GysMa/CStBjXikVmzQpqKRkrzFmhquDUCo+0+ojpWwikteuThoiIq+IyAER8RORc3ZZuIMoESklIotFxMee94iIDLDv1Rt23mdE5DsROSUiD0TkioisFZG6TubNLiKzReSGiNwVkTUiUijMPF52XMdF5J6I3BSRgyIyMnbfGZWU6JmCSkpSioizO1w9NMb4Ovz/MtZN1ccB/wJNsG6ikgfoGjyTiJQFNmGNehk878vAcOA5oIPDvHmxhmjPijX2/m4gFdZQ53WAtQ6vnwrYDGzHGuE2H9b9tZeISAljTKA93zigm72+0VijAD8L1IryO6JUWK4eplUf+ojrB1CDyIc4Xm7Pl9f+PxAo47C8YN2DwwAVHcr/AAKAUmHmnWfPW9uhfKVdVt9JfG4Of2/EyQ2RgPfDLg9cB1a6+v3VR+J6aPWRSkomAXWdPD4KM99aY8ze4H+MMQYYYf/bHEBEsmDdbH2pMeZAmHmHhZk3A9AAWG2c3DDGhL+5UxDwXZiy9fbzsw5lt4DiIlIigu1VKtq0+kglJSeNMb9HYb6jTsqO2M/57ed89vPhCOYNcpi3INYZxL4oxnnJGOMXpszHfs7oUPYOMB04KCKnsO64twxY5iTRKBUleqagVHhRufOURGN9wfNG9Y5WgZFMC3ldY8wSrCqvjlhnErWxbvm4UUQ8ohGfUiE0KSgVXrFIyk6FeS7uZN4iWN+t4HlOYiWE52MrwGDGmOvGmBnGmJ5YZyYjgKpYt2ZVKto0KSgVXl0RKRP8j4gIMMD+91cAY8wV4E/gZcc6fXveQfa/i+15rwOrgIYiUifsi9nLRIuIuNs3hA9ht2cEV1FliO46lQJtU1BJSxkReTWCab86/P0XsF5ExmHdYL0p1mWj040x2xzmexvrktQt9rz/Ao2B+sAsY8w6h3nfwEoiq0RkGrAHSAFUAM4AH0RzW7yByyKyFCsRXMFq53gd6+bwy6K5PqUATQoqaXnFfjjzLNblpQBLgeNYR/yFsX5wP7cfIYwxu0WkMvA/oA9W/4JTWD/w34SZ97Tdr+FjoBHQCevH+y+sq6Ki6x4wBqsdoQ6QGiuBLQW+NMZcisE6lUKsM06llN3B7DTwP2PMENdGo5RraJuCUkqpEJoUlFJKhdCkoJRSKoS2KSillAqhZwpKKaVCaFJQSikVQpOCUkqpEJoUlFJKhdCkoJRSKsT/ARQtIh1ZXAy0AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from typing import Optional\n",
    "from pathlib import Path\n",
    "import torch\n",
    "\n",
    "from type_hints import ModelCheckpoint\n",
    "\n",
    "CHECKPOINT_PATH = Path(\"checkpoints\")\n",
    "#checkpoint: Optional[ModelCheckpoint] = torch.load(sorted((CHECKPOINT_PATH.glob('*.tar')), reverse=True)[0])\n",
    "checkpoint = torch.load(CHECKPOINT_PATH / \"checkpoint_28_epochs_5_min_freq_05_conf_threshold.tar\")\n",
    "#checkpoint = torch.load(CHECKPOINT_PATH / \"checkpoint_30_epochs.tar\")\n",
    "\n",
    "\n",
    "loss_train = checkpoint['train_loss_hist']\n",
    "loss_val = checkpoint['val_loss_hist']\n",
    "plt.rcParams.update({'font.size': 18})\n",
    "plt.plot(loss_train, 'g', label='Training loss')\n",
    "plt.plot(loss_val, 'b', label='Validation loss')\n",
    "plt.title('Training and Validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading train dataset\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root = '/mnt/data/datasets/LSA-T/data/cuts'\n",
    "max_frames = 75\n",
    "batch_size = 128\n",
    "keypoints_to_use = [i for i in range(94, 136)]\n",
    "words_min_freq = 5\n",
    "confidence_threshold = 0.5\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "CHECKPOINT_PATH = Path(\"checkpoints/\")\n",
    "CHECKPOINT_PATH.mkdir(exist_ok=True)\n",
    "\n",
    "#wandb.init(project=\"all-db-train\", entity=\"pedroodb\")\n",
    "\n",
    "keypoints_transform = Compose([\n",
    "    get_frames_reduction_transform(max_frames),\n",
    "    interpolate_keypoints\n",
    "])\n",
    "keypoints_transform_each = Compose([\n",
    "    get_keypoint_format_transform(keypoints_to_use),\n",
    "    keypoints_norm_to_nose\n",
    "])\n",
    "\n",
    "print(\"Loading train dataset\")\n",
    "train_dataset = LSA_Dataset(\n",
    "    root,\n",
    "    mode = \"train\",\n",
    "    words_min_freq = words_min_freq,\n",
    "    signer_confidence_threshold = confidence_threshold,\n",
    "    label_as_idx = False,\n",
    "    load_videos = False,\n",
    "    keypoints_transform = keypoints_transform,\n",
    "    keypoints_transform_each = keypoints_transform_each\n",
    ")\n",
    "\n",
    "checkpoint = torch.load(CHECKPOINT_PATH / \"checkpoint_28_epochs_5_min_freq_05_conf_threshold.tar\")\n",
    "model = KeypointModel(max_frames, train_dataset.max_label_len + 2, len(keypoints_to_use), len(train_dataset.vocab)).to(DEVICE)\n",
    "model.load_state_dict(checkpoint['model_state_dict'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Callable\n",
    "\n",
    "\n",
    "def edit_distance(prediction_tokens: list[str], reference_tokens: list[str], ignore_tokens: list[str] = []) -> int:\n",
    "    \"\"\"Standard dynamic programming algorithm to compute the edit distance adding a list of tokens to ignore\n",
    "    Args:\n",
    "        prediction_tokens: A tokenized predicted sentence\n",
    "        reference_tokens: A tokenized reference sentence\n",
    "        ignore_tokens: Tokens to ignore substitutions and insertions\n",
    "    Returns:\n",
    "        Edit distance between the predicted sentence and the reference sentence, ignoring insertions and substitutions of ignore_tokens\n",
    "    \"\"\"\n",
    "    dp = [[0] * (len(reference_tokens) + 1) for _ in range(len(prediction_tokens) + 1)]\n",
    "    for i in range(len(prediction_tokens) + 1):\n",
    "        dp[i][0] = i\n",
    "    for j in range(1, len(reference_tokens) + 1):\n",
    "        dp[0][j] = dp[0][j - 1] + (1 if reference_tokens[j - 1] not in ignore_tokens else 0)\n",
    "    for i in range(1, len(prediction_tokens) + 1):\n",
    "        for j in range(1, len(reference_tokens) + 1):\n",
    "            dp[i][j] = min(\n",
    "                dp[i - 1][j] + 1,\n",
    "                dp[i][j - 1] + (1 if reference_tokens[j - 1] not in ignore_tokens else 0),\n",
    "                dp[i - 1][j - 1] + (1 if prediction_tokens[i - 1] != reference_tokens[j - 1]\n",
    "                                        and reference_tokens[j - 1] not in ignore_tokens else 0)\n",
    "            )\n",
    "    return dp[-1][-1]\n",
    "\n",
    "def wer_n(prediction: str, target: str, ignore_tokens: list[str], tokenizer: Callable[[str], list[str]] = str.split) -> float:\n",
    "    return edit_distance(tokenizer(prediction), tokenizer(target), ignore_tokens) / len(tokenizer(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0/3767\n",
      "100/3767\n",
      "200/3767\n",
      "300/3767\n",
      "400/3767\n",
      "500/3767\n",
      "600/3767\n",
      "700/3767\n",
      "800/3767\n",
      "900/3767\n",
      "1000/3767\n",
      "1100/3767\n",
      "1200/3767\n",
      "1300/3767\n",
      "1400/3767\n",
      "1500/3767\n",
      "1600/3767\n",
      "1700/3767\n",
      "1800/3767\n",
      "1900/3767\n",
      "2000/3767\n",
      "2100/3767\n",
      "2200/3767\n",
      "2300/3767\n",
      "2400/3767\n",
      "2500/3767\n",
      "2600/3767\n",
      "2700/3767\n",
      "2800/3767\n",
      "2900/3767\n",
      "3000/3767\n",
      "3100/3767\n",
      "3200/3767\n",
      "3300/3767\n",
      "3400/3767\n",
      "3500/3767\n",
      "3600/3767\n",
      "3700/3767\n"
     ]
    }
   ],
   "source": [
    "dataset = train_dataset\n",
    "\n",
    "labels: list[str] = []\n",
    "preds: list[str] = []\n",
    "for i in range(0, dataset.__len__()):\n",
    "    if i % 100 == 0:\n",
    "        print(f\"{i}/{len(dataset)}\")\n",
    "    _, keys, label = dataset.__getitem__(i)\n",
    "    res = translate(model, keys, dataset, DEVICE)\n",
    "    preds.append(res)\n",
    "    labels.append(' '.join(label))\n",
    "    #print(i, res, ' '.join(label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11931\n",
      "[' ¿ cómo es el personas ? ', ' el comunidad . ', ' el comunidad . ', ' ¿ cómo es el personas ? ', ' ¿ cómo es el personas ? ']\n",
      "['a disfrutar el día todos los padres de argentina .', 'nos interesa continuar estudiando y capacitarnos en todo lo que conlleva', 'una tradición argentina que se realiza todos los años en el mes de marzo .', 'hay que seguir cuidándose como siempre .', 'quisimos adentrarnos en la temática , conocer distintos sabores y demás .']\n"
     ]
    }
   ],
   "source": [
    "preds, labels = [], []\n",
    "with open(\"preds/train/preds.txt\") as f:\n",
    "    for line in f:\n",
    "        preds.append(line[:-1])\n",
    "with open(\"preds/train/labels.txt\") as f:\n",
    "    for line in f:\n",
    "        labels.append(line[:-1])\n",
    "print(len(preds))\n",
    "print(preds[:5])\n",
    "print(labels[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9548790378180836\n",
      "0.9397863627884643\n",
      "0.8324705649744524\n",
      "0.7536014772170618\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "from statistics import mean\n",
    "\n",
    "freqs = Counter([word for label in labels for word in label.split(' ')])\n",
    "singletons = [word for (word, freq) in freqs.items() if freq <= 1]\n",
    "singletons5 = [word for (word, freq) in freqs.items() if freq <= 5]\n",
    "singletons10 = [word for (word, freq) in freqs.items() if freq <= 10]\n",
    "\n",
    "wer = mean([wer_n(preds[i], labels[i], []) for i in range(len(labels))])\n",
    "wer1 = mean([wer_n(preds[i], labels[i], singletons) for i in range(len(labels))])\n",
    "wer5 = mean([wer_n(preds[i], labels[i], singletons5) for i in range(len(labels))])\n",
    "wer10 = mean([wer_n(preds[i], labels[i], singletons10) for i in range(len(labels))])\n",
    "\n",
    "print(wer)\n",
    "print(wer1)\n",
    "print(wer5)\n",
    "print(wer10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "with open('preds/reduced/preds.txt','w') as preds_f:\n",
    "    preds_f.writelines(map(lambda p: p+'\\n', preds))\n",
    "with open('preds/reduced/labels.txt','w') as labels_f:\n",
    "    labels_f.writelines(map(lambda p: p+'\\n', labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from typing import Callable\n",
    "from torchmetrics.functional import word_error_rate, bleu_score\n",
    "\n",
    "def get_remove_singletons(singletons: list[str]) -> Callable[[str], str]:\n",
    "    def remove_singletons(label: str) -> str:\n",
    "        return ' '.join([word for word in label.split(' ') if word not in singletons])\n",
    "    return remove_singletons\n",
    "\n",
    "freqs = Counter([word for label in labels for word in label.split(' ')])\n",
    "singletons = [word for (word, freq) in freqs.items() if freq <= 10]\n",
    "remove_singletons = get_remove_singletons(singletons)\n",
    "\n",
    "clean_labels: list[str] = []\n",
    "diff_labels: list[tuple[int, str, str]] = []\n",
    "for i, label in enumerate(labels):\n",
    "    clean_label = remove_singletons(label)\n",
    "    clean_labels.append(clean_label)\n",
    "    if label != clean_label:\n",
    "        diff_labels.append((i, label, clean_label))\n",
    "clean_preds = list(map(get_remove_singletons(singletons), preds))\n",
    "\n",
    "no_sing_preds, no_sing_labels = zip(\n",
    "    *map(lambda i_l: (preds[i_l[0]], i_l[1]),\n",
    "    filter(lambda i_l: not any(w in singletons for w in i_l[1].split(' ')), enumerate(labels))))\n",
    "\n",
    "wer = word_error_rate(preds, labels)\n",
    "bleu = bleu_score(preds, labels)\n",
    "wer_no_sing = word_error_rate(preds, clean_labels)\n",
    "bleu_no_sing = bleu_score(preds, clean_labels)\n",
    "wer_no_sing_samples = word_error_rate(list(no_sing_preds), list(no_sing_labels))\n",
    "bleu_no_sing_samples = bleu_score(preds, clean_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3043/3767\n",
      "724/3767\n",
      "tensor(0.9312) tensor(0.)\n",
      "tensor(0.9150) tensor(0.)\n",
      "tensor(0.9187) tensor(0.)\n"
     ]
    }
   ],
   "source": [
    "print(f\"{len(diff_labels)}/{len(labels)}\")\n",
    "print(f\"{len(no_sing_labels)}/{len(labels)}\")\n",
    "print(wer, bleu)\n",
    "print(wer_no_sing, bleu_no_sing)\n",
    "print(wer_no_sing_samples, bleu_no_sing_samples)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c39aee66d4694ab6179e36ecdc8652752b93bf45b611fd43c1db6ceb5e4ba9d5"
  },
  "kernelspec": {
   "display_name": "Python 3.9.11 ('model')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}