03-neural-dekoniser-simple-rnn-mlp.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"11Sef4F0MBYfB1t45tSvI8Sp2tt-xEMkH","timestamp":1715096133162},{"file_id":"1s-o5axy2u3eFh6apgCS5U--gN3NzHFu8","timestamp":1715075632633}],"authorship_tag":"ABX9TyNnLRe5TBn5yYAw3Ya1s76H"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"zaPaHzVAPWMS","executionInfo":{"status":"ok","timestamp":1715096177119,"user_tz":-330,"elapsed":6509,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}}},"outputs":[],"source":["import math\n","from torch import nn, Tensor\n","import torch"]},{"cell_type":"code","source":["vocab = ['a','b','c']\n","\n","words = []\n","for i in vocab:\n","  for j in vocab:\n","    for k in vocab:\n","      words.append(i+j+k)\n","\n","\n","print('words',words)\n","vocab.append('0')\n","print('vocab',vocab)\n","\n","letter_to_index = {letter: index for index, letter in enumerate(vocab)}\n","index_to_letter = {index: letter for index, letter in enumerate(vocab)}\n","\n","word_to_index = {word: index for index, word in enumerate(words)}\n","index_to_word = {index: word for index, word in enumerate(words)}\n","\n","\n","\n","#print(letter_to_index)\n","#print(index_to_letter)\n","\n","#print(word_to_index)\n","#print(index_to_word)\n","\n","N = len(words)\n","# includes special end of word character\n","V = len(vocab)\n","# includes end of word character\n","L = max(len(word) for word in words)+1\n","\n","print('# of words',N)\n","print('# of chars',V)\n","print('seq len', L)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"esgvDr19Pyu4","executionInfo":{"status":"ok","timestamp":1715096264764,"user_tz":-330,"elapsed":601,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"0b145691-7902-44dc-bd88-a472f5d2582a"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["words ['aaa', 'aab', 'aac', 'aba', 'abb', 'abc', 'aca', 'acb', 'acc', 'baa', 'bab', 'bac', 'bba', 'bbb', 'bbc', 'bca', 'bcb', 'bcc', 'caa', 'cab', 'cac', 'cba', 'cbb', 'cbc', 'cca', 'ccb', 'ccc']\n","vocab ['a', 'b', 'c', '0']\n","# of words 27\n","# of chars 4\n","seq len 4\n"]}]},{"cell_type":"code","source":["# Define the dimensions\n","# Prepare training set\n","X_train = []\n","Y_train = []\n","\n","for word in words:\n","\n","    chars = list(word)\n","    x = torch.zeros(1, L, N)\n","    y = torch.zeros(1, L)\n","\n","    n = len(chars)\n","    for i in range(L):\n","        x[0, i, word_to_index[word]] = 1\n","        if i < n:\n","            y[0,i] = letter_to_index[chars[i]]\n","        else:\n","            y[0,i] = V-1\n","\n","    X_train.append(x)\n","    Y_train.append(y)\n","\n","# Convert the training set to tensors\n","X_train = torch.cat(X_train, dim=0)\n","Y_train = torch.cat(Y_train, dim=0)\n","\n","# Print the shape of the training set\n","print(\"X_train shape:\", X_train.shape)\n","print(\"Y_train shape:\", Y_train.shape)\n","# prepare training set\n","\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jV1wAqYjPaGT","executionInfo":{"status":"ok","timestamp":1715096269603,"user_tz":-330,"elapsed":4,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"f71b6c7d-c493-480b-aea2-869327015b02"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["X_train shape: torch.Size([27, 4, 27])\n","Y_train shape: torch.Size([27, 4])\n"]}]},{"cell_type":"code","source":["# https://github.com/KasperGroesLudvigsen/influenza_transformer/blob/main/positional_encoder.py\n","\n","class MLP(nn.Module):\n","    def __init__(self, input_size, hidden_sizes, output_size):\n","        super(MLP, self).__init__()\n","        self.input_size = input_size\n","        self.hidden_sizes = hidden_sizes\n","        self.output_size = output_size\n","\n","        layers = []\n","        prev_size = input_size\n","        for size in hidden_sizes:\n","            layers.append(nn.Linear(prev_size, size))\n","            layers.append(nn.ReLU())\n","            prev_size = size\n","        layers.append(nn.Linear(prev_size, output_size))\n","\n","        self.model = nn.Sequential(*layers)\n","\n","    def forward(self, x):\n","        return self.model(x)\n","\n","class PositionalEncoder(nn.Module):\n","    \"\"\"\n","    The authors of the original transformer paper describe very succinctly what\n","    the positional encoding layer does and why it is needed:\n","\n","    \"Since our model contains no recurrence and no convolution, in order for the\n","    model to make use of the order of the sequence, we must inject some\n","    information about the relative or absolute position of the tokens in the\n","    sequence.\" (Vaswani et al, 2017)\n","    Adapted from:\n","    https://pytorch.org/tutorials/beginner/transformer_tutorial.html\n","    \"\"\"\n","\n","    def __init__(\n","        self,\n","        dropout: float=0.1,\n","        max_seq_len: int=100,\n","        d_model: int=512,\n","        batch_first: bool=True\n","        ):\n","\n","        \"\"\"\n","        Parameters:\n","            dropout: the dropout rate\n","            max_seq_len: the maximum length of the input sequences\n","            d_model: The dimension of the output of sub-layers in the model\n","                     (Vaswani et al, 2017)\n","        \"\"\"\n","\n","        super().__init__()\n","\n","        self.d_model = d_model\n","\n","        self.dropout = nn.Dropout(p=dropout)\n","\n","        self.batch_first = batch_first\n","\n","        # adapted from PyTorch tutorial\n","        position = torch.arange(max_seq_len).unsqueeze(1)\n","\n","\n","        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n","\n","\n","        if self.batch_first:\n","\n","            pe = torch.zeros(1, max_seq_len, d_model)\n","\n","            pe[0, :, 0::2] = torch.sin(position * div_term)\n","\n","            pe[0, :, 1::2] = torch.cos(position * div_term)[:,:(d_model//2)]\n","\n","\n","        else:\n","            pe = torch.zeros(max_seq_len, 1, d_model)\n","\n","            pe[:, 0, 0::2] = torch.sin(position * div_term)\n","\n","            pe[:, 0, 1::2] = torch.cos(position * div_term)\n","\n","        self.register_buffer('pe', pe)\n","\n","    def forward(self, x: Tensor) -> Tensor:\n","        \"\"\"\n","        Args:\n","            x: Tensor, shape [batch_size, enc_seq_len, dim_val] or\n","               [enc_seq_len, batch_size, dim_val]\n","        \"\"\"\n","        if self.batch_first:\n","            x = x + self.pe[:,:x.size(1)]\n","        else:\n","            x = x + self.pe[:x.size(0)]\n","\n","        return self.dropout(x)\n","\n","class myRNN(nn.Module):\n","    def __init__(self, input_size, hidden_size, num_classes, num_layers=1, max_seq_len=L, MLP_hidden_sizes=None):\n","        super(myRNN, self).__init__()\n","        self.input_size = input_size\n","        self.pos_encoder = PositionalEncoder(d_model = input_size, dropout=0.1, max_seq_len=max_seq_len)\n","        self.hidden_size = hidden_size\n","        self.rnn = nn.RNN(input_size, hidden_size, num_layers,batch_first=True)\n","        self.mlp = nn.Linear(hidden_size, num_classes)\n","\n","        if MLP_hidden_sizes is not None:\n","          # assume MLP_hidden_sizes is a list\n","          self.mlp = MLP(hidden_size, MLP_hidden_sizes, num_classes)\n","\n","\n","\n","    def forward(self, x):\n","        # Initialize hidden state with zeros\n","        # h0 = torch.zeros(1, x.size(0), self.hidden_size).to(x.device)\n","\n","        #h0 can be improved (h0 = x*toen_embedding)\n","\n","        # Forward pass through RNN\n","        #out, _ = self.rnn(x, h0)\n","        x = x * math.sqrt(self.input_size)\n","        x = self.pos_encoder(x)\n","        out, _ = self.rnn(x)\n","\n","        # Concatenate the output of RNN with y\n","        #out = torch.cat((out[:, -1, :], y.unsqueeze(1)), dim=1)\n","        # out = out[:, -1, :]\n","\n","        # Pass the concatenated output through the fully connected layer\n","        out = self.mlp(out)\n","\n","        return out"],"metadata":{"id":"ZrOoXs-bPxEv","executionInfo":{"status":"ok","timestamp":1715096905255,"user_tz":-330,"elapsed":816,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["input_size = N\n","hidden_size = 5*V\n","num_classes = V\n","num_layers = 3\n","num_epochs = 5000\n","batch_size = 15\n","MLP_hidden_sizes = [5*V,5*V]\n","lr = 0.001\n","\n","# Create an instance of the RNN model\n","model = myRNN(input_size, hidden_size, num_classes, num_layers=num_layers, max_seq_len=L,MLP_hidden_sizes=MLP_hidden_sizes)\n","\n","# Print the model architecture\n","print(model)\n","\n","# Define the loss function and optimizer\n","\n","criterion = nn.CrossEntropyLoss()\n","optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n","\n","\n","# Train the model\n","\n","\n","loss_acc = []\n","for epoch in range(num_epochs):\n","    # Shuffle the training data\n","    indices = torch.randperm(X_train.size(0))\n","    X_train_shuffled = X_train[indices]\n","    Y_train_shuffled = Y_train[indices]\n","\n","    # Split the training data into batches\n","    num_batches = X_train.size(0) // batch_size\n","    for batch_idx in range(num_batches):\n","        # Get the batch inputs and targets\n","        start_idx = batch_idx * batch_size\n","        end_idx = start_idx + batch_size\n","        batch_inputs = X_train_shuffled[start_idx:end_idx]\n","        batch_targets = Y_train_shuffled[start_idx:end_idx]\n","\n","        # Forward pass\n","        outputs = model(batch_inputs)\n","\n","        # Compute the loss\n","        loss = criterion(outputs.view(-1, num_classes), batch_targets.view(-1).long())\n","\n","        # Backward pass and optimization\n","        optimizer.zero_grad()\n","        loss.backward()\n","        optimizer.step()\n","\n","    # Print the loss for this epoch\n","    #print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item()}\")\n","    loss_acc.append(loss.item())\n","\n","import matplotlib.pyplot as plt\n","plt.plot(loss_acc)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"Jx-QEMVQfqlh","executionInfo":{"status":"ok","timestamp":1715097069348,"user_tz":-330,"elapsed":27669,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"cc336e2f-0982-409d-a222-97ec4332fe5e"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["myRNN(\n","  (pos_encoder): PositionalEncoder(\n","    (dropout): Dropout(p=0.1, inplace=False)\n","  )\n","  (rnn): RNN(27, 20, num_layers=3, batch_first=True)\n","  (mlp): MLP(\n","    (model): Sequential(\n","      (0): Linear(in_features=20, out_features=20, bias=True)\n","      (1): ReLU()\n","      (2): Linear(in_features=20, out_features=20, bias=True)\n","      (3): ReLU()\n","      (4): Linear(in_features=20, out_features=4, bias=True)\n","    )\n","  )\n",")\n","aaa aaa True\n","aab aab True\n","aac aac True\n","aba aba True\n","abb abb True\n","abc abc True\n","aca aca True\n","acb acb True\n","acc acc True\n","baa baa True\n","bab bab True\n","bac bac True\n","bba bba True\n","bbb bbb True\n","bbc bbc True\n","bca bca True\n","bcb bcb True\n","bcc bcc True\n","caa caa True\n","cab cab True\n","cac cac True\n","cba cba True\n","cbb cbb True\n","cbc cbc True\n","cca cca True\n","ccb ccb True\n","ccc ccc True\n","100.0\n"]},{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRc0lEQVR4nO3deXhTdd428DttaUqFtiC0ZSmrCCJSNqlVURmriAyOM87IozzCMIqvCPPg1HFBBdyG4gKDC4uiCDoiCCPgwiIWCgIVaGmBspStpaV0gy7pviS/94+2IWmTNCc5OSdp7s915YKenOWbk+Scb36rRgghQERERKQSH7UDICIiIu/GZISIiIhUxWSEiIiIVMVkhIiIiFTFZISIiIhUxWSEiIiIVMVkhIiIiFTFZISIiIhU5ad2APYwGAy4fPkyOnbsCI1Go3Y4REREZAchBMrKytC9e3f4+Fgv//CIZOTy5cuIiIhQOwwiIiJyQHZ2Nnr27Gn1eY9IRjp27Aig4cUEBQWpHA0RERHZQ6fTISIiwngft8YjkpGmqpmgoCAmI0RERB6mtSYWbMBKREREqmIyQkRERKpiMkJERESqYjJCREREqmIyQkRERKpiMkJERESqYjJCREREqmIyQkRERKpiMkJERESqYjJCREREqmIyQkRERKqSnIzs3bsXEydORPfu3aHRaLB582a7t92/fz/8/PwwbNgwqYclIiKiNkpyMlJRUYHIyEgsXbpU0nYlJSWYMmUK7r33XqmHdJnP92Vg3pY0nMkvUzsUIiIiryV51t7x48dj/Pjxkg/0zDPP4PHHH4evr6+k0hRX+vHYZaRkleDOG7rgxjDb0xsTERGRayjSZuSLL77AhQsXMH/+fLvWr6mpgU6nM3u4QoCfLwCgqk7vkv0TERFR61yejJw9exYvv/wy/vOf/8DPz76CmLi4OAQHBxsfERERLokt8cJVAMDsdaku2T8RERG1zqXJiF6vx+OPP4433ngDN954o93bzZkzB6WlpcZHdna2C6MkIiIiNUluMyJFWVkZkpKSkJKSglmzZgEADAYDhBDw8/PDzz//jN/97nctttNqtdBqta4MrQW9QcDXR6PoMYmIiMjFyUhQUBCOHz9utmzZsmXYtWsXNm7ciL59+7ry8JJsTM7GpFt7qR0GERGR15GcjJSXl+PcuXPGvzMyMpCamorOnTujV69emDNnDnJycvDll1/Cx8cHQ4YMMds+NDQUAQEBLZarLTW7hMkIERGRCiQnI0lJSRg7dqzx79jYWADA1KlTsXr1auTm5iIrK0u+CBVSrxdqh0BEROSVNEIIt78L63Q6BAcHo7S0FEFBQbLt99ezhXji80MAgI4Bfjj++jjZ9k1EROTt7L1/e/XcNHf072L8f1l1vYqREBEReS+vTkZ82HuGiIhIdV6djBAREZH6mIwQERGRqrw+Gfn48eFqh0BEROTVvD4Zua3f9cb/6w1u37GIiIiozfH6ZCTQ39f4/21puSpGQkRE5J28PhkJ8LuWjOQUV6kYCRERkXfy+mTEtHtvZa1exUiIiIi8k9cnI6Y+iD+rdghEREReh8mIiY5al05iTERERBYwGQEw457+AIC7B3ZVORIiIiLvw2QEQP+uHQAAOs5PQ0REpDgmI2iYsRcAyqrrVI6EiIjI+zAZARDcvh0AoLiiVuVIiIiIvA+TEQBdO2oBAFfLmYwQEREpjckIAK1fw2mo0RtUjoSIiMj7MBkBoG0chbW23gAhOD8NERGRkpiMANC2u3Yaalk6QkREpCgmI7hWTQMANfVMRoiIiJTEZASAv++10/DrmSsqRkJEROR9mIwA0GiuTZbXIYBDwhMRESmJyUijgWEdAYANWImIiBTGZKRR0yis1XV6lSMhIiLyLkxGGrX3b+jeW8VkhIiISFFMRhoFtGtMRmrZm4aIiEhJTEYaBTaWjFTWcuZeIiIiJTEZadS+sWSEbUaIiIiUxWSkkbGahskIERGRopiMNDI2YGWbESIiIkUxGWkUaCwZYZsRIiIiJTEZaXStZITVNEREREpiMtLIv3GyPE6UR0REpCwmI42aZu6tZTJCRESkKCYjjZpKRmr1TEaIiIiUxGSkkb9vQ5uRmjomI0REREpiMtKoqZqmhiUjREREipKcjOzduxcTJ05E9+7dodFosHnzZpvrf/fdd7jvvvvQtWtXBAUFITo6Gjt27HA0XpfxZ5sRIiIiVUhORioqKhAZGYmlS5fatf7evXtx3333YevWrUhOTsbYsWMxceJEpKSkSA7Wla71pmHXXiIiIiX5Sd1g/PjxGD9+vN3rL1myxOzvBQsWYMuWLfjhhx8wfPhwqYd3maZqmooaDnpGRESkJMnJiLMMBgPKysrQuXNnq+vU1NSgpqbG+LdOp3N5XE0lI/m6GlTV6o2DoBEREZFrKd6A9f3330d5eTkeffRRq+vExcUhODjY+IiIiHB5XP6+105FSlaxy49HREREDRRNRtauXYs33ngD3377LUJDQ62uN2fOHJSWlhof2dnZLo9No7n2f+HyoxEREVETxapp1q1bh6eeegobNmxATEyMzXW1Wi20Wq1CkTW5lo2YJiZERETkWoqUjHzzzTeYNm0avvnmG0yYMEGJQ0oWGnQt+fFhNkJERKQYySUj5eXlOHfunPHvjIwMpKamonPnzujVqxfmzJmDnJwcfPnllwAaqmamTp2KDz74AFFRUcjLywMAtG/fHsHBwTK9DOcFBbQz/l9vYEUNERGRUiSXjCQlJWH48OHGbrmxsbEYPnw45s2bBwDIzc1FVlaWcf1PP/0U9fX1mDlzJrp162Z8zJ49W6aXIJ/B3YIAAPVMRoiIiBQjuWTknnvugRDWb9arV682+zshIUHqIVTTzreheqaeQ8ITEREphnPTmPBr7N5bp2fJCBERkVKYjJjw82ksGTGwZISIiEgpTEZM+DVW07ABKxERkXKYjJjw82E1DRERkdKYjJhgA1YiIiLlMRkx4WtsM8KSESIiIqUwGTHR1JuGJSNERETKYTJioh1LRoiIiBTHZMSEb2MDViYjREREymEyYqJpnBF27SUiIlIOkxETO042TOL33o50lSMhIiLyHkxGTJRU1qkdAhERkddhMkJERESqYjJCREREqmIyQkRERKpiMmJi+pi+aodARETkdZiMmJgS3QcAEOjvq24gREREXoTJiAm/xony6jgcPBERkWKYjJho1zg3TZ1eQAgOfEZERKQEJiMmmpIRgEPCExERKYXJiIl2jdU0AKtqiIiIlMJkxIRpyUidniUjRERESmAyYqJpojyAJSNERERKYTJiQqPRGKtqmIwQEREpg8lIM01VNfWspiEiIlIEk5FmmqpqalkyQkREpAgmI834+zWNNcJkhIiISAlMRpphNQ0REZGymIw00zQkPKtpiIiIlMFkpBnjkPD1TEaIiIiUwGSkGf+mahoOB09ERKQIJiPNZBVVAgBO55WpHAkREZF3YDLSTGWtHgDw1o8nVY6EiIjIOzAZISIiIlUxGSEiIiJVMRlpZkBoBwDAPQO7qhwJERGRd5CcjOzduxcTJ05E9+7dodFosHnz5la3SUhIwIgRI6DVanHDDTdg9erVDoSqjEdG9gQAXH+dVuVIiIiIvIPkZKSiogKRkZFYunSpXetnZGRgwoQJGDt2LFJTU/Hcc8/hqaeewo4dOyQHq4SmuWk4HDwREZEy/KRuMH78eIwfP97u9VesWIG+ffti0aJFAICbbroJ+/btw7///W+MGzdO6uFdrmlumnoDkxEiIiIluLzNSGJiImJiYsyWjRs3DomJia4+tEP8fJomyuOgZ0REREqQXDIiVV5eHsLCwsyWhYWFQafToaqqCu3bt2+xTU1NDWpqaox/63Q6V4dp1DQ3zc6T+Yodk4iIyJu5ZW+auLg4BAcHGx8RERGKHfvghSLFjkVEREQKJCPh4eHIzzcvZcjPz0dQUJDFUhEAmDNnDkpLS42P7OxsV4dpVFRR0/pKREREJBuXV9NER0dj69atZst27tyJ6Ohoq9totVpotep0rfVt7E1DREREypBcMlJeXo7U1FSkpqYCaOi6m5qaiqysLAANpRpTpkwxrv/MM8/gwoULePHFF3H69GksW7YM3377Lf7xj3/I8wpk5qNhMkJERKQkyclIUlIShg8fjuHDhwMAYmNjMXz4cMybNw8AkJuba0xMAKBv37746aefsHPnTkRGRmLRokX47LPP3LJbLwDc2qez2iEQERF5FY0Qwu37sOp0OgQHB6O0tBRBQUEuPVZeaTVui4sHAGQunODSYxEREbVl9t6/3bI3jZqauvYCgAfkaURERB6PyUgzfiYNWA3MRYiIiFyOyUgzPibJCIeEJyIicj0mI82YloyczS9XMRIiIiLvwGSkGdNxRn48lqtiJERERN6ByUgzTRPlAUDHAJePCUdEROT1mIw0YzoAa+fr/NULhIiIyEswGWlGYzIC65zvjqsYCRERkXdgMkJERESqYjJCREREqmIyQkRERKpiMkJERESqYjJCREREqmIyQkRERKpiMkJERESqYjJCREREqmIyYkF0v+vVDoGIiMhrMBmx4ObuQcb/CyFUjISIiKjtYzJigWn6oTcwGSEiInIlJiMWDArvaPx/nZ7JCBERkSsxGbHggSHhxv/vPJWvYiRERERtH5MRC9r5XjstxRW1KkZCRETU9jEZscDXR2P8//zvT6gYCRERUdvHZMQCP5NkhIiIiFyLyYgFGg2TESIiIqUwGSEiIiJVMRmxYlTvTmqHQERE5BWYjFjRMcDP+H8OfEZEROQ6TEasuKVniPH/v124ql4gREREbRyTESuevae/8f+TPzuoYiRERERtG5MRKwLa+Zr9fexSiTqBEBERtXFMRuz00Mf71Q6BiIioTWIyQkRERKpiMmLDN9NvM/v7UnGlSpEQERG1XUxGbIjuf73Z33e+sxu19QaVoiEiImqbmIxIVF5Tr3YIREREbQqTEYk2JGWrHQIREVGb4lAysnTpUvTp0wcBAQGIiorCoUOHbK6/ZMkSDBw4EO3bt0dERAT+8Y9/oLq62qGAlTaiV4jZ33HbTuNCYbk6wRAREbVBkpOR9evXIzY2FvPnz8eRI0cQGRmJcePGoaCgwOL6a9euxcsvv4z58+fj1KlT+Pzzz7F+/Xq88sorTgevhDceGtJi2eUSz0ikiIiIPIHkZGTx4sWYPn06pk2bhsGDB2PFihUIDAzEqlWrLK5/4MAB3HHHHXj88cfRp08f3H///XjsscdaLU1xFwPDO7ZYVm9gI1YiIiK5SEpGamtrkZycjJiYmGs78PFBTEwMEhMTLW5z++23Izk52Zh8XLhwAVu3bsWDDz5o9Tg1NTXQ6XRmD7X4+7U8RX/94jBe2HBUhWiIiIjaHknJyJUrV6DX6xEWFma2PCwsDHl5eRa3efzxx/Hmm2/izjvvRLt27dC/f3/cc889Nqtp4uLiEBwcbHxERERICVMRG5IvoU7PEhIiIiJnubw3TUJCAhYsWIBly5bhyJEj+O677/DTTz/hrbfesrrNnDlzUFpaanxkZ6vbg0Wjsby8Xi+UDYSIiKgN8pOycpcuXeDr64v8/Hyz5fn5+QgPD7e4zdy5c/HEE0/gqaeeAgDccsstqKiowNNPP41XX30VPj4t8yGtVgutVislNJfqHOiPqxW1LZbPWnsEn//1VhUiIiIiajsklYz4+/tj5MiRiI+PNy4zGAyIj49HdHS0xW0qKytbJBy+vg0z4grhGSULq6wkHPGnLfcgIiIiIvtJrqaJjY3FypUrsWbNGpw6dQozZsxARUUFpk2bBgCYMmUK5syZY1x/4sSJWL58OdatW4eMjAzs3LkTc+fOxcSJE41JibuLjAjBf2dYTraIiIjIOZKqaQBg0qRJKCwsxLx585CXl4dhw4Zh+/btxkatWVlZZiUhr732GjQaDV577TXk5OSga9eumDhxIv71r3/J9yoUENkzRO0QiIiI2iSN8IC6Ep1Oh+DgYJSWliIoKEi1OPq8/FOLZZkLJ6gQCRERkfuz9/7NuWkkePMPN6sdAhERUZvDZESCKdF9MLpPZ7Nlp/PUG5CNiIioLWAyItHa6VFmfz+w5FeVIiEiImobmIxI5OfLU0ZERCQn3lllYDC4fRtgIiIit8VkRAZ3vbcbWVcr1Q6DiIjIIzEZkcGl4ir837oUtcMgIiLySExGZJKaXaJ2CERERB6JyYgD/jCsu9ohEBERtRlMRhzwrz/eonYIREREbQaTEQd00Eqe0oeIiIisYDJCREREqmIy4qClj49QOwQiIqI2gcmIgyYM7aZ2CERERG0CkxEZ1esNaodARETkcZiMyGjtoSy1QyAiIvI4TEZklHj+qtohEBEReRwmI05495GhZn9zRl8iIiLpePd0wl9G9TT7e9/ZQpUiISIi8lxMRpyg0WjM/i6urMOH8WdhMAiVIiIiIvI8TEZktnjnGfxw7LLaYRAREXkMJiMukF1UqXYIREREHoPJiJOevLNvi2Ubky+pEAkREZFnYjLiJGGheUjmVZaMEBER2YvJiJPuurGL2iEQERF5NCYjTrr7xq5qh0BEROTRmIw4qXn3XiIiIpKGyQgRERGpiskIERERqYrJiAx6Xx+odghEREQei8mIDF598Ca1QyAiIvJYTEZkEHNTmNohEBEReSwmIzLw8WGPGiIiIkcxGSEiIiJVMRlxkdKqOrVDICIi8ghMRmQSHhRg9veRi8UqRUJERORZmIzI5Nmx/dUOgYiIyCM5lIwsXboUffr0QUBAAKKionDo0CGb65eUlGDmzJno1q0btFotbrzxRmzdutWhgN1VO1/mdURERI7wk7rB+vXrERsbixUrViAqKgpLlizBuHHjkJ6ejtDQ0Bbr19bW4r777kNoaCg2btyIHj164OLFiwgJCZEjfrdx4PxVtUMgIiLySJJ/zi9evBjTp0/HtGnTMHjwYKxYsQKBgYFYtWqVxfVXrVqFoqIibN68GXfccQf69OmDu+++G5GRkU4H707GDuTsvURERI6QlIzU1tYiOTkZMTEx13bg44OYmBgkJiZa3Ob7779HdHQ0Zs6cibCwMAwZMgQLFiyAXq+3epyamhrodDqzh7vrFtze7G8BoVIkREREnkVSMnLlyhXo9XqEhZmPOBoWFoa8vDyL21y4cAEbN26EXq/H1q1bMXfuXCxatAhvv/221ePExcUhODjY+IiIiJASpio0HPeMiIjIIS5vdWkwGBAaGopPP/0UI0eOxKRJk/Dqq69ixYoVVreZM2cOSktLjY/s7GxXh+k05iJERESOkdSAtUuXLvD19UV+fr7Z8vz8fISHh1vcplu3bmjXrh18fX2Ny2666Sbk5eWhtrYW/v7+LbbRarXQarVSQlNd8yHhj2aX4neDOGcNERFRaySVjPj7+2PkyJGIj483LjMYDIiPj0d0dLTFbe644w6cO3cOBoPBuOzMmTPo1q2bxUTEUzUvGfkg/qwqcRAREXkaydU0sbGxWLlyJdasWYNTp05hxowZqKiowLRp0wAAU6ZMwZw5c4zrz5gxA0VFRZg9ezbOnDmDn376CQsWLMDMmTPlexVugJPlEREROUbyOCOTJk1CYWEh5s2bh7y8PAwbNgzbt283NmrNysqCj8+1HCciIgI7duzAP/7xDwwdOhQ9evTA7Nmz8dJLL8n3KtxAZM8QtUMgIiLySBohhNv3QdXpdAgODkZpaSmCgoLUDseqxTvP4EOT6pnMhRNUjIaIiEhd9t6/OYa5jHzZv5eIiEgyJiMyCvT3Nft75d4L2Jh8CVELfsHG5EsqRUVEROTeJLcZIeva+ZqXjPxr6ynj//+54Sj+PLKn0iERERG5PZaMyEjjQDXNvC1pmPRJIur1htZXJiIiaoOYjMioT5frJG/zZeJFHMwowm8XilwQERERkftjMiKjuwZ0cXhbvft3aiIiInIJJiMycqSahoiIyNsxGSEiIiJVMRkhIiIiVTEZISIiIlUxGSEiIiJVMRkhIiIiVTEZISIiIlUxGZFZ/67SBz4jIiLyZkxGZHZbv+vVDoGIiMijMBmR2ex7B6gdAhERkUdhMiKz0KAAq89lXKlQMBIiIiLPwGREQWPfT1A7BCIiIrfDZERh5wrK1A6BiIjIrTAZUVh1nUHtEIiIiNwKkxEiIiJSFZMRIifpquuwO70AdXqWehEROYLJSBuTdbUS6w9n8caooCc+O4hpXxzGx7vOqR0KEZFHYjLiAnPGD1Lt2He9txsv/fc4Vu/PVC0Gb3P0UikAYFNKjsqREBF5JiYjLvCnET3VDgEHM4rUDoGIiMguTEZcoGtHreRtNLJHIWTfIxERkSswGXETbT11qNcbsHp/BtLzOM4KERGZ81M7APIOXx/Mwus/nAQAZC6coHI0RETkTlgyQoo4eqlE7RCIiMhNMRlRWPLFYrVDICIicitMRhQ2//sTaoegCo0LmugSEVHbwGSESCaizTdDJnKtK+U1iF2fisOZHJrA2zAZaaNEG7wvFuiqkV1UKXk7IQT+/k0K/vXTSRdERURymbclDd+l5OAvKxLVDoUUxmTEDQkhUF5Tr3YYstLIUEszekE8xry7G6WVdZK2O5Vbhh+OXsbKXzOcD8IGVkUROSfzivQfG9Q2MBlRwcWrFS2Wmd7Gpn+ZjCHzd+BcQbnN/egNAn9bfRiLfk6XOUL3ll3c+gVLbxDYdjwXeaXVnKeHiMjNMRlRwd3vJbRYZlqr8supfADAf367aHM/e88WYtfpAnzkARO0KV1m8PXBi5jx9RGMfT9B4SOTNTX1evw3+RIKdNVqh0JEbobJiAerqXP+F3+d3oDki8Wob2OlB3vSCwEAVXV6xZqVsgGrbUt+OYvnNxzFH5buVzsUInIzDiUjS5cuRZ8+fRAQEICoqCgcOnTIru3WrVsHjUaDhx9+2JHDeqXCshpsSMpGdZ3ewrPO3/xe3XQcjyw/gLd/OuX0vohs+eVkQ4lfbilLRojInORkZP369YiNjcX8+fNx5MgRREZGYty4cSgoKLC5XWZmJv75z39izJgxDgfrSSJ7Bkta31I1hhACjyw/gBc2HsPCbaflCayZb5MuAQBWH8h0yf7dgVJVRGzASkTkGMnJyOLFizF9+nRMmzYNgwcPxooVKxAYGIhVq1ZZ3Uav12Py5Ml444030K9fP6cC9hSfPDFK0vqWyjgEgKzGrqw7G39VmuPNzx6sPCEicm+SkpHa2lokJycjJibm2g58fBATE4PEROv9wt98802EhobiySeftOs4NTU10Ol0Zg9PEx4coMBRPOc2K0fXXnc+HsA2I0TO4jfIe0lKRq5cuQK9Xo+wsDCz5WFhYcjLy7O4zb59+/D5559j5cqVdh8nLi4OwcHBxkdERISUMAn8UhMRkedwaW+asrIyPPHEE1i5ciW6dOli93Zz5sxBaWmp8ZGdne3CKN2D5TYjiofhkF2n8xGzeA+O2ZiZV832FGwzQuQZ+A3yXn5SVu7SpQt8fX2Rn2/efiE/Px/h4eEt1j9//jwyMzMxceJE4zKDoaELqZ+fH9LT09G/f/8W22m1Wmi1WimheRwhBOoNHpJttOJvq5Ma/z2MpNfuUzmaltrGWZaf3iBwOk+Hm8KD4OPD2wARqUdSyYi/vz9GjhyJ+Ph44zKDwYD4+HhER0e3WH/QoEE4fvw4UlNTjY+HHnoIY8eORWpqqldXv8R+exQ3z9th/NtyA1bPuo1W1FjqfqwW97+5rjmQiflb0iBUKgJ7/fsTmPDhPryzwzU9tZrzrE8zqYGfEe8lqWQEAGJjYzF16lSMGjUKo0ePxpIlS1BRUYFp06YBAKZMmYIePXogLi4OAQEBGDJkiNn2ISEhANBiubfZlJLj0v2rdYOzxtUNSnedbiit+92gpvZMyr9+qcnj/O9PAADuvzkcd9xgfzWmXL5qHOH3kz0XMGf8TYofn4ioieRkZNKkSSgsLMS8efOQl5eHYcOGYfv27cZGrVlZWfDx4cCuUnlymxG1VdbWG6uKTrwxDtdpJX+sVTX5s4N4bHQE4v40VO1QXMr9y6pIbfyMeC+HrtqzZs3CrFmzLD6XkJBgc9vVq1c7ckiPtHZ6FB5fedDh7ZmLXGOrpKfaZFj8qjp9YzKi/GXNmQas3xzKbvPJCBGRNSzCcKHb+3fBLT3sG4m1rSceclbTNN+XadtLA4uT3BbfGSKyhsmIG1t7MMv4/8raevx75xmk55WpGJFtajW4NSuRsBCCUu1nPK3BMRGRu2Ay4mL2/lJvreCguLIOH8Sfxbgle50PysO1OKW2cxGvU+emMzCzPQC15mSu5422TfJgMuJi/bp2UDsEl9ArPEaKrZyutWoajRpjwztp6e5z+CoxU/J22UWVGDxvO17bfFz+oIhc6HQeExFvxmTExeZPHGzXek230Oo6+8fqsHWDdmWqsCU1B4PmbjNOCW8fywnBukNZ+GTPeUnHb55bmCYbajYZad6A1WAQ+HxfBpIvFkvaT3ZRJd7bkY65W05IjmHFnvOo0wv857es1lcmciNFFbVqh0AqYjLiYl06SBtJdoXEG7PS6vUGzF6Xijq9wFNfJpk9Z6s3ibXCiZe/O464baeRdbVSlviachHT46k15srWtFy89eNJPLL8gKTtKmrrjf93t/FiiFyF0yl4NyYjbubkZfuLKpWuffhifwZufG2b1eedacBZVlNn97q27s8GFYfYb/76zxWUO7QfZy7KHlgjRQSAn11vx2TETTR9D6V8IW3dlEur6lBVK+/w7G/8cBKuute3VgBg6+nWSg88sc1IExaMEJE3YDLiZuQqqkzJKsHQN3a0vqKJ7CJ5qkpczVZuYenm7WhVx8WrFZLa8DTHYmci+/Hb4t2YjLgJV/wArtNL2+sfl0lr2yCFUheapt40zh7vcGYR7n4vAb//aJ/d27gi+ZD6uVA6Adp39gpe//6EXUkbC3mIyBomIwr4v9/dYPe6arYov1Jeo9qxW2OrdMP0mdbGdbG3lGRz40SGUtp9yDXomTs0vrXX/35+EKsPZOLzfRlqh0JEHozJiAJ8fFr/tapBw6/xQ5lFrg+oDXPvW3fbdam49So+FsOTLZ7ctoucx2REAfb+uP1iv2f/ujSdsK45pa4zbl6QIJnkapo2ej3fnJKDJCbqbVpb/eySfZiMKMCeG0pbuYeuOZBp97pyVUG4SwLSvL2GoxdX083c5bXJwdGXkpZTiufWp+LPKxJljUcOX/12ET8cvax2GG0CcxFzaTmleGpNEs7mu+98ZHJiMqIEO+4oCekF2Ho8z87dNexP6sVdiTlL3t+R7tB2znTtdQVPzQHa4gX9okwD4skt62ol5m5Ow9+/SVE7FGqD/rhsP345lY/Jnx1UOxRFMBlRwMPDe7S6zhf7M+3e36Kfz0iOYd6WNAyet122kU6laq2Xh9wz3lorlbCnpGHX6XyzGZPt5YpZez1lJmB7zmtbS5SKKzl8uZxYTWOuqTdkQZn7diyQE5MRBcg9Wd7Hu88BkHZx/zLxIur0Assbh5v/dO95vLfjtKxxSSVbFYTM9+u/rU5qfSUXMr0oc74O9+UZaaInYTbizZiMeDDHLoYNWy3YehpLd7v3PDim1Gw70dTNVyonBnU3/i86bhcKdNX2b+nGPy/lfAv//k0Knlx92O27PhORfZiMkFM2JGXLsh/57iktd2Tvrr/67SI2pVxqsfy59al2be+qAccOZrh/LxJX5gTNq6qq6/T44ehlxJ8uQHZRlesOTIpy4zyaFOCndgDkuPKa+tZXApCSZT6FvZy/Jl/YeMyu9SxdaOSrpWm5J2uJgbVj5pZWYe7mNJkiIktcca9pbZA7IvIMLBnxYG/9eLLVdSpr6106zLslQggcOH9FUvWCK28p9twEy6rtS+xs8ZTGpp6Gc/wQtX1MRjyYPTdQS+vY+jE5f0sanvj8IPROTM+750whHl95EKMXxBuXWbqdSCmhceZGb8+W7nS788TianveH0ffQSZ53sEDP/YkI1bTKESjkbde3d4qGqnWJF4E0DA0/W39rndoH/vOXpEzpFa5a0m9JyYVnsZN33pygDs3vibXY8mIh5q88je71rM0OqQ9F3BnSkYc2dKZdixSqlhc2fvC2eoEOQal84bruTe8Rm/Et9W7MRnxUEcvldq13ts/nXJo/3JfGCz96pGSFtjKIe56b7fN9ZS6yDlbnXDL6ztwKlfn8Ci27q6t3WzYrZhIPqymUYgGHlakLNOdo7K2HoH+lj9meaXXGrhKOTdSSyDsajPigjul1GLn6joDxn/wa4vlHvW5scEVr8ObEgK9QaC8ph7B7dupHQoA4OuDF5FbUo1/jhsoy/5Y4uXdWDKiEHeqD1XyAj543g6rz415d7fV55rklVbjuyOXzKowbJVAuNFpVoVaPU+8KCdQzcNL9yPyjZ9xqdg95up5dVMaPt59Dicu21dK2xr2mvJuLBkhi5y5MDS/MZmWgEj1wAd7UVJZh2fv6e/QsVs873Ak6pHyTnhDMuatic/xnIab/va0PDw1pp/K0Vyjq3JNY3pyTL3eAD9fzytn8LyIyWlKX8vXHmp90jlrN5iSyjoAwO70QknHNL0pp+XI88ut1WO2wV92WVcr8VH8WZQ2vg/OcPTs2Eo+5Pwsn8rVQVft/OukBicv6zBz7RFcKCxXOxSv8ene8xjw2jYkZbr/qM3NMRlRiKfdppT/hW3/beWwxOHR5205Ycdazr/gtjgexu8/+hWLdp7BK5uPO70vdz47hzKKMP6DX3G3HVWHlrii6rOoohZlHpwcPbxsP346lou/fnHYrvW9oVTP1RZsPQ0hgDnfOf99VRqTEYX4+rjHN00I1xdzt7gp23HAR5Yn2v2r9PUfTuJquftNq+0O86SYfsqq6/RO70/X2G364AXbCaBLR9BV4Kvzy6l8AECxhBIgV77m8pp6jHhrJ255/WcXHuWatJxSXJH5O1Vb39DOK6vIPdq4yE0IgU/2nMeeM9JKbZXgzom/NUxGFLLif0eqHYIkcl7/DcK+G8rEj/bZvc9CKxdOT/wSusrY9xPUDkEWNqtp2ugbnlFYodix0nJK8fuP9mHU2784tH1bLBG0x69nryBu22lMXXVI7VDaBCYjChkWEeLyY1TW2teQTOrF41xBGbKuOv7rxt7JzC7aOIbUYnBryY+13Sjx67u6To/Z61Lw47GWA9G5Qq6EhsOLd55xYSQNzhXY13bgcGYRHv0kEadydS6OSD6enBT9duGqczuQ6bV7WjVNTon6JaGmjmaXGP/viV3emYwoxEeBb5o9E+etO5xt176auiKXVtUhZvFes4HFbLH0FZAymOvpPB3+uGw/DpxzbEh5e7+Duuo69Hn5J/xh6X6HjuOIVfszsCX1MmatTZG0naQxWBz8mH0YfxaFZbaK6W1HIee17y8rEnEoo8jqL04PvM5K5g435rScUpRU1qodhttS8y26VFzZopeiktcyV2AyohCNAmd6x4l82ffpSLfc5jcLAWF3T5MnVychJasEj392UPJxpXji84YbXdOvCVdcWJrfUK6UufeFvVaG4ejlVGAzOWriBZmJCxw4dwUZV6xXBSVlFuH3H+3D7Qt3tbovud6BttgbzRUqaupx5zu7cVtcPAxOTNvhbpiMKMSdvmb2/LJ09JeZxc0kfF+cbURnbxVUpo0LsavIek6tHsOZ8WGsnzu1SiPkblTpKk2nJ/liMc67eVfWE5dL8fhnB41tiix9ZnanFwAAKmudbwRtL3coDSrQVeOQxN56SjNN0uutJCOemKIwGVGIEtU0cmqKVmr7EsvVNMLuC42rT5MnNrZTKmJnEg5XnddRb/+CY5dKXH4sR+rYm29yqbgSjyw/gHsX7ZEpqqbjmB/I2Zvlicutt8UxPaRc7Q8uXq1Ato2eNe5wiRy9IB6PfpKIJb+ccdsxZ0xPkydez6xxKBlZunQp+vTpg4CAAERFReHQIeutiVeuXIkxY8agU6dO6NSpE2JiYmyu31Z5XDLSGK4c16H4UwXO78RO7tyewFWfgA9+OYv5W9JctPcGap7Wbw5Zb+fkTu+3rWoPOf18Uv7qWFv+vCLR6eqAqlo97n4vAWPe3S3L7NSutuSXs/jL8kSb67j1Jd2Nvhf2kpyMrF+/HrGxsZg/fz6OHDmCyMhIjBs3DgUFlm84CQkJeOyxx7B7924kJiYiIiIC999/P3JycpwO3pO49QdXRpZe5gUXXqQtjaVxvrAcW4/nWd2mrLoOpVXmv3rkmjvI9IakVB34v385gzWJF3G+sNzpI9bpDW7daPFsfpnaIVjkyt4LaidcyReLcdZGTyh74is2+UwpWfXjjHQ3/ay1VZKTkcWLF2P69OmYNm0aBg8ejBUrViAwMBCrVq2yuP7XX3+NZ599FsOGDcOgQYPw2WefwWAwID4+3ungPYk7JSMLt522Y62GgKVeCK2tbu/Lr66z/KvpdJ75heHghSLoDQL/btYlVQjg/76x3Vvl9rjWG+U56q9fWC/1k/MzcLmkqsUNsMbCuXvmq2RsO55r1z6FaBjrZdibO6VPxqbQDfO+f+9V/easBNPPiru/XHuqCuz57LMBq33c6V4iJ0nJSG1tLZKTkxETE3NtBz4+iImJQWKi7SKtJpWVlairq0Pnzp2trlNTUwOdTmf28HSurqbJ19nf62X1gUy715WrTvKzfRnG/7+w4Sg+ij/r1P7mf38Cy3afw4HzLcdIaF7qYUoIoKzGdRN72RorJV/nWGPM5p+cz369gNsX7sJ7O9Jb3Xb7iTzM+PqIXccREMakr3nPLDXHLbD1zdmUkgO9DD0KrJWM6Q1C9dF+XX3um7/y5IvF+DbJvGpM1rYJ7p5dOaC6To/cUtePO5J8sRhPrkky/m3to+GJp1hSMnLlyhXo9XqEhYWZLQ8LC0NenvVicVMvvfQSunfvbpbQNBcXF4fg4GDjIyIiQkqYbsnVyUjUgngUVchXvF7fWK8r9Tpoz6vckHwJi2QYZMvSBHwC9jeWVVJtvQHfH3VssLOmt6CqVo8z+WV4+6dTAIBlCedbrmzltdtzbNP3ut6N6vVtfQSXJZy3a3ydVo9h5YM+bfVhjHz7FxzJKra9vdMRmDMtJVD6xvLI8gO4Um5+LfGG0ihnjH0/AdFxu3CuwLVVO48sP2D34IGeRtHeNAsXLsS6deuwadMmBAQEWF1vzpw5KC0tNT6ys+0bqMuducnUNHab9OlvDs25IKDsr+jmv9he//6EzTliyqotl4pUyFhaUlpZ1+Ic/OBgImJq4sf7cP+/9zq0bWtVV81Z6zLoDppHJqWkT6q9jd+B/yRedMn+iypqVR8t057kXYkQHSl90RsELl5Vvps+YJ4wNo12rGRjfcBGyYgHZo+SkpEuXbrA19cX+fnmRbj5+fkIDw+3ue3777+PhQsX4ueff8bQoUNtrqvVahEUFGT28HRyNZBU0tRVhyRfhMpdWAVij8OZtn/Bxm09ZXH5UyZFn86KfPNnTP8y2ewiX1PveElD024s/SKS86Jjuid36vGg9DfH0kB/cpzl1fszsHT3OePfv5zMx4i3duKVTbZnWHX0LVbqhmTXuEVW3sXSqjosTzgvvY1So2e/Tsbd7yVgc4p3dYho4rVde/39/TFy5EizxqdNjVGjo6Otbvfuu+/irbfewvbt2zFq1CjHo/VwPULaqx2CIta46FekJWk50toTWRtjIU9Cmxt7NM0C28SZi4atLZvPP+NMI0DTm1fzdhitzWZrK0a5q3zkLMWy5La4lo3rW5tfqbUbssEg8PoPJ/HejnRcbpzT5P2fG9r8mHZdFkLgaHYJququvUZHPjtfJWZi+Fs7kZZTKnlbS+S86Znu65VNx/HO9tP407IDDu2rqW3Tp3svAACyiypRpVRvHTf4femBBSBWSa6miY2NxcqVK7FmzRqcOnUKM2bMQEVFBaZNmwYAmDJlCubMmWNc/5133sHcuXOxatUq9OnTB3l5ecjLy0N5edus97Jl30tj1Q5BsqPNBpxSkivqX5UsoLLVhkfqr9YtqZZ/+ck5WqRc1zXT15aQXoBBc7fjWzvnRLKkeYPjqAXK98TbknoZsetTce+iBBy/1PIGf7yVm75pMlNloTt6k++PXsYflu7HIyZjXAgBFEtsDzZ3ywmUVNbhnxuOmi2/eLUCL248JmlfTTEcySq2+Hmz53Nj7Xu372zDHFT2Df1ve/8nL+sw5t3duHdRglP78iTWzn3Tck8aLl5yMjJp0iS8//77mDdvHoYNG4bU1FRs377d2Kg1KysLubnXuhIuX74ctbW1+POf/4xu3boZH++//758r8JDeGJVzfzvT6h27JjFjrWPkMpVxdlNv9YajuH4frKuVmD2ulSLz9k7I7JUFTWO/bo8klWMUW//gu+OXAIAPP1lMuoNAi/+V/oN0BUOZRQhvVk3cXt743yXkoPzhRV46svDLZ57+sskJF+0XUVo1zGOWE46P3Cy91mTN39wrLFvvUHgT8sO4NFPEp0emdR6OwendovtJxo6UVx2YD4tORVV1LpkrJ7mn9vWZF6pwLA3f1ZkRm45+Dmy0axZszBr1iyLzyUkJJj9nZmZ6cghyE3I0W3SnVhKCLel2dcTTE6ncu2/sNjqEtz8At5avnuuoAx+Pj7o0+U6m/tatT8D/zPavBfb3jOFGN23MwLa+Vrd/zNfJeNqRS1ivz2KP43oaTsYhRzOLEJ4UAB8fDR49JOGEofMhRMANJyP33+0z+r4NpZcS9SunbCrFbVY8ot5wlBaWYfP913AH4b3QO/OgXbt29r7J1dbLEuTIdrzE6nWpM2TrqoOQQHtJB3X2jHk+n3mLr/zsosrMeKtnQCA8wsehK+MPRee35DaYpm1H1JCAO/9nA5ddT0+jD+L2PtulC0OV+HcNORVLF0afj0rvdeQVM0vGQ9++Kvd29q60P7LpEGuRmP7xlJWXYeYxXtxz/sJVopvzZc176UyZdUhzLMy7HzTRbF58qp2A7tTuTr8ZUUixry722Kvi4Xb0iUlIoD9JWnzvk/Dh7vO4b7Fzs1VY2+JQV5pNban5Zq9B3IVnNmeRNHx+auUbPNw8WoFZq09YrUdTaHEqiJL37X//HZtuIHKWtsJZFl1Hban5VocRdqSuvqWJ6st/VRkMkJe5WRuywasnlh91kTK2DKmF1tL1TvNF1m6UXybdMnmMdzt4th8kr3mXPXWZxdVGrtzG4T958VSOPYmdPe8vxvP/OcI1h1uOf6Os5onEJKrNl3cfVhjR9Ptp9Yk4cdjufj9R/ssPh/7barjAVhg+nLSckrxu0UJ+PnEtVLYp79MxjP/OYI3fmi9Kry4otbiZ7XOSi+9rKJK9/sytoLJCHk9RVIRN2j2bpp0WSoYsfd3l9TGlO7iiAxtOuw15t3dZufY9O239XmzlBj/66dTdvWKaSrhSUh3fUmfKXs+2dY+/kr+DjhfeK3ThKXq59TsElmPZ/qa/99XybhQWIGnv0pumOsnvwyJFxpGj97QSoIPAJV1eosDZ458+xeLDaoB4Cc7p4FwF0xGFPbds7erHQI1o8QF0ZlUxN6JxXRVdVYbQAKtTz1ub740vLFO3GxbNIyD0rykRskcrKCsGjX15ufK9Pjv/9yyIZ8jb72zL0nq9l8fzGoxN5PN/ZsmQc2OZqnbqz0lgx82a0Ar9X2d/NlBk22ln8HaVsbpkfodPpjRchoJ2UsSTPZXYVJl88jyA7hP4uCFQlgfWfqd7aetJiSehMmIwkb06oRJozx/ePu2xNVD9TtrY3Lrv5wAYObaFJvjpZj+8rPnfiD1nhHjZNsIAA4PfnW+sByj/xWPB5bY3xYHUO6Xub1VLXKH0/w9THKwdMjSHFBS2DOEefNzdCpXh/S8MmxJzcGNr23Df+38HlhjraTKVWQdm0VYv05pNMCxnBLZjqUWh3rTkHPCg60PhU/2az7gl6MszXbria60MqGbaVG/PV2C5bhgS93Fne/sdug4Oxrr4jOumDdSbS3ZcGSQuKbzIuX82FtNY226Anei0ThXiGBvA9bxH5gnls9vOIpHRlruoSX1XbTcNkfiPlr5cMmd8MjVMaeyth4/HsvFvYNCcX0HrTw7lQFLRlTwtzv6qh0CmVif1PqAXHvPFDo1HokbNBkxuzFbbjPieE8Ya6+vta7hV8prsDH5kt09CqyRe/p5WzM/y8VS4nEoU45B7KR92KSWDgkBHLxgUlJi4XC2phNwxXdB7w5fMBtstxOydyeWV/z17BW8uslyLzdL3vzhJF7ceMys6swdMBlRgbYdT7unmbLqEH485niDMHeYuOp4KyUj9vSmkcLWIE1ljQNn/WVFIv654SgWWJkzyF7NfzUaDMKuBMfajWDaF4dsbpd8scjyrMl2+vVsIXJKXDPlvHmbEflV1+nxuGkbkGZH2XOmEANe3WZ1e9P1Tc+/M5+35tNCLNx22q4RnIUQmP5lEv654ajk72hrOYSw8n9HyVUysrWxYauUdkhK4F1RBW7eRIGs2J2u7IyccvvtwrVf3cLCD9cWyYiTx1uWcM7qc7e8/jPe2X7aWK2y44RzA8/FbTtt9vejnyRi0NztKKqwXcJh7bt4JKvE6hD8AgKPLE/ErtOOfx4WNotXTVKTgHd3pNt8ftbaI60c0PFjL7eRAJq+lyv2nLdrhuuTuTrsPJmPjcmXLJYWOsPe5MaeUj1bbUakUv9nkWVMRojsZeFbbO8Q4Cv2XGh9JQVZKhmRMsy3PRfaLamXbT5v68birKaGmrtO57eypnXWhuB35Be86TY5JVW4Wu667tHWwrM2CNc3h66NS/LLydbP10471rHFWny25uxp8s52+5M4m8lF4319wofXxhyxt1pSCIH3dpzGj8dsf75N92YrjbA0Km7LfQmnSkaSL1qu/ssuqpRtMkVnsQGrCuSu3yb1/L+vkuxaT+5ZgZ1lKRlp3u3T1k1XiGZF7E7GY2vIe2dY+66V19Sjg9ZPse+i6Y3uic9tVwE5wrRtjmmJTdbVSiRfLMKl4iqzpMOUaXH9U1/a93k21VrpU3NmjXlNTv9fViS2XNlFLL3vrSWZNfV6XCmvxfmCcizd3XoiLee8UUI4d9+Yuuow0t4Y12L5mHcbGozve2ksenayb8oCV2EyQuSEeg+du8eesG21ufCUV22tQei/fjqF6jq9QwND2WqcaU25i3vJfJWYaXF5rd5gNgNwc9MdSD6a++eGo/hzYy8XXXVdqz2CBATq9QbohZC1Mev+c1daLItdn4ppd/TFLT2DW92+xspYJkIIzFqbYvysTI3ubVc8cjcTc6aWprymHjtP5mN0384Wnz9bUM5kxBupPV8HOca8S6LA4cxij+iKaYmlapbmQ+XbulE3bC9vqYIrZjq1ZltaLkoqHesx40j+aTqDsytsSrE+2J0tzla5NJm9LgVLJg3DZ3a8TiGA+/69F4VlNbJOxHk4s2WV6XcpOfguJcc4MaIjCspqzL4L3x+1XT3TRM5kRMD5NiPTv0xCv67XWT7nbnBLYpsRFbhBxwpywKaUHGQXVcJgEHhufSoe/STRY2c1thT2e600TLS1fVGF89UsVxUcZt7RRMRR9iStrY0TY4vUyf7ktiX1Mk5c1tnV7kOgYTyY8pp6u9Z3BY1GQgPTZjmAvTPxyltNY30EVikuFFbYPaKz0piMEEkw5t3d+OZwVquNM92dsxfK5qV7+885N0InIF85i7UqCzXZUxqqc2BskwuF5diSmoN6g/oD99XUG+z69a5GN/dJn7SsqrL3d4Sfj/lt0t6JNaW+yvhT+Thx2XJjUjlKRmxxh9J6VtOowN+XOaAn2+phE1BZ4nQy4oJrl1wJ3twtrc+CqjR7zpcjN5vfLXJ+CH5ZuXh2XkcdzDBvO7R6fyZG9e7U6nYJ6QUtJh60910ySCg1PXapBE+uaWi/k7lwQottmzcYl9vHu85h7MBQVWcw511RBT4+Gpyw0LKZPIMcpQBqc/aGEH+qAHcs3CVPMI0+aDYZW1uyoZV5VQp01R4//tCstUeQmlXS6nruMNjW9hN5dpWM/PWLw1h9INNsmb1Jo+l3rLiVasHm8+4s3d18jB7h0kThSFYJtqc5N9aPs5iMqOQ6rR9LSEg1zpaMzFx7xGUjiHqj0QviUVHjnnX59sotrW5RAmGJsz148kqrHZ5Q0ZSjVRP2dtN/66eTAOwrIWm+xqe/mjcENgj5RmC15lKxut9nVtMQeSEPbXfbpjXvzUSW3RYXDwA4+aZzpcuuri5q6qnkyGGax7Yx+RIKy1wzFo+74E9zFY3oHaJ2COSl/rb6sNohUDP/3HBU7RA8ypUy53pfydnbxZo3fjiB03mtJ5mmoRzKKGoR26d7L+DE5badrDIZUdFHj41AO18Prygmj9Q0JwyRpyqpci4ZUaIh7Rf7M82GnLcai0n5yaOfJLpt91tXYjKioq4dtXjpgUFqh0FEHqakshYbkrLVDkNVD32836ntlSgZ8SS/WhjBVklsM6Ky2/t3UTsEIvIww97cqXYIHs/S8PFqcYe8aO+ZwtZXciGWjKhscPcgbH9ujNphEBF5lWf+c0TtEIy+Pmh5EkNvwmTEDQwKD2qxbO1TUdj6f0xSiIio7WM1jZu6voMWA8M7qh0GERGRyzEZcRObZ96BjcnZ6BToj5p6gzER+c+TUfjfzw+qHB0REZHrMBlxE8MiQjAsIqTF8jsHsIErERG1bWwz4gG2zLxD7RCIiKiNU2NG5SZMRjxAZEQIbu9/vdphEBFRG5aer94khkxGPMSS/xmG6WP6Ytfzd6sdChERtUFFFc6NausMJiMeIrRjAF6dMBj9unbAlOjeLj1Wvy7XuXT/REREppiMeKDXJgzGh48NR1CAHyZH9bJ7uwdvCbf5/IheITj55jhEWmhIS0REbZyKI8GyN40H8vfzwUOR3fH7W7rBx0eD/7m1FyZ+3PpkTP26dLD63M//uAs3hjV0J3594s3YlJJjdzyj+3bGoYwiu9cnIiIyxZIRD+bj0zDj7y09g3HolXttrjtmQBfMuKc/vn4qCrf26dTi+aZEBACCA9tJisPV1UZEROR612nVK59gMtJGhAYF4NcXxxr/Tn4txvj/mJtC8dWTUbhO64c7buiCr5+6DSN7d8LjUb1wz8CuePfPQ63ud1hECO4Z2NXq8w/cHI47b5B3LJRxN4fJuj8iImrd0J7Bqh3boTRo6dKleO+995CXl4fIyEh89NFHGD16tNX1N2zYgLlz5yIzMxMDBgzAO++8gwcffNDhoMmyiM6BWPO30eig9YOfz7U88+2HbzFbz9/PB/+dcbvNff0w606UVNVizICuEELgfGEF1hzIxPQx/XD+SjmSMovwUGQP40ix7/8lEjtP5mH+xJsRHhSAfq9sBQB0Cw5Abmm11eOcevMBfLL3PEoq69A9JAB9rr8OaTml2HEi39HTQEREDtBoNKodW3LJyPr16xEbG4v58+fjyJEjiIyMxLhx41BQUGBx/QMHDuCxxx7Dk08+iZSUFDz88MN4+OGHkZaW5nTw1NLdN3bFyN6dcJ3W17gsRGK1C9BQ9TNmQEOJiEajwQ2hHfDWw0PQ6/pAjB0YihfGDTKbO+fPI3vikydGoXtIe2g0QI+Q9uig9UPCC/fgT8N74L7BYTj3r/F4feJgjO7b2bhde39fPBdzI15/6GY8fVd/3H9zOJ65p79ZLNufG4Ok12JwW7/OsFdIYDu8++ehiLkpFGNNSnYS/nmPxfWPzL0Pz97TH9/PsjzA3BiOhEtEbZjcJdxSaYTEIdeioqJw66234uOPPwYAGAwGRERE4O9//ztefvnlFutPmjQJFRUV+PHHH43LbrvtNgwbNgwrVqyw65g6nQ7BwcEoLS1FUFDLGW7JsqvlNTAIoGtHreLHrtMbYBACWj/fFs9V1erx1JeHce+gMPztzr4Wt39/Rzo+3n0O258bY5zVWG8Q6N9Y4gI0NLp9ak0SsooqjcsyF05AVa0e7f2vHTdfV43n1qVi6u298cCQbjh5WYf/HrmEPw7vgSE9WhZLns0vw33/3mu27OAr96Kdrw9GvLXT6mtOei0Go97+BQAwILQDpt3RF69sOm51fVvOvD0en+w5j0U7zzi0vam/3t4HZwvKsP/cVaf3RURty7HX70c7Hx8EtPNxScmIvfdvSclIbW0tAgMDsXHjRjz88MPG5VOnTkVJSQm2bNnSYptevXohNjYWzz33nHHZ/PnzsXnzZhw9elTWF0NtX3FFLbal5WHCLd2MDW2TLxbjy8RMPDKiJ+660Xr7Fin0BoF8XTV8fTQoq67DDaENpUDpeWUYt6QhURncLQg5JVXofX0g/jvjdrTzbVnQmJZTisLyGtze/3pU1ugx3Eoys/apKMz7/gSyiirxziO34I/DewJoSIyKKmqxKSUHgf5+WLU/A69PHAw/Xx+EBQVg+pdJeP6+G/G/t/XGFwcy0btzIJ7f0PC9evXBm1BUWYsXxw2ERqOBwSBwOq8Mm1Nz8OneC1Zfu6+PBq8+eBPe/PGkU+fQUb2vD8TFq5Wtr+gmdj1/N363aI/aYRBJFtkzGFtm3enSY9h7/5bUZuTKlSvQ6/UICzNvYBgWFobTp09b3CYvL8/i+nl5eVaPU1NTg5qaGuPfOp1OSpjUhnW6zh+PNxtbZWTvThjZu2UPIWf4+mjQPaQ9ACAsKMC4fGB4R2QunGD3fkxLXrR+vkh+LQZpl3UYc0MXY2+oJr/Ethxdd0BjL6eofg3TAbw0fqBZadPJN8ch0L/haxx7340AgBtCO6DeIFqcEx8fDQZ3D8Lg7kF45cGbAAAr917Ah/Fn8d2zt0Pr54te1wca19cbBC5cKcez99wAfz8fRC2Ix/+7qx+ev38gNBpg3eFsfP3bRQDAsskjcLmkGnM2HcPU6D7oHtIez397FHN/PxjL95zD/9zaC2fyy7Al9bIx1sXNSn0WPxqJPwzrgdKqOvzvZwdx8WoF2vn5YGBYR4wZ0AUThnbH3785gvCg9hgU3hE/HLuMtx8egrd+PIkz+eUtzp3Wzwc19QaL78umZ2/H5/sy8OOxXLTz1WDd07fhkeWJ6Kj1w7Q7+yIlqxi/nr0CoKGRdk5JFY7nlLZ8f0I7oF/XDpga3RtrEi/i+uv8cdXOUSwfuDkc2080XAd7dmqPS8VVxudCAtuhpLIOABDaUYuIzoFIvlgMALi9//U4cF6+Ui6NBvj90O744ehli8/PuKc/Dpy7gqOXWr5+eyyfPAIzvj7SYvna6VF4fGXLGck/emw4DEJg9rpUh44HAP27XofzhRUtls8ZPwhx2yzfq+Qy+94B+CD+rCz72jZ7DMZ/8Kss+7Jk0aPDXLZvqSSVjFy+fBk9evTAgQMHEB0dbVz+4osvYs+ePTh4sOUHy9/fH2vWrMFjjz1mXLZs2TK88cYbyM+33Ejx9ddfxxtvvNFiOUtGiDyH3iDg62O92Le4ohaBWl/4+fjYXE8uNfV6+Pn4oE5vQEC7ltWH9iivqcfpXB0GhHa02QVeCIHiyjp0vs7f+HdTEXi93gC/xlI0g0FAo2lol1VZW4+kzGLc1u96+PpojOek6RJdXWcwq36s0xtwobACIYHtEBYUgNp6A/x8NNBogItXK1FeU4/B3YKMSa8QAsdzSpGeV4b7BochJNAftfUG+PtdK9G7Wl6DToH+OJJVjNTsEvztjr7G7QvLauDno0FOSRUC/X1RUFaD2/pdmzOruk6P5IvFuCG0A8qq69Cr83Vm+66u06NWb0BuSTX6dml47nxhOXKKq1BVp8epXB0iI0IwdmAoAKCsug4ZVyowuFuQ8XxV1+mRW1qNoopadA8JwJ70Qoy/pRuC27cznmMhBAwCOJxZhKE9g+Hv21D9oEFDQi6EQHlNPUqr6tDO1wcGIRDaMQC+PhpcLa/BsUuluKVnME5c1iGqb2ezz0pxRS0EgE0pOeig9cXFq5Xo17UDbu3TCZW1etzULQhCCKRkl+DkZR1OXNZhzIAuuHNAF2gA7DpdgNzSasTcFIoNSZcwOao32vlpcKGwAqnZJUjPK8P3Ry/jgZvD8cIDA9G/a8PYUFfKaxDcvh0yrlTgSlkNuoe0x6HMInQK9MfYgV1RVadHTb0BeaXVGNIjGAVlDedoS+pllFbVoWsHLSbdGoGOAX7ILqpCny6BqK4zGD+fruTR1TSWSkYiIiKYjBAREXkQe5MRSb1p/P39MXLkSMTHxxuXGQwGxMfHm5WUmIqOjjZbHwB27txpdX0A0Gq1CAoKMnsQERFR2yR5nJHY2FhMnToVo0aNwujRo7FkyRJUVFRg2rRpAIApU6agR48eiIuLAwDMnj0bd999NxYtWoQJEyZg3bp1SEpKwqeffirvKyEiIiKPJDkZmTRpEgoLCzFv3jzk5eVh2LBh2L59u7GRalZWFnxMBty6/fbbsXbtWrz22mt45ZVXMGDAAGzevBlDhgyR71UQERGRx5I8zoga2LWXiIjI87ikzQgRERGR3JiMEBERkaqYjBAREZGqmIwQERGRqpiMEBERkaqYjBAREZGqmIwQERGRqpiMEBERkaqYjBAREZGqJA8Hr4amQWJ1Op3KkRAREZG9mu7brQ327hHJSFlZGQAgIiJC5UiIiIhIqrKyMgQHB1t93iPmpjEYDLh8+TI6duwIjUYj2351Oh0iIiKQnZ3NOW9ciOdZOTzXyuB5VgbPszJceZ6FECgrK0P37t3NJtFtziNKRnx8fNCzZ0+X7T8oKIgfdAXwPCuH51oZPM/K4HlWhqvOs60SkSZswEpERESqYjJCREREqvLqZESr1WL+/PnQarVqh9Km8Twrh+daGTzPyuB5VoY7nGePaMBKREREbZdXl4wQERGR+piMEBERkaqYjBAREZGqmIwQERGRqrw6GVm6dCn69OmDgIAAREVF4dChQ2qH5Lb27t2LiRMnonv37tBoNNi8ebPZ80IIzJs3D926dUP79u0RExODs2fPmq1TVFSEyZMnIygoCCEhIXjyySdRXl5uts6xY8cwZswYBAQEICIiAu+++66rX5pbiYuLw6233oqOHTsiNDQUDz/8MNLT083Wqa6uxsyZM3H99dejQ4cOeOSRR5Cfn2+2TlZWFiZMmIDAwECEhobihRdeQH19vdk6CQkJGDFiBLRaLW644QasXr3a1S/PbSxfvhxDhw41DvIUHR2Nbdu2GZ/nOXaNhQsXQqPR4LnnnjMu47mWx+uvvw6NRmP2GDRokPF5tz/PwkutW7dO+Pv7i1WrVokTJ06I6dOni5CQEJGfn692aG5p69at4tVXXxXfffedACA2bdpk9vzChQtFcHCw2Lx5szh69Kh46KGHRN++fUVVVZVxnQceeEBERkaK3377Tfz666/ihhtuEI899pjx+dLSUhEWFiYmT54s0tLSxDfffCPat28vPvnkE6VepurGjRsnvvjiC5GWliZSU1PFgw8+KHr16iXKy8uN6zzzzDMiIiJCxMfHi6SkJHHbbbeJ22+/3fh8fX29GDJkiIiJiREpKSli69atokuXLmLOnDnGdS5cuCACAwNFbGysOHnypPjoo4+Er6+v2L59u6KvVy3ff/+9+Omnn8SZM2dEenq6eOWVV0S7du1EWlqaEILn2BUOHTok+vTpI4YOHSpmz55tXM5zLY/58+eLm2++WeTm5hofhYWFxufd/Tx7bTIyevRoMXPmTOPfer1edO/eXcTFxakYlWdonowYDAYRHh4u3nvvPeOykpISodVqxTfffCOEEOLkyZMCgDh8+LBxnW3btgmNRiNycnKEEEIsW7ZMdOrUSdTU1BjXeemll8TAgQNd/IrcV0FBgQAg9uzZI4RoOK/t2rUTGzZsMK5z6tQpAUAkJiYKIRoSRx8fH5GXl2dcZ/ny5SIoKMh4bl988UVx8803mx1r0qRJYty4ca5+SW6rU6dO4rPPPuM5doGysjIxYMAAsXPnTnH33XcbkxGea/nMnz9fREZGWnzOE86zV1bT1NbWIjk5GTExMcZlPj4+iImJQWJiooqReaaMjAzk5eWZnc/g4GBERUUZz2diYiJCQkIwatQo4zoxMTHw8fHBwYMHjevcdddd8Pf3N64zbtw4pKeno7i4WKFX415KS0sBAJ07dwYAJCcno66uzuxcDxo0CL169TI717fccgvCwsKM64wbNw46nQ4nTpwwrmO6j6Z1vPHzr9frsW7dOlRUVCA6Oprn2AVmzpyJCRMmtDgfPNfyOnv2LLp3745+/fph8uTJyMrKAuAZ59krk5ErV65Ar9ebnXQACAsLQ15enkpRea6mc2brfObl5SE0NNTseT8/P3Tu3NlsHUv7MD2GNzEYDHjuuedwxx13YMiQIQAazoO/vz9CQkLM1m1+rls7j9bW0el0qKqqcsXLcTvHjx9Hhw4doNVq8cwzz2DTpk0YPHgwz7HM1q1bhyNHjiAuLq7FczzX8omKisLq1auxfft2LF++HBkZGRgzZgzKyso84jx7xKy9RN5o5syZSEtLw759+9QOpU0aOHAgUlNTUVpaio0bN2Lq1KnYs2eP2mG1KdnZ2Zg9ezZ27tyJgIAAtcNp08aPH2/8/9ChQxEVFYXevXvj22+/Rfv27VWMzD5eWTLSpUsX+Pr6tmhJnJ+fj/DwcJWi8lxN58zW+QwPD0dBQYHZ8/X19SgqKjJbx9I+TI/hLWbNmoUff/wRu3fvRs+ePY3Lw8PDUVtbi5KSErP1m5/r1s6jtXWCgoI84sIlB39/f9xwww0YOXIk4uLiEBkZiQ8++IDnWEbJyckoKCjAiBEj4OfnBz8/P+zZswcffvgh/Pz8EBYWxnPtIiEhIbjxxhtx7tw5j/hMe2Uy4u/vj5EjRyI+Pt64zGAwID4+HtHR0SpG5pn69u2L8PBws/Op0+lw8OBB4/mMjo5GSUkJkpOTjevs2rULBoMBUVFRxnX27t2Luro64zo7d+7EwIED0alTJ4VejbqEEJg1axY2bdqEXbt2oW/fvmbPjxw5Eu3atTM71+np6cjKyjI718ePHzdL/nbu3ImgoCAMHjzYuI7pPprW8ebPv8FgQE1NDc+xjO69914cP34cqampxseoUaMwefJk4/95rl2jvLwc58+fR7du3TzjM+10E1gPtW7dOqHVasXq1avFyZMnxdNPPy1CQkLMWhLTNWVlZSIlJUWkpKQIAGLx4sUiJSVFXLx4UQjR0LU3JCREbNmyRRw7dkz84Q9/sNi1d/jw4eLgwYNi3759YsCAAWZde0tKSkRYWJh44oknRFpamli3bp0IDAz0qq69M2bMEMHBwSIhIcGsi15lZaVxnWeeeUb06tVL7Nq1SyQlJYno6GgRHR1tfL6pi979998vUlNTxfbt20XXrl0tdtF74YUXxKlTp8TSpUu9qivkyy+/LPbs2SMyMjLEsWPHxMsvvyw0Go34+eefhRA8x65k2ptGCJ5ruTz//PMiISFBZGRkiP3794uYmBjRpUsXUVBQIIRw//PstcmIEEJ89NFHolevXsLf31+MHj1a/Pbbb2qH5LZ2794tALR4TJ06VQjR0L137ty5IiwsTGi1WnHvvfeK9PR0s31cvXpVPPbYY6JDhw4iKChITJs2TZSVlZmtc/ToUXHnnXcKrVYrevToIRYuXKjUS3QLls4xAPHFF18Y16mqqhLPPvus6NSpkwgMDBR//OMfRW5urtl+MjMzxfjx40X79u1Fly5dxPPPPy/q6urM1tm9e7cYNmyY8Pf3F/369TM7Rlv3t7/9TfTu3Vv4+/uLrl27invvvdeYiAjBc+xKzZMRnmt5TJo0SXTr1k34+/uLHj16iEmTJolz584Zn3f386wRQgjny1eIiIiIHOOVbUaIiIjIfTAZISIiIlUxGSEiIiJVMRkhIiIiVTEZISIiIlUxGSEiIiJVMRkhIiIiVTEZISIiIlUxGSEiIiJVMRkhIiIiVTEZISIiIlUxGSEiIiJV/X/4uKdMAuFRkQAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"code","source":["test_word_ids= list(range(0, N))\n","X_test = X_train[test_word_ids]\n","\n","model.eval()\n","\n","def Detokenise(X):\n","  with torch.no_grad():\n","    outputs = model(X)\n","  predicted_words = []\n","\n","  for output in outputs:\n","    predicted_word = ''\n","    for i in range(L):\n","      index = torch.argmax(output[i])\n","      if index == V-1:\n","        break\n","      predicted_word += index_to_letter[index.item()]\n","    predicted_words.append(predicted_word)\n","  return predicted_words\n","\n","predicted_words = Detokenise(X_test)\n","\n","acc = 0\n","for i in range(len(test_word_ids)):\n","  w = words[test_word_ids[i]]\n","  wh = predicted_words[i]\n","  acc += w==wh\n","  print(w,wh, w==wh)\n","print(100*acc/N)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YpyL5GKHxCtC","executionInfo":{"status":"ok","timestamp":1715097380535,"user_tz":-330,"elapsed":417,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"ebafd934-9a45-411b-f9ba-384636f2273a"},"execution_count":15,"outputs":[{"output_type":"stream","name":"stdout","text":["aaa aaa True\n","aab aab True\n","aac aac True\n","aba aba True\n","abb abb True\n","abc abc True\n","aca aca True\n","acb acb True\n","acc acc True\n","baa baa True\n","bab bab True\n","bac bac True\n","bba bba True\n","bbb bbb True\n","bbc bbc True\n","bca bca True\n","bcb bcb True\n","bcc bcc True\n","caa caa True\n","cab cab True\n","cac cac True\n","cba cba True\n","cbb cbb True\n","cbc cbc True\n","cca cca True\n","ccb ccb True\n","ccc ccc True\n","100.0\n"]}]},{"cell_type":"code","source":["# let us create some random sequences and see what we get\n","X_rand = torch.randn((100,L,N))\n","predicted_words = Detokenise(X_rand)\n","print(predicted_words)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-hpeg3STxYJN","executionInfo":{"status":"ok","timestamp":1715100402401,"user_tz":-330,"elapsed":481,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"def0d596-33dd-4117-efd0-e07f396ec50b"},"execution_count":17,"outputs":[{"output_type":"stream","name":"stdout","text":["['accc', 'cbbc', 'bbbb', 'bbb', 'bcc', 'ccb', 'bcb', 'cccc', 'cc', 'bbb', 'acc', 'ccab', 'cbb', 'aab', 'bab', 'cc', 'ccc', 'bb', 'cacc', 'cc', 'ca', 'ccc', 'cba', 'bcb', 'ca', 'acb', 'ba', 'aab', 'cc', 'bbbc', 'bb', 'bab', 'acc', 'cccc', 'bba', 'babc', 'ab', 'baa', 'caa', 'ca', 'ac', 'cccb', 'baac', 'ccbb', 'cbcc', 'cccb', 'cbbb', 'ccc', 'acc', 'bab', 'cb', 'baa', 'cb', 'a', '', 'aab', 'cbbb', 'cccb', 'aa', 'bcb', 'aaac', 'cbac', 'ccb', 'bbb', 'acc', 'ccc', 'cccc', 'cccb', 'bba', 'bba', 'cc', 'bcb', 'cc', 'abcb', 'ab', 'cccc', 'cbcb', 'cc', 'bbb', 'bbbb', 'bb', 'aba', 'cbcb', 'bb', 'bbc', 'cbcc', 'acb', 'bca', 'bbac', 'babc', 'ccc', 'c', 'aca', 'ab', 'cbcb', 'bba', 'ccbb', 'cc', 'c', 'ccaa']\n"]}]},{"cell_type":"code","source":["torch.save(model.state_dict(), \"./detok_toy_trained_model_kron_mlp.pth\")"],"metadata":{"id":"qzgIFalBVuik","executionInfo":{"status":"ok","timestamp":1715097085583,"user_tz":-330,"elapsed":408,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["# Get the weights of the RNN model\n","rnn_weights = model.rnn.state_dict()\n","\n","# Visualize the weights\n","for name, weight in rnn_weights.items():\n","    print(f\"Layer: {name}\")\n","    print(weight.shape)\n","    print()\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QPrP04f1WNhh","executionInfo":{"status":"ok","timestamp":1715097088247,"user_tz":-330,"elapsed":693,"user":{"displayName":"soma dhavala","userId":"15308128232301337351"}},"outputId":"d9db16c4-c13f-4a39-a4f9-06e4f01320e0"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["Layer: weight_ih_l0\n","torch.Size([20, 27])\n","\n","Layer: weight_hh_l0\n","torch.Size([20, 20])\n","\n","Layer: bias_ih_l0\n","torch.Size([20])\n","\n","Layer: bias_hh_l0\n","torch.Size([20])\n","\n","Layer: weight_ih_l1\n","torch.Size([20, 20])\n","\n","Layer: weight_hh_l1\n","torch.Size([20, 20])\n","\n","Layer: bias_ih_l1\n","torch.Size([20])\n","\n","Layer: bias_hh_l1\n","torch.Size([20])\n","\n","Layer: weight_ih_l2\n","torch.Size([20, 20])\n","\n","Layer: weight_hh_l2\n","torch.Size([20, 20])\n","\n","Layer: bias_ih_l2\n","torch.Size([20])\n","\n","Layer: bias_hh_l2\n","torch.Size([20])\n","\n"]}]},{"cell_type":"code","source":[],"metadata":{"id":"BobSRe3kwRjh"},"execution_count":null,"outputs":[]}]}