NLP_introduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-CTFz5HEzIxh"
   },
   "source": [
    "## Introduction to NLP\n",
    "\n",
    "__Goals:__\n",
    "\n",
    "- Implementing RNN based language models.\n",
    "- Implementing and applying a Recurrent Neural Network on text classification problem using TensorFlow.\n",
    "- Implementing __many to one__ and __many to many__ RNN sequence processing.\n",
    "\n",
    "\n",
    "__DataSets__: 1) English Literature for language model task (part 1 to 4) and 2) 20Newsgroups for text classification (part 5). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "7C-eNni0117y",
    "outputId": "b336a947-deec-4257-ee43-4ca0dddeac1b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "colab_type": "code",
    "id": "Hg7diNTCzIxm",
    "outputId": "8865d48b-05e9-41f4-a5bc-e324c79bc634"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tensorflow-1.15.0/python3.6/tensorflow_core/python/compat/v2_compat.py:68: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow.compat.v1 as tf\n",
    "tf.disable_v2_behavior()\n",
    "# from tensorflow.keras.layers import Embedding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "g-f0vuN5zIxw",
    "outputId": "804490be-229b-4c1d-fbcf-875393cebf07"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yXp5pAXOayLb"
   },
   "outputs": [],
   "source": [
    "f = open('/content/drive/My Drive/Colab Notebooks/English Literature.txt')\n",
    "text = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yhHQ-rj1i2gV"
   },
   "source": [
    "**Part 1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NFyW3Tn8aqBU"
   },
   "outputs": [],
   "source": [
    "# Cleaning the data (lower casing, adding stop word after every sentence)\n",
    "\n",
    "a = nltk.tokenize.sent_tokenize(text)\n",
    "\n",
    "temp = []\n",
    "for sent in a:\n",
    "  w=word_tokenize(sent)\n",
    "  w.append('stop')\n",
    "  word_new = []\n",
    "  sent_new = []\n",
    "  for word in w:\n",
    "    if word.isalnum():\n",
    "      word_new.append(word.lower())\n",
    "  sent_new = ' '.join(wrd for wrd in word_new) \n",
    "  temp.append(sent_new)\n",
    "\n",
    "a_nn = []\n",
    "a_nn.append(' '.join(snt for snt in temp))\n",
    "\n",
    "a_new = word_tokenize(a_nn[0])\n",
    "# print(a_new)\n",
    "# np.shape(a_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ej5dZhBPIBph"
   },
   "outputs": [],
   "source": [
    "# Assigning a number to all different words\n",
    "\n",
    "dictionary = {}\n",
    "repeat = 0\n",
    "index = 0\n",
    "\n",
    "for key in a_new:\n",
    "    if key in dictionary:\n",
    "        repeat+=1\n",
    "    else:\n",
    "        dictionary[key] = index\n",
    "        index+=1\n",
    "# print(len(a_new), len(set(a_new)), repeat, index)        \n",
    "vocabsize = index \n",
    "datasize = len(a_new)\n",
    "# print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "cqYcI3FczIyh",
    "outputId": "308d883d-728f-4a75-888f-3e80cd523ddf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X =  [[0.000e+00]\n",
      " [1.000e+00]\n",
      " [2.000e+00]\n",
      " ...\n",
      " [4.620e+02]\n",
      " [4.650e+02]\n",
      " [3.095e+03]]\n",
      "y =  [[1.000e+00]\n",
      " [2.000e+00]\n",
      " [3.000e+00]\n",
      " ...\n",
      " [4.650e+02]\n",
      " [3.095e+03]\n",
      " [1.000e+01]]\n"
     ]
    }
   ],
   "source": [
    "# Preparing the inputs to the RNN\n",
    "\n",
    "i=0\n",
    "inputs = np.zeros((datasize+1, 1))\n",
    "for word in a_new:\n",
    "    inputs[i] = dictionary[word]\n",
    "    i+=1\n",
    "inputs[datasize] = -1 #-1 for END\n",
    "\n",
    "X = inputs[0:datasize-1] #d-1 to avoid END; +1 else\n",
    "y = inputs[1:datasize] #d to avoid END; +1 else\n",
    "\n",
    "print('X = ', X)\n",
    "print('y = ', y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "dm8RffRbzIyn",
    "outputId": "efbb1706-433a-47d6-e0e2-c9379020543f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 1, 32)             356384    \n",
      "_________________________________________________________________\n",
      "simple_rnn_9 (SimpleRNN)     (None, 500)               266500    \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 11137)             5579637   \n",
      "=================================================================\n",
      "Total params: 6,202,521\n",
      "Trainable params: 6,202,521\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# RNN model \n",
    "\n",
    "model1 = tf.keras.Sequential()\n",
    "model1.add(tf.keras.layers.Embedding(vocabsize, 32, input_length=1))\n",
    "model1.add(tf.keras.layers.SimpleRNN(500, activation='sigmoid'))\n",
    "model1.add(tf.keras.layers.Dense(vocabsize, activation='softmax'))\n",
    "\n",
    "print(model1.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UJKK1v_TzIyw"
   },
   "outputs": [],
   "source": [
    "# sgd = tf.keras.optimizers.SGD(learning_rate=0.1, decay=1e-4, momentum=0.99)\n",
    "adam = tf.keras.optimizers.Adam(learning_rate=0.05, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "model1.compile(loss='sparse_categorical_crossentropy', optimizer=adam, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 884
    },
    "colab_type": "code",
    "id": "i63H1lgIzIy7",
    "outputId": "3c7f83b6-312c-48f7-d9b5-4d410f644627"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 191734 samples, validate on 21304 samples\n",
      "Epoch 1/25\n",
      "191734/191734 [==============================] - 5s 28us/sample - loss: 7.9176 - acc: 0.0410 - val_loss: 6.7994 - val_acc: 0.0683\n",
      "Epoch 2/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 6.3699 - acc: 0.0626 - val_loss: 6.7049 - val_acc: 0.0803\n",
      "Epoch 3/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 6.1246 - acc: 0.0832 - val_loss: 6.5644 - val_acc: 0.0850\n",
      "Epoch 4/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.9313 - acc: 0.0922 - val_loss: 6.6227 - val_acc: 0.0840\n",
      "Epoch 5/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.8378 - acc: 0.0963 - val_loss: 6.8155 - val_acc: 0.0890\n",
      "Epoch 6/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.7894 - acc: 0.1001 - val_loss: 7.0072 - val_acc: 0.0856\n",
      "Epoch 7/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.7295 - acc: 0.1018 - val_loss: 7.1450 - val_acc: 0.0886\n",
      "Epoch 8/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.6702 - acc: 0.1036 - val_loss: 7.1712 - val_acc: 0.0884\n",
      "Epoch 9/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.6251 - acc: 0.1052 - val_loss: 7.4337 - val_acc: 0.0908\n",
      "Epoch 10/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.6149 - acc: 0.1056 - val_loss: 7.3413 - val_acc: 0.0864\n",
      "Epoch 11/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5859 - acc: 0.1066 - val_loss: 7.4715 - val_acc: 0.0847\n",
      "Epoch 12/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5615 - acc: 0.1070 - val_loss: 7.4981 - val_acc: 0.0837\n",
      "Epoch 13/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5647 - acc: 0.1074 - val_loss: 7.5658 - val_acc: 0.0863\n",
      "Epoch 14/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5698 - acc: 0.1061 - val_loss: 7.7092 - val_acc: 0.0836\n",
      "Epoch 15/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5490 - acc: 0.1079 - val_loss: 7.7418 - val_acc: 0.0837\n",
      "Epoch 16/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5458 - acc: 0.1081 - val_loss: 7.7585 - val_acc: 0.0877\n",
      "Epoch 17/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5455 - acc: 0.1082 - val_loss: 7.8231 - val_acc: 0.0873\n",
      "Epoch 18/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5514 - acc: 0.1079 - val_loss: 7.9226 - val_acc: 0.0839\n",
      "Epoch 19/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5454 - acc: 0.1077 - val_loss: 8.0373 - val_acc: 0.0878\n",
      "Epoch 20/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5364 - acc: 0.1075 - val_loss: 8.0662 - val_acc: 0.0888\n",
      "Epoch 21/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5573 - acc: 0.1080 - val_loss: 7.9924 - val_acc: 0.0894\n",
      "Epoch 22/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5506 - acc: 0.1076 - val_loss: 8.0311 - val_acc: 0.0882\n",
      "Epoch 23/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5839 - acc: 0.1076 - val_loss: 8.1594 - val_acc: 0.0868\n",
      "Epoch 24/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5911 - acc: 0.1064 - val_loss: 8.2589 - val_acc: 0.0876\n",
      "Epoch 25/25\n",
      "191734/191734 [==============================] - 5s 26us/sample - loss: 5.5909 - acc: 0.1071 - val_loss: 8.2069 - val_acc: 0.0842\n"
     ]
    }
   ],
   "source": [
    "q1 = model1.fit(X, y, validation_split=0.1, batch_size=500, epochs=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FrKiB1aSzIzC"
   },
   "source": [
    "In the above solution, we used the RNN based laguage model with TBTT (t=1) and after 25 epochs, we have a model cross entropy loss of 5.59 for the training data.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HPiX5HNekYeR"
   },
   "source": [
    "**Part 2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7FcGXU-hbeFY"
   },
   "outputs": [],
   "source": [
    "# Cleaning the data  (lower casing)\n",
    "\n",
    "b=nltk.tokenize.sent_tokenize(text)\n",
    "\n",
    "b_new = []\n",
    "for sent in b:\n",
    "  w = word_tokenize(sent)\n",
    "\n",
    "  word_new = []\n",
    "  for word in w:\n",
    "    if word.isalnum():\n",
    "      word_new.append(word.lower())\n",
    "\n",
    "  sent_new = ' '.join(word for word in word_new)\n",
    "  b_new.append(sent_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l2AXfT0meAHf"
   },
   "outputs": [],
   "source": [
    "# Adding the STOP word to every sentence\n",
    "\n",
    "b_nn = []\n",
    "for sent in b_new:\n",
    "  sent = sent +' STOP'\n",
    "  b_nn.append(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "sVzWSLvjhEVc",
    "outputId": "b24c50d1-cf43-4f34-bfb4-ba6d958c13a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11138\n"
     ]
    }
   ],
   "source": [
    "# Word indexing\n",
    "\n",
    "dictionary2 = {}\n",
    "index = 0\n",
    "repeat = 0\n",
    "for sent in b_nn:\n",
    "  w = word_tokenize(sent)\n",
    "  for word in w:\n",
    "    if word in dictionary2:\n",
    "      repeat+=1\n",
    "    else:\n",
    "      dictionary2[word] = index\n",
    "      index+=1\n",
    "vocabsize = index\n",
    "print(vocabsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "fW3_jWRthEmm",
    "outputId": "0674a243-321b-4453-f960-1190edd7b722"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avglen =  17.097833065810594\n"
     ]
    }
   ],
   "source": [
    "# Checking the avg sentence length\n",
    "\n",
    "count = 0\n",
    "length = 0\n",
    "for sent in b_nn:\n",
    "  count +=1\n",
    "  length += len(sent.split())\n",
    "print('avglen = ',length/count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BrAGHqI_W9hH"
   },
   "outputs": [],
   "source": [
    "# Preparing the data\n",
    "# print(b_nn)\n",
    "\n",
    "data= []\n",
    "temp_data = []\n",
    "\n",
    "for sent in b_nn:\n",
    "  w=word_tokenize(sent)\n",
    "  temp_data=[]\n",
    "  for word in w:\n",
    "    temp_data.append(dictionary2[word])\n",
    "  data.append(temp_data)\n",
    "# print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "octc7R31ZwsN"
   },
   "outputs": [],
   "source": [
    "# Preparing the inputs\n",
    "\n",
    "X = []\n",
    "y = []\n",
    "\n",
    "for row in data:\n",
    "  X.append(row[:-1])\n",
    "  y.append(row[1:])\n",
    "# print(X)\n",
    "# print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aqg76lL_Zw-n"
   },
   "outputs": [],
   "source": [
    "# Padding the inputs\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "X_pad=pad_sequences(X, maxlen=30,padding='post',truncating='post',value=0.0)\n",
    "y_pad=pad_sequences(y, maxlen=30,padding='post',truncating='post',value=0.0)\n",
    "\n",
    "y_pad = np.expand_dims(y_pad,-1)\n",
    "\n",
    "# print(X_pad)\n",
    "# print(y_pad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "uUxNjfUyZxIU",
    "outputId": "f601cf58-da19-4900-eb94-a80295315b5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_14\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_14 (Embedding)     (None, 30, 32)            356416    \n",
      "_________________________________________________________________\n",
      "simple_rnn_14 (SimpleRNN)    (None, 30, 500)           266500    \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, 30, 11138)         5580138   \n",
      "=================================================================\n",
      "Total params: 6,203,054\n",
      "Trainable params: 6,203,054\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# RNN model \n",
    "\n",
    "model2 = tf.keras.Sequential()\n",
    "model2.add(tf.keras.layers.Embedding(vocabsize, 32, input_length=30))\n",
    "model2.add(tf.keras.layers.SimpleRNN(500, activation='sigmoid', return_sequences=True))\n",
    "model2.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(vocabsize, activation='softmax')))\n",
    "\n",
    "print(model2.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4FH51GihxaLp"
   },
   "outputs": [],
   "source": [
    "# sgd = tf.keras.optimizers.SGD(learning_rate=0.2, decay=1e-4, momentum=0.99)\n",
    "# adam = tf.keras.optimizers.Adam(learning_rate=0.1, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "model2.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "RYLzuEfqZlW7",
    "outputId": "9cfd0129-119d-4a7a-908b-8b40204bf565"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_32\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_32 (Embedding)     (None, 30, 32)            356416    \n",
      "_________________________________________________________________\n",
      "simple_rnn_21 (SimpleRNN)    (None, 30, 500)           266500    \n",
      "_________________________________________________________________\n",
      "time_distributed_10 (TimeDis (None, 30, 11138)         5580138   \n",
      "=================================================================\n",
      "Total params: 6,203,054\n",
      "Trainable params: 6,203,054\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 11214 samples, validate on 1246 samples\n",
      "Epoch 1/100\n",
      "11214/11214 [==============================] - 13s 1ms/sample - loss: 3.7258 - acc: 0.5379 - val_loss: 2.4781 - val_acc: 0.6491\n",
      "Epoch 2/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.9348 - acc: 0.5677 - val_loss: 2.4107 - val_acc: 0.6508\n",
      "Epoch 3/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.8432 - acc: 0.5721 - val_loss: 2.3836 - val_acc: 0.6517\n",
      "Epoch 4/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.7789 - acc: 0.5751 - val_loss: 2.3564 - val_acc: 0.6549\n",
      "Epoch 5/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.7115 - acc: 0.5795 - val_loss: 2.3289 - val_acc: 0.6582\n",
      "Epoch 6/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.6536 - acc: 0.5837 - val_loss: 2.3133 - val_acc: 0.6604\n",
      "Epoch 7/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.6066 - acc: 0.5863 - val_loss: 2.3076 - val_acc: 0.6605\n",
      "Epoch 8/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.5640 - acc: 0.5890 - val_loss: 2.3022 - val_acc: 0.6605\n",
      "Epoch 9/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.5246 - acc: 0.5911 - val_loss: 2.3028 - val_acc: 0.6591\n",
      "Epoch 10/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.4878 - acc: 0.5928 - val_loss: 2.3063 - val_acc: 0.6604\n",
      "Epoch 11/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.4515 - acc: 0.5947 - val_loss: 2.3025 - val_acc: 0.6610\n",
      "Epoch 12/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.4156 - acc: 0.5964 - val_loss: 2.3127 - val_acc: 0.6605\n",
      "Epoch 13/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.3804 - acc: 0.5977 - val_loss: 2.3199 - val_acc: 0.6602\n",
      "Epoch 14/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.3454 - acc: 0.5991 - val_loss: 2.3224 - val_acc: 0.6620\n",
      "Epoch 15/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.3094 - acc: 0.6003 - val_loss: 2.3284 - val_acc: 0.6623\n",
      "Epoch 16/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.2734 - acc: 0.6013 - val_loss: 2.3340 - val_acc: 0.6613\n",
      "Epoch 17/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.2367 - acc: 0.6030 - val_loss: 2.3435 - val_acc: 0.6610\n",
      "Epoch 18/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.1999 - acc: 0.6042 - val_loss: 2.3490 - val_acc: 0.6613\n",
      "Epoch 19/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.1624 - acc: 0.6065 - val_loss: 2.3564 - val_acc: 0.6626\n",
      "Epoch 20/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.1246 - acc: 0.6090 - val_loss: 2.3641 - val_acc: 0.6624\n",
      "Epoch 21/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.0872 - acc: 0.6122 - val_loss: 2.3725 - val_acc: 0.6634\n",
      "Epoch 22/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.0491 - acc: 0.6157 - val_loss: 2.3810 - val_acc: 0.6632\n",
      "Epoch 23/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 2.0117 - acc: 0.6196 - val_loss: 2.3891 - val_acc: 0.6624\n",
      "Epoch 24/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.9755 - acc: 0.6238 - val_loss: 2.3938 - val_acc: 0.6630\n",
      "Epoch 25/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.9392 - acc: 0.6277 - val_loss: 2.4040 - val_acc: 0.6633\n",
      "Epoch 26/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.9037 - acc: 0.6320 - val_loss: 2.4155 - val_acc: 0.6627\n",
      "Epoch 27/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.8698 - acc: 0.6361 - val_loss: 2.4212 - val_acc: 0.6628\n",
      "Epoch 28/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.8360 - acc: 0.6399 - val_loss: 2.4331 - val_acc: 0.6620\n",
      "Epoch 29/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.8036 - acc: 0.6440 - val_loss: 2.4414 - val_acc: 0.6620\n",
      "Epoch 30/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.7722 - acc: 0.6486 - val_loss: 2.4489 - val_acc: 0.6620\n",
      "Epoch 31/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.7417 - acc: 0.6522 - val_loss: 2.4582 - val_acc: 0.6608\n",
      "Epoch 32/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.7122 - acc: 0.6564 - val_loss: 2.4673 - val_acc: 0.6610\n",
      "Epoch 33/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.6839 - acc: 0.6600 - val_loss: 2.4780 - val_acc: 0.6609\n",
      "Epoch 34/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.6561 - acc: 0.6642 - val_loss: 2.4826 - val_acc: 0.6595\n",
      "Epoch 35/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.6294 - acc: 0.6681 - val_loss: 2.4926 - val_acc: 0.6605\n",
      "Epoch 36/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.6038 - acc: 0.6715 - val_loss: 2.4993 - val_acc: 0.6598\n",
      "Epoch 37/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.5790 - acc: 0.6755 - val_loss: 2.5086 - val_acc: 0.6590\n",
      "Epoch 38/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.5545 - acc: 0.6787 - val_loss: 2.5171 - val_acc: 0.6600\n",
      "Epoch 39/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.5306 - acc: 0.6826 - val_loss: 2.5248 - val_acc: 0.6584\n",
      "Epoch 40/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.5075 - acc: 0.6862 - val_loss: 2.5339 - val_acc: 0.6580\n",
      "Epoch 41/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.4853 - acc: 0.6894 - val_loss: 2.5425 - val_acc: 0.6588\n",
      "Epoch 42/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.4636 - acc: 0.6931 - val_loss: 2.5513 - val_acc: 0.6572\n",
      "Epoch 43/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.4423 - acc: 0.6967 - val_loss: 2.5572 - val_acc: 0.6584\n",
      "Epoch 44/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.4216 - acc: 0.6999 - val_loss: 2.5661 - val_acc: 0.6573\n",
      "Epoch 45/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.4013 - acc: 0.7034 - val_loss: 2.5763 - val_acc: 0.6559\n",
      "Epoch 46/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.3817 - acc: 0.7069 - val_loss: 2.5846 - val_acc: 0.6569\n",
      "Epoch 47/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.3627 - acc: 0.7099 - val_loss: 2.5935 - val_acc: 0.6568\n",
      "Epoch 48/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.3433 - acc: 0.7138 - val_loss: 2.6036 - val_acc: 0.6554\n",
      "Epoch 49/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.3253 - acc: 0.7168 - val_loss: 2.6114 - val_acc: 0.6550\n",
      "Epoch 50/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.3071 - acc: 0.7198 - val_loss: 2.6144 - val_acc: 0.6558\n",
      "Epoch 51/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2895 - acc: 0.7231 - val_loss: 2.6266 - val_acc: 0.6552\n",
      "Epoch 52/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2721 - acc: 0.7261 - val_loss: 2.6341 - val_acc: 0.6557\n",
      "Epoch 53/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2554 - acc: 0.7294 - val_loss: 2.6429 - val_acc: 0.6563\n",
      "Epoch 54/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2384 - acc: 0.7321 - val_loss: 2.6492 - val_acc: 0.6554\n",
      "Epoch 55/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2225 - acc: 0.7349 - val_loss: 2.6567 - val_acc: 0.6559\n",
      "Epoch 56/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.2060 - acc: 0.7377 - val_loss: 2.6689 - val_acc: 0.6555\n",
      "Epoch 57/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1906 - acc: 0.7406 - val_loss: 2.6740 - val_acc: 0.6550\n",
      "Epoch 58/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1750 - acc: 0.7438 - val_loss: 2.6858 - val_acc: 0.6544\n",
      "Epoch 59/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1599 - acc: 0.7469 - val_loss: 2.6967 - val_acc: 0.6535\n",
      "Epoch 60/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1454 - acc: 0.7494 - val_loss: 2.7004 - val_acc: 0.6547\n",
      "Epoch 61/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1304 - acc: 0.7526 - val_loss: 2.7082 - val_acc: 0.6538\n",
      "Epoch 62/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1161 - acc: 0.7554 - val_loss: 2.7205 - val_acc: 0.6538\n",
      "Epoch 63/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.1020 - acc: 0.7583 - val_loss: 2.7232 - val_acc: 0.6544\n",
      "Epoch 64/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0880 - acc: 0.7605 - val_loss: 2.7354 - val_acc: 0.6539\n",
      "Epoch 65/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0743 - acc: 0.7635 - val_loss: 2.7439 - val_acc: 0.6531\n",
      "Epoch 66/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0605 - acc: 0.7664 - val_loss: 2.7504 - val_acc: 0.6536\n",
      "Epoch 67/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0473 - acc: 0.7691 - val_loss: 2.7626 - val_acc: 0.6528\n",
      "Epoch 68/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0347 - acc: 0.7719 - val_loss: 2.7676 - val_acc: 0.6532\n",
      "Epoch 69/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0216 - acc: 0.7746 - val_loss: 2.7776 - val_acc: 0.6523\n",
      "Epoch 70/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 1.0091 - acc: 0.7771 - val_loss: 2.7896 - val_acc: 0.6518\n",
      "Epoch 71/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.9964 - acc: 0.7798 - val_loss: 2.7954 - val_acc: 0.6517\n",
      "Epoch 72/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.9846 - acc: 0.7821 - val_loss: 2.8026 - val_acc: 0.6523\n",
      "Epoch 73/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.9724 - acc: 0.7849 - val_loss: 2.8128 - val_acc: 0.6517\n",
      "Epoch 74/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.9606 - acc: 0.7869 - val_loss: 2.8231 - val_acc: 0.6516\n",
      "Epoch 75/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.9485 - acc: 0.7898 - val_loss: 2.8289 - val_acc: 0.6530\n",
      "Epoch 76/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.9370 - acc: 0.7924 - val_loss: 2.8391 - val_acc: 0.6507\n",
      "Epoch 77/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.9257 - acc: 0.7944 - val_loss: 2.8482 - val_acc: 0.6512\n",
      "Epoch 78/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.9142 - acc: 0.7971 - val_loss: 2.8555 - val_acc: 0.6517\n",
      "Epoch 79/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.9033 - acc: 0.7999 - val_loss: 2.8686 - val_acc: 0.6498\n",
      "Epoch 80/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8930 - acc: 0.8021 - val_loss: 2.8752 - val_acc: 0.6512\n",
      "Epoch 81/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8817 - acc: 0.8041 - val_loss: 2.8855 - val_acc: 0.6487\n",
      "Epoch 82/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8714 - acc: 0.8066 - val_loss: 2.8887 - val_acc: 0.6511\n",
      "Epoch 83/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.8608 - acc: 0.8088 - val_loss: 2.9004 - val_acc: 0.6502\n",
      "Epoch 84/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8505 - acc: 0.8114 - val_loss: 2.9089 - val_acc: 0.6497\n",
      "Epoch 85/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.8404 - acc: 0.8135 - val_loss: 2.9176 - val_acc: 0.6510\n",
      "Epoch 86/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8305 - acc: 0.8155 - val_loss: 2.9263 - val_acc: 0.6493\n",
      "Epoch 87/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8204 - acc: 0.8180 - val_loss: 2.9356 - val_acc: 0.6499\n",
      "Epoch 88/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8103 - acc: 0.8201 - val_loss: 2.9422 - val_acc: 0.6492\n",
      "Epoch 89/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.8012 - acc: 0.8225 - val_loss: 2.9512 - val_acc: 0.6500\n",
      "Epoch 90/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7916 - acc: 0.8243 - val_loss: 2.9605 - val_acc: 0.6495\n",
      "Epoch 91/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.7825 - acc: 0.8262 - val_loss: 2.9714 - val_acc: 0.6498\n",
      "Epoch 92/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7732 - acc: 0.8290 - val_loss: 2.9825 - val_acc: 0.6482\n",
      "Epoch 93/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7642 - acc: 0.8307 - val_loss: 2.9899 - val_acc: 0.6482\n",
      "Epoch 94/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7556 - acc: 0.8326 - val_loss: 2.9985 - val_acc: 0.6482\n",
      "Epoch 95/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.7466 - acc: 0.8343 - val_loss: 3.0069 - val_acc: 0.6482\n",
      "Epoch 96/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7381 - acc: 0.8368 - val_loss: 3.0154 - val_acc: 0.6488\n",
      "Epoch 97/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.7293 - acc: 0.8388 - val_loss: 3.0197 - val_acc: 0.6491\n",
      "Epoch 98/100\n",
      "11214/11214 [==============================] - 12s 1ms/sample - loss: 0.7210 - acc: 0.8405 - val_loss: 3.0353 - val_acc: 0.6481\n",
      "Epoch 99/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.7123 - acc: 0.8424 - val_loss: 3.0419 - val_acc: 0.6477\n",
      "Epoch 100/100\n",
      "11214/11214 [==============================] - 11s 1ms/sample - loss: 0.7043 - acc: 0.8443 - val_loss: 3.0511 - val_acc: 0.6481\n"
     ]
    }
   ],
   "source": [
    "q2 = model2.fit(X_pad, y_pad, validation_split=0.1, batch_size=50, epochs=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LWb8Wdrfxafl"
   },
   "source": [
    "In this above solution, we implemented a Elman RNN model with BTT with one sentence (padded/truncated) of length 30. After 100 epochs of training, we end up with a cross entropy loss of 0.7043 for the training data.\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7HBqsMxFslcv"
   },
   "source": [
    "**Part 3**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "YSDDIgqpZxPz",
    "outputId": "52d427f4-4a4b-4853-a2a3-f0e9a9bd9a62"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_33\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_33 (Embedding)     (None, 30, 32)            356416    \n",
      "_________________________________________________________________\n",
      "gru_11 (GRU)                 (None, 30, 500)           799500    \n",
      "_________________________________________________________________\n",
      "time_distributed_11 (TimeDis (None, 30, 11138)         5580138   \n",
      "=================================================================\n",
      "Total params: 6,736,054\n",
      "Trainable params: 6,736,054\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model3 = tf.keras.Sequential()\n",
    "model3.add(tf.keras.layers.Embedding(vocabsize, 32, input_length=30))\n",
    "model3.add(tf.keras.layers.GRU(500, activation='sigmoid', return_sequences=True))\n",
    "model3.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(vocabsize, activation='softmax')))\n",
    "\n",
    "print(model3.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6HnE5YjRZxYL"
   },
   "outputs": [],
   "source": [
    "# sgd = tf.keras.optimizers.SGD(learning_rate=0.2, decay=1e-4, momentum=0.99)\n",
    "# adam = tf.keras.optimizers.Adam(learning_rate=0.1, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "model3.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "JeC_w9iYZxf_",
    "outputId": "69180632-7b40-41bd-9ef6-ebac39d78107"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11214 samples, validate on 1246 samples\n",
      "Epoch 1/100\n",
      "11214/11214 [==============================] - 19s 2ms/sample - loss: 3.5159 - acc: 0.5377 - val_loss: 2.5017 - val_acc: 0.6464\n",
      "Epoch 2/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.9636 - acc: 0.5676 - val_loss: 2.4290 - val_acc: 0.6524\n",
      "Epoch 3/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.8739 - acc: 0.5721 - val_loss: 2.3900 - val_acc: 0.6535\n",
      "Epoch 4/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.8001 - acc: 0.5747 - val_loss: 2.3633 - val_acc: 0.6547\n",
      "Epoch 5/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.7398 - acc: 0.5773 - val_loss: 2.3503 - val_acc: 0.6563\n",
      "Epoch 6/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.6913 - acc: 0.5807 - val_loss: 2.3363 - val_acc: 0.6589\n",
      "Epoch 7/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.6423 - acc: 0.5847 - val_loss: 2.3234 - val_acc: 0.6604\n",
      "Epoch 8/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.5944 - acc: 0.5874 - val_loss: 2.3126 - val_acc: 0.6608\n",
      "Epoch 9/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.5480 - acc: 0.5897 - val_loss: 2.3062 - val_acc: 0.6613\n",
      "Epoch 10/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.5041 - acc: 0.5916 - val_loss: 2.3024 - val_acc: 0.6612\n",
      "Epoch 11/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.4617 - acc: 0.5933 - val_loss: 2.3069 - val_acc: 0.6611\n",
      "Epoch 12/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.4200 - acc: 0.5947 - val_loss: 2.3098 - val_acc: 0.6616\n",
      "Epoch 13/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.3790 - acc: 0.5961 - val_loss: 2.3122 - val_acc: 0.6624\n",
      "Epoch 14/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.3376 - acc: 0.5978 - val_loss: 2.3156 - val_acc: 0.6619\n",
      "Epoch 15/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.2960 - acc: 0.5991 - val_loss: 2.3219 - val_acc: 0.6624\n",
      "Epoch 16/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.2545 - acc: 0.6007 - val_loss: 2.3320 - val_acc: 0.6626\n",
      "Epoch 17/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.2126 - acc: 0.6029 - val_loss: 2.3379 - val_acc: 0.6623\n",
      "Epoch 18/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.1685 - acc: 0.6055 - val_loss: 2.3455 - val_acc: 0.6628\n",
      "Epoch 19/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.1250 - acc: 0.6091 - val_loss: 2.3557 - val_acc: 0.6626\n",
      "Epoch 20/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 2.0805 - acc: 0.6139 - val_loss: 2.3679 - val_acc: 0.6621\n",
      "Epoch 21/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 2.0358 - acc: 0.6190 - val_loss: 2.3744 - val_acc: 0.6616\n",
      "Epoch 22/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.9914 - acc: 0.6242 - val_loss: 2.3870 - val_acc: 0.6609\n",
      "Epoch 23/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.9477 - acc: 0.6290 - val_loss: 2.3917 - val_acc: 0.6628\n",
      "Epoch 24/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.9050 - acc: 0.6340 - val_loss: 2.4056 - val_acc: 0.6620\n",
      "Epoch 25/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.8645 - acc: 0.6387 - val_loss: 2.4141 - val_acc: 0.6619\n",
      "Epoch 26/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.8245 - acc: 0.6435 - val_loss: 2.4204 - val_acc: 0.6617\n",
      "Epoch 27/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.7865 - acc: 0.6484 - val_loss: 2.4281 - val_acc: 0.6606\n",
      "Epoch 28/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.7501 - acc: 0.6535 - val_loss: 2.4385 - val_acc: 0.6600\n",
      "Epoch 29/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.7145 - acc: 0.6580 - val_loss: 2.4493 - val_acc: 0.6602\n",
      "Epoch 30/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.6796 - acc: 0.6627 - val_loss: 2.4542 - val_acc: 0.6602\n",
      "Epoch 31/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.6456 - acc: 0.6676 - val_loss: 2.4666 - val_acc: 0.6608\n",
      "Epoch 32/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.6134 - acc: 0.6722 - val_loss: 2.4774 - val_acc: 0.6599\n",
      "Epoch 33/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.5823 - acc: 0.6763 - val_loss: 2.4865 - val_acc: 0.6610\n",
      "Epoch 34/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.5515 - acc: 0.6811 - val_loss: 2.4956 - val_acc: 0.6584\n",
      "Epoch 35/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.5225 - acc: 0.6852 - val_loss: 2.5037 - val_acc: 0.6593\n",
      "Epoch 36/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.4924 - acc: 0.6902 - val_loss: 2.5163 - val_acc: 0.6582\n",
      "Epoch 37/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.4639 - acc: 0.6944 - val_loss: 2.5259 - val_acc: 0.6603\n",
      "Epoch 38/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.4366 - acc: 0.6988 - val_loss: 2.5367 - val_acc: 0.6588\n",
      "Epoch 39/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.4090 - acc: 0.7038 - val_loss: 2.5449 - val_acc: 0.6594\n",
      "Epoch 40/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.3829 - acc: 0.7082 - val_loss: 2.5563 - val_acc: 0.6582\n",
      "Epoch 41/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.3573 - acc: 0.7127 - val_loss: 2.5648 - val_acc: 0.6582\n",
      "Epoch 42/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.3317 - acc: 0.7171 - val_loss: 2.5791 - val_acc: 0.6581\n",
      "Epoch 43/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.3073 - acc: 0.7215 - val_loss: 2.5938 - val_acc: 0.6587\n",
      "Epoch 44/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.2827 - acc: 0.7260 - val_loss: 2.6020 - val_acc: 0.6582\n",
      "Epoch 45/100\n",
      "11214/11214 [==============================] - 17s 2ms/sample - loss: 1.2587 - acc: 0.7302 - val_loss: 2.6087 - val_acc: 0.6573\n",
      "Epoch 46/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.2362 - acc: 0.7342 - val_loss: 2.6206 - val_acc: 0.6564\n",
      "Epoch 47/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.2131 - acc: 0.7388 - val_loss: 2.6360 - val_acc: 0.6565\n",
      "Epoch 48/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.1908 - acc: 0.7431 - val_loss: 2.6492 - val_acc: 0.6569\n",
      "Epoch 49/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.1687 - acc: 0.7476 - val_loss: 2.6622 - val_acc: 0.6569\n",
      "Epoch 50/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.1467 - acc: 0.7516 - val_loss: 2.6734 - val_acc: 0.6554\n",
      "Epoch 51/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.1263 - acc: 0.7557 - val_loss: 2.6851 - val_acc: 0.6560\n",
      "Epoch 52/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.1055 - acc: 0.7601 - val_loss: 2.6967 - val_acc: 0.6555\n",
      "Epoch 53/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.0847 - acc: 0.7640 - val_loss: 2.7128 - val_acc: 0.6555\n",
      "Epoch 54/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.0646 - acc: 0.7686 - val_loss: 2.7231 - val_acc: 0.6543\n",
      "Epoch 55/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.0450 - acc: 0.7727 - val_loss: 2.7403 - val_acc: 0.6555\n",
      "Epoch 56/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.0258 - acc: 0.7766 - val_loss: 2.7473 - val_acc: 0.6546\n",
      "Epoch 57/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 1.0065 - acc: 0.7807 - val_loss: 2.7675 - val_acc: 0.6547\n",
      "Epoch 58/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9883 - acc: 0.7848 - val_loss: 2.7794 - val_acc: 0.6541\n",
      "Epoch 59/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9705 - acc: 0.7883 - val_loss: 2.7904 - val_acc: 0.6539\n",
      "Epoch 60/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9523 - acc: 0.7924 - val_loss: 2.8036 - val_acc: 0.6538\n",
      "Epoch 61/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9350 - acc: 0.7959 - val_loss: 2.8183 - val_acc: 0.6533\n",
      "Epoch 62/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9179 - acc: 0.7999 - val_loss: 2.8295 - val_acc: 0.6532\n",
      "Epoch 63/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.9011 - acc: 0.8041 - val_loss: 2.8465 - val_acc: 0.6535\n",
      "Epoch 64/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8845 - acc: 0.8074 - val_loss: 2.8538 - val_acc: 0.6530\n",
      "Epoch 65/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8681 - acc: 0.8111 - val_loss: 2.8752 - val_acc: 0.6538\n",
      "Epoch 66/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8521 - acc: 0.8146 - val_loss: 2.8851 - val_acc: 0.6529\n",
      "Epoch 67/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8363 - acc: 0.8182 - val_loss: 2.8985 - val_acc: 0.6524\n",
      "Epoch 68/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8210 - acc: 0.8219 - val_loss: 2.9125 - val_acc: 0.6532\n",
      "Epoch 69/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.8067 - acc: 0.8253 - val_loss: 2.9217 - val_acc: 0.6522\n",
      "Epoch 70/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7920 - acc: 0.8285 - val_loss: 2.9353 - val_acc: 0.6518\n",
      "Epoch 71/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7780 - acc: 0.8320 - val_loss: 2.9543 - val_acc: 0.6518\n",
      "Epoch 72/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7633 - acc: 0.8351 - val_loss: 2.9604 - val_acc: 0.6512\n",
      "Epoch 73/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7492 - acc: 0.8383 - val_loss: 2.9744 - val_acc: 0.6517\n",
      "Epoch 74/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7356 - acc: 0.8420 - val_loss: 2.9920 - val_acc: 0.6516\n",
      "Epoch 75/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7220 - acc: 0.8447 - val_loss: 3.0056 - val_acc: 0.6508\n",
      "Epoch 76/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.7094 - acc: 0.8475 - val_loss: 3.0143 - val_acc: 0.6509\n",
      "Epoch 77/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6969 - acc: 0.8506 - val_loss: 3.0333 - val_acc: 0.6506\n",
      "Epoch 78/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6846 - acc: 0.8538 - val_loss: 3.0391 - val_acc: 0.6498\n",
      "Epoch 79/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6723 - acc: 0.8568 - val_loss: 3.0575 - val_acc: 0.6494\n",
      "Epoch 80/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6599 - acc: 0.8597 - val_loss: 3.0677 - val_acc: 0.6497\n",
      "Epoch 81/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6487 - acc: 0.8625 - val_loss: 3.0825 - val_acc: 0.6499\n",
      "Epoch 82/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6370 - acc: 0.8651 - val_loss: 3.0998 - val_acc: 0.6496\n",
      "Epoch 83/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6259 - acc: 0.8674 - val_loss: 3.1081 - val_acc: 0.6493\n",
      "Epoch 84/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6153 - acc: 0.8704 - val_loss: 3.1242 - val_acc: 0.6498\n",
      "Epoch 85/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.6048 - acc: 0.8727 - val_loss: 3.1281 - val_acc: 0.6494\n",
      "Epoch 86/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5949 - acc: 0.8747 - val_loss: 3.1444 - val_acc: 0.6493\n",
      "Epoch 87/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5847 - acc: 0.8775 - val_loss: 3.1611 - val_acc: 0.6505\n",
      "Epoch 88/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5738 - acc: 0.8796 - val_loss: 3.1711 - val_acc: 0.6489\n",
      "Epoch 89/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5640 - acc: 0.8822 - val_loss: 3.1834 - val_acc: 0.6493\n",
      "Epoch 90/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5550 - acc: 0.8843 - val_loss: 3.1992 - val_acc: 0.6489\n",
      "Epoch 91/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5455 - acc: 0.8864 - val_loss: 3.2161 - val_acc: 0.6487\n",
      "Epoch 92/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5367 - acc: 0.8882 - val_loss: 3.2206 - val_acc: 0.6492\n",
      "Epoch 93/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5275 - acc: 0.8905 - val_loss: 3.2412 - val_acc: 0.6498\n",
      "Epoch 94/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5191 - acc: 0.8927 - val_loss: 3.2477 - val_acc: 0.6476\n",
      "Epoch 95/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5113 - acc: 0.8939 - val_loss: 3.2633 - val_acc: 0.6488\n",
      "Epoch 96/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.5031 - acc: 0.8959 - val_loss: 3.2697 - val_acc: 0.6485\n",
      "Epoch 97/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.4941 - acc: 0.8981 - val_loss: 3.2863 - val_acc: 0.6486\n",
      "Epoch 98/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.4866 - acc: 0.8998 - val_loss: 3.3002 - val_acc: 0.6491\n",
      "Epoch 99/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.4793 - acc: 0.9012 - val_loss: 3.3083 - val_acc: 0.6481\n",
      "Epoch 100/100\n",
      "11214/11214 [==============================] - 17s 1ms/sample - loss: 0.4713 - acc: 0.9030 - val_loss: 3.3191 - val_acc: 0.6484\n"
     ]
    }
   ],
   "source": [
    "q3 = model3.fit(X_pad, y_pad, validation_split=0.1, batch_size=50, epochs=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SCOl5kCUvdEp"
   },
   "source": [
    "In this above solution, we use GRU instead of a Simple RNN unit and after 100 epochs we have a cross entropy loss of 0.47 which is significantly less than the loss value observed with RNN model in part 2 with the same training conditions. RNN with TBTT (t=1) had a cross entropy loss of 5.59 (part 1), where as RNN with BTT had a cross entropy loss of 0.70. From these results, we can say that GRU is better than the first two RNN models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Epw-Pr2IW9tH"
   },
   "outputs": [],
   "source": [
    "# Creating inverse dictionaries for model 1 and model (2,3)\n",
    "\n",
    "inv_dict1 = {}\n",
    "for key in dictionary:\n",
    "  value = dictionary[key]\n",
    "  inv_dict1[value] = key\n",
    "\n",
    "inv_dict2 = {}\n",
    "for key in dictionary2:\n",
    "  value = dictionary2[key]\n",
    "  inv_dict2[value] = key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 867
    },
    "colab_type": "code",
    "id": "2moBkWFRW96F",
    "outputId": "096b9b98-4a7b-4919-d380-60c05dbbdfdd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sentence  1\n",
      "Model1: locks stop king i will have i will have i will have i will have \n",
      " Model2: locks her STOP paulina STOP for a while STOP so much for that by the \n",
      " Model3: locks me away STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP \n",
      "\n",
      " Sentence  2\n",
      "Model1: guard stop king i will have i will have i will have i will have \n",
      " Model2: guard you deserve to be the wish of my heart and in possession of his \n",
      " Model3: guard you go go and bring them not the sight of your will not seem \n",
      "\n",
      " Sentence  3\n",
      "Model1: well i will have i will have i will have i will have i will \n",
      " Model2: well i will walk you in STOP may not to be found again lament and \n",
      " Model3: well i am glad to hear you speak STOP STOP STOP STOP STOP to be \n",
      "\n",
      " Sentence  4\n",
      "Model1: shoulders and i will have i will have i will have i will have i \n",
      " Model2: shoulders with his blood STOP and make her airy tongue more hoarse than mine with \n",
      " Model3: shoulders not STOP for her honour STOP STOP and as i do see the world \n",
      "\n",
      " Sentence  5\n",
      "Model1: pronounce you stop king i will have i will have i will have i will \n",
      " Model2: pronounce this sentence then women may fall for rome were one of sweet no man \n",
      " Model3: pronounce that my father is lawful to my lord bid him word a fool STOP \n",
      "\n",
      " Sentence  6\n",
      "Model1: unless it stop king i will have i will have i will have i will \n",
      " Model2: unless the heavens have our voices that will be upon this STOP till they have \n",
      " Model3: unless philosophy can make a juliet displant a town reverse a prince doom it helps \n",
      "\n",
      " Sentence  7\n",
      "Model1: yourself stop king i will have i will have i will have i will have \n",
      " Model2: yourself STOP fast STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP STOP \n",
      " Model3: yourself and let him speak to our own authority we have no cause when is \n",
      "\n",
      " Sentence  8\n",
      "Model1: cheated of the lord stop king i will have i will have i will have \n",
      " Model2: cheated i warrant you STOP you STOP yourself STOP where the rest STOP that we \n",
      " Model3: cheated for the king STOP open the king STOP in STOP STOP STOP STOP STOP \n",
      "\n",
      " Sentence  9\n",
      "Model1: gems of the lord stop king i will have i will have i will have \n",
      " Model2: gems i warrant you to the sanctuary STOP to the ears STOP at all the \n",
      " Model3: gems for this night yield to make thee both and we will fly our thoughts \n",
      "\n",
      " Sentence  10\n",
      "Model1: fed the lord stop king i will have i will have i will have i \n",
      " Model2: fed the other closure of the world STOP whom the navel of the state and \n",
      " Model3: fed the guilty man STOP was to be so we are made to her STOP \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for times in range(10):\n",
    "  print(' Sentence ', times+1)\n",
    "  new_index = np.random.randint(0, 5000)\n",
    "  word1 = inv_dict1[new_index]\n",
    "  ni2 = dictionary2[word1]\n",
    "  ni3 = ni2\n",
    "  generated1 = ''\n",
    "  generated2 = ''\n",
    "  generated3 = ''\n",
    "  sentence1 = inv_dict1[new_index]\n",
    "  sentence2 = inv_dict2[ni2]\n",
    "  sentence3 = inv_dict2[ni3]\n",
    "  X_test1 = np.zeros((1,30))\n",
    "  X_test2 = np.zeros((1,30))\n",
    "  X_test3 = np.zeros((1,30))\n",
    "  for i in range(14):\n",
    "    X_test1[0,i] = new_index\n",
    "    X_test2[0,i] = ni2\n",
    "    X_test3[0,i] = ni3\n",
    "    preds1 = model1.predict([[new_index]])\n",
    "    preds2 = model2.predict(X_test2)\n",
    "    preds3 = model3.predict(X_test3)\n",
    "    new_index = preds1.argmax()\n",
    "    ni2 = preds2[0][i][:].argmax()\n",
    "    ni3 = preds3[0][i][:].argmax()\n",
    "    sentence1 = sentence1 + ' ' + inv_dict1[new_index]\n",
    "    sentence2 = sentence2 + ' ' + inv_dict2[ni2]\n",
    "    sentence3 = sentence3 + ' ' + inv_dict2[ni3]\n",
    "  print('Model1:', sentence1,'\\n','Model2:',sentence2,'\\n','Model3:',sentence3,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qpAQrNLogcPE"
   },
   "source": [
    "From the above experiment of generating sentences from the 3 implemented models, we can see that RNN model with TBTT(t=1) is very poor as it keeps repeating the same words that forms a sentence with no inherent meaning. The second model, the BTT RNN is better than the first as it produces sentences that do not at least repeat the same phrases. The third model, GRU, produces more meaningful sentences comparitively and is the best of the bunch. Although subjective, with respect to the meaningfulness and property of language of the generated sentences, the third model is better than the first two. Also, the second and the third model, tend to be similar to the training data to a certain extent, as we could see some word combinations that can be seen in the training corpus as well. This can be seen from the fact that the models are slightly overfit and hence the mimicking of the training corpus.\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0yHDIrO8gcKT"
   },
   "source": [
    "**Part 4**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ULH3acLegcCx"
   },
   "source": [
    "In order to evaluate an embedding obtained from the trained RNN model, there are 2 popular methods- Intrinsic evaluation and Extrinsic evaluation. \n",
    "\n",
    "Intrinsic evaluation is a comparitively cheaper method where we evaluate the embedding performance using a simple analogy-like game, where we input two words related in a specific way, and input a query word and the evaluation has to return a word that is related in the same way to the query word as the input words. The evaluation is a simple linear combination of the word embeddings of the three words, which makes this evaluation a computationally simpler approach.\n",
    "\n",
    "Extrinsic evaluation, on the other hand, typically is an elaborate process that focuses on a task broader than the subtasks in intrinsic evaluation and is a holistic evaluation of components that are part of the application. This is usually a more expensive approach and hence, we make use of intrinsic evaluation of the word embeddings to compare the performance of our models.  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vpsLm7_zgb6k"
   },
   "source": [
    "We can evaluate the models by using the word embeddings in an intrinsic evaluation. The pipeline is as follows:\n",
    "\n",
    "\n",
    "\n",
    "1.   Choose 2 words (eg: say, saying) and corresponding embeddings (e1, e2)\n",
    "2.   Choose a query word (eg: move) and its embedding (e3)\n",
    "\n",
    "1.   Calculate the embedding of the output like e4 = e1-e2+e3\n",
    "2.   If this embedding is close to the embedding of the expected word (in our case, moving), then we can say that the learned embeddings are good.\n",
    "\n",
    "So, in order to compare the models, we can calculate the distance (euclidean) of the predicted embedding with that of the actual embedding; and whichever model has the lowest distance value would be assumed to be the best embedding.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TG2Hs-_Egbdj"
   },
   "source": [
    "**Part 5**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "38ySylitdQ9E"
   },
   "outputs": [],
   "source": [
    "# !tar -xvf \"/content/drive/My Drive/Colab Notebooks/20Newsgroups_subsampled.tar.gz\" -C \"/content/drive/My Drive/Colab Notebooks/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "oPt8cNFKzIzS",
    "outputId": "bee91e7b-3c3a-4e7d-931d-aeecb11a1e7d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category:comp.graphics\n",
      "Category:comp.os.ms-windows.misc\n",
      "Category:comp.sys.ibm.pc.hardware\n",
      "Category:comp.sys.mac.hardware\n",
      "Category:comp.windows.x\n",
      "Category:talk.politics.guns\n",
      "Category:talk.politics.mideast\n",
      "Category:talk.politics.misc\n",
      "Category:rec.autos\n",
      "Category:rec.motorcycles\n",
      "Category:rec.sport.baseball\n",
      "Category:rec.sport.hockey\n",
      "Category:soc.religion.christian\n",
      "Category:talk.religion.misc\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# \"\"\"This code is used to read all news and their labels\"\"\"\n",
    "import os\n",
    "import glob\n",
    "\n",
    "def to_categories(name, cat=[\"politics\",\"rec\",\"comp\",\"religion\"]):\n",
    "    for i in range(len(cat)):\n",
    "        if str.find(name,cat[i])>-1:\n",
    "            return(i)\n",
    "    print(\"Unexpected folder: \" + name) # print the folder name which does not include expected categories\n",
    "    return(\"wth\")\n",
    "\n",
    "def data_loader(images_dir):\n",
    "    categories = os.listdir(data_path)\n",
    "    news = [] # news content\n",
    "    groups = [] # category which it belong to\n",
    "    \n",
    "    for cat in categories:\n",
    "        print(\"Category:\"+cat)\n",
    "        for the_new_path in glob.glob(data_path + '/' + cat + '/*'):\n",
    "            news.append(open(the_new_path,encoding = \"ISO-8859-1\", mode ='r').read())\n",
    "            groups.append(cat)\n",
    "\n",
    "    return news, list(map(to_categories, groups))\n",
    "\n",
    "\n",
    "\n",
    "# data_path = \"datasets/20news_subsampled\"\n",
    "data_path = \"/content/drive/My Drive/Colab Notebooks/20news_subsampled\"\n",
    "news, groups = data_loader(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2kCSJt57gPJj"
   },
   "outputs": [],
   "source": [
    "# Sentence tokenized\n",
    "\n",
    "e_st = []\n",
    "for row in news:\n",
    "  e=nltk.tokenize.sent_tokenize(row)\n",
    "  e_st.append(e)\n",
    "\n",
    "# Preprocessing (cleaning the data)\n",
    "\n",
    "e_new = []\n",
    "for mail in e_st:\n",
    "  mail_new=[]\n",
    "  for sent in mail:\n",
    "    w=word_tokenize(sent)\n",
    "    w.append('stop')\n",
    "    word_new = []\n",
    "    sent_new = []\n",
    "    for word in w:\n",
    "      if word.isalnum():\n",
    "        word_new.append(word.lower())\n",
    "    sent_new = ' '.join(wrd for wrd in word_new)   \n",
    "    mail_new.append(sent_new)\n",
    "  e_new.append(mail_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jMPM_lUo0UMy"
   },
   "outputs": [],
   "source": [
    "e_nn = []\n",
    "for mail in e_new:\n",
    "  mail_new=[]\n",
    "  mail_new.append(' '.join(snt for snt in mail))\n",
    "  e_nn.append(mail_new)\n",
    "\n",
    "e_final = np.squeeze(e_nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "wRsh7ykZgPwn",
    "outputId": "c964dc75-3064-47b4-bb1f-d26db0061aaa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the preprocessing\n",
      "['From: tsa@cellar.org (The Silent Assassin)\\nSubject: Questions about Plane Eqn method of Hidden Surface removal\\n\\nWhat are the main advantages of this method?', 'I have seen it described, and\\nthe algiorithm seems a  little bit long.', 'I developed my own method, which\\nrequires that the points be in a counter-clockwise order, and returns\\nwhether you are looking at the back or the front, similar to the plane eqn\\nmethod.', 'It uses few calculations however, basically it is several\\ncomparisons.', 'The only disadvantage I see is that it must be done after the\\ntransformation from view coordinates to coordinates to display on the\\nscreen, which means that a little more calculation isneeded beforehand.', 'My\\nmethod basically figures out whether the points that will appear on the\\nscreen are clockwise or counterclockwise.', 'When looking at the back of\\nsomething, the points occur in an opposite direction from the front, so\\nmerely by figuring out which way the points go, you can tell whether you are\\nlooking at the back or front of a 2d polygon.', 'Has anyone heard of this\\nmethod before?', 'It is so simple, I doubt i am the first to think of it.', 'Libertarian, atheist, semi-anarchal Techno-Rat.', 'I define myself--tsa@cellar.org']\n",
      "['from tsa the silent assassin subject questions about plane eqn method of hidden surface removal what are the main advantages of this method stop', 'i have seen it described and the algiorithm seems a little bit long stop', 'i developed my own method which requires that the points be in a order and returns whether you are looking at the back or the front similar to the plane eqn method stop', 'it uses few calculations however basically it is several comparisons stop', 'the only disadvantage i see is that it must be done after the transformation from view coordinates to coordinates to display on the screen which means that a little more calculation isneeded beforehand stop', 'my method basically figures out whether the points that will appear on the screen are clockwise or counterclockwise stop', 'when looking at the back of something the points occur in an opposite direction from the front so merely by figuring out which way the points go you can tell whether you are looking at the back or front of a 2d polygon stop', 'has anyone heard of this method before stop', 'it is so simple i doubt i am the first to think of it stop', 'libertarian atheist stop', 'i define myself tsa stop']\n",
      "['from tsa the silent assassin subject questions about plane eqn method of hidden surface removal what are the main advantages of this method stop i have seen it described and the algiorithm seems a little bit long stop i developed my own method which requires that the points be in a order and returns whether you are looking at the back or the front similar to the plane eqn method stop it uses few calculations however basically it is several comparisons stop the only disadvantage i see is that it must be done after the transformation from view coordinates to coordinates to display on the screen which means that a little more calculation isneeded beforehand stop my method basically figures out whether the points that will appear on the screen are clockwise or counterclockwise stop when looking at the back of something the points occur in an opposite direction from the front so merely by figuring out which way the points go you can tell whether you are looking at the back or front of a 2d polygon stop has anyone heard of this method before stop it is so simple i doubt i am the first to think of it stop libertarian atheist stop i define myself tsa stop']\n",
      "from tsa the silent assassin subject questions about plane eqn method of hidden surface removal what are the main advantages of this method stop i have seen it described and the algiorithm seems a little bit long stop i developed my own method which requires that the points be in a order and returns whether you are looking at the back or the front similar to the plane eqn method stop it uses few calculations however basically it is several comparisons stop the only disadvantage i see is that it must be done after the transformation from view coordinates to coordinates to display on the screen which means that a little more calculation isneeded beforehand stop my method basically figures out whether the points that will appear on the screen are clockwise or counterclockwise stop when looking at the back of something the points occur in an opposite direction from the front so merely by figuring out which way the points go you can tell whether you are looking at the back or front of a 2d polygon stop has anyone heard of this method before stop it is so simple i doubt i am the first to think of it stop libertarian atheist stop i define myself tsa stop\n"
     ]
    }
   ],
   "source": [
    "print('Testing the preprocessing')\n",
    "print(e_st[100])\n",
    "print(e_new[100])\n",
    "print(e_nn[100])\n",
    "print(e_final[100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "VO1EAyJDgRML",
    "outputId": "569c0810-ba7c-43b7-f272-3e70de715a55"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avglen =  283.84032651815687\n"
     ]
    }
   ],
   "source": [
    "# Checking the avg sentence length\n",
    "\n",
    "count = 0\n",
    "length = 0\n",
    "for sent in e_final:\n",
    "  count +=1\n",
    "  length += len(sent.split())\n",
    "print('avglen = ',length/count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IObADN7jgRhR"
   },
   "outputs": [],
   "source": [
    "# Word indexing\n",
    "\n",
    "dictionary5 = {}\n",
    "index = 0\n",
    "repeat = 0\n",
    "for sent in e_final:\n",
    "  w = word_tokenize(sent)\n",
    "  for word in w:\n",
    "    if word in dictionary5:\n",
    "      repeat+=1\n",
    "    else:\n",
    "      dictionary5[word] = index\n",
    "      index+=1\n",
    "vocabsize = index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "TuVuoT3zgRq_",
    "outputId": "240ebf0f-eb04-43f2-924c-cf40df82da2f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 20, 7, 8, 9, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 12, 26, 39, 5, 40, 28, 41, 12, 18, 42, 24, 43, 44, 45, 46, 47, 48, 39, 26, 49, 50, 12, 51, 5, 52, 53, 54, 55, 56, 57, 58, 59, 24, 60, 14, 61, 62, 63, 64, 0, 12, 50, 65, 42, 66, 67, 61, 68, 12, 18, 69, 70, 63, 71, 10, 28, 72, 25, 73, 58, 28, 74, 67, 53, 75, 12, 76, 77, 61, 71, 78, 12, 18, 79, 80, 81, 22, 82, 83, 18, 84, 85, 86, 87, 88, 61, 89, 63, 90, 91, 81, 92, 12, 1, 93, 94, 95, 32, 94, 96, 97, 98, 99, 100, 78, 101, 102, 103, 104, 105, 106, 0, 107, 108, 109, 110, 111, 112, 113, 3, 7, 114, 55, 115, 78, 20, 7, 114, 116, 117, 118, 119, 7, 114, 120, 121, 122, 12, 123, 63, 124, 20, 125, 10, 69, 126, 127, 12, 81, 123, 121, 128, 129, 69, 126, 18, 130, 12, 61, 131, 121, 132, 20, 133, 10, 126, 119, 67, 134, 112, 135, 61, 114, 12, 48, 39, 58, 136, 137, 138, 112, 43, 139, 12, 61, 140, 141, 63, 25, 18, 142, 61, 143, 144, 0, 61, 145, 123, 112, 146, 50, 147, 148, 12, 149, 94, 39, 86, 83, 84, 150, 151, 81, 63, 152, 153, 67, 12, 119, 7, 114, 150, 50, 63, 87, 39, 154, 58, 81, 155, 156, 55, 157, 158, 39, 159, 28, 160, 94, 161, 162, 12, 87, 39, 154, 163, 112, 164, 81, 165, 25, 166, 61, 167, 94, 168, 12, 23, 87, 61, 169, 94, 170, 53, 7, 171, 172, 173, 39, 0, 174, 175, 50, 12, 176, 48, 61, 177, 112, 178, 94, 61, 179, 113, 63, 88, 150, 180, 50, 112, 83, 129, 81, 22, 12, 153, 67, 12, 114, 63, 20, 69, 9, 25, 181, 39, 129, 182, 172, 10, 19, 183, 184, 32, 185, 23, 186, 187, 188, 77, 189, 190, 191, 23, 192, 193, 194, 195, 12, 195, 196, 197, 182, 198, 12, 114, 199, 200, 7, 201, 37, 23, 28, 202, 161, 129, 182, 203, 0, 204, 53, 165, 12, 114, 42, 199, 200, 205, 206, 129, 207, 208, 23, 209, 210, 48, 39, 211, 212, 20, 213, 83, 214, 81, 63, 197, 215, 216, 10, 149, 86, 84, 152, 50, 12, 61, 131, 25, 217, 129, 50, 218, 219, 112, 220, 150, 50, 151, 221, 222, 67, 12, 114, 181, 20, 223, 224, 112, 225, 53, 226, 161, 172, 129, 227, 228, 39, 26, 135, 114, 112, 83, 61, 7, 229, 23, 53, 226, 172, 112, 83, 61, 230, 161, 12, 231, 25, 114, 42, 181, 161, 172, 228, 39, 83, 58, 112, 214, 152, 25, 48, 39, 83, 163, 112, 114, 232, 55, 233, 234, 12, 235, 10, 61, 236, 23, 237, 238, 179, 239, 12, 61, 140, 123, 240, 241, 23, 128, 67, 178, 11, 12, 207, 234, 242, 71, 12, 114, 232, 129, 243, 244, 245, 25, 246, 39, 61, 247, 94, 7, 8, 23, 134, 112, 135, 61, 22, 12, 48, 248, 63, 249, 250, 18, 218, 251, 61, 114, 112, 243, 9, 12, 252, 18, 253, 25, 61, 160, 50, 63, 254, 121, 255, 50, 256, 112, 135, 10, 20, 257, 258, 259, 260, 61, 261, 12, 114, 63, 138, 10, 178, 262, 12, 153, 61, 263, 264, 12, 265, 107, 110, 12], [0, 266, 267, 268, 3, 269, 94, 270, 271, 152, 20, 272, 273, 18, 274, 275, 67, 20, 276, 25, 277, 112, 66, 278, 182, 23, 279, 280, 112, 66, 281, 282, 12, 18, 283, 284, 48, 85, 67, 61, 285, 180, 286, 81, 276, 23, 287, 28, 288, 289, 112, 50, 12, 61, 276, 63, 112, 290, 243, 291, 94, 20, 271, 292, 293, 233, 294, 209, 295, 296, 12, 10, 297, 61, 276, 298, 299, 129, 233, 300, 94, 301, 158, 302, 303, 94, 301, 63, 67, 61, 304, 94, 243, 305, 306, 307, 308, 32, 233, 194, 12, 309, 310, 311, 61, 312, 88, 306, 313, 55, 193, 194, 314, 12, 112, 290, 61, 305, 307, 98, 315, 20, 295, 270, 316, 61, 301, 12, 302, 270, 63, 317, 318, 32, 20, 319, 94, 34, 320, 321, 61, 322, 12, 302, 270, 180, 243, 323, 324, 94, 320, 12, 307, 325, 25, 61, 326, 327, 328, 302, 270, 63, 294, 88, 196, 329, 61, 59, 330, 328, 302, 270, 293, 178, 331, 332, 12, 333, 334, 302, 327, 335, 328, 302, 270, 23, 336, 50, 112, 61, 326, 327, 94, 61, 337, 270, 12, 135, 61, 338, 94, 320, 112, 339, 233, 340, 306, 218, 66, 341, 55, 112, 342, 61, 271, 12, 343, 344, 345, 61, 340, 346, 334, 302, 347, 55, 348, 349, 75, 350, 20, 351, 352, 290, 23, 342, 61, 347, 12, 334, 61, 304, 94, 61, 347, 25, 63, 328, 61, 304, 94, 61, 271, 23, 290, 20, 34, 328, 25, 304, 353, 129, 20, 354, 304, 342, 50, 63, 61, 304, 355, 55, 356, 346, 25, 63, 357, 61, 358, 94, 61, 276, 12, 61, 359, 63, 112, 317, 291, 61, 304, 94, 61, 271, 12, 61, 285, 360, 252, 361, 180, 362, 94, 363, 23, 364, 55, 61, 304, 94, 61, 271, 12, 61, 365, 10, 81, 242, 366, 367, 12, 368, 369, 207, 370, 136, 23, 136, 371, 0, 61, 372, 61, 296, 12, 48, 373, 152, 81, 276, 374, 375, 307, 376, 377, 378, 152, 140, 275, 55, 81, 379, 12, 12], [0, 380, 381, 382, 3, 4, 383, 384, 55, 385, 386, 387, 386, 15, 0, 386, 387, 386, 3, 4, 383, 384, 102, 388, 97, 106, 55, 385, 389, 390, 391, 392, 15, 393, 394, 61, 395, 383, 384, 18, 396, 283, 196, 55, 393, 397, 94, 175, 81, 12, 18, 398, 243, 399, 257, 152, 151, 400, 94, 393, 8, 293, 401, 81, 402, 0, 403, 112, 404, 112, 393, 405, 12, 18, 197, 69, 406, 104, 407, 408, 280, 94, 81, 393, 63, 20, 409, 23, 218, 309, 360, 55, 20, 410, 411, 20, 412, 413, 402, 12, 393, 18, 414, 94, 415, 61, 416, 94, 417, 235, 249, 418, 10, 393, 419, 151, 400, 94, 8, 12, 85, 420, 137, 81, 160, 12, 393, 197, 421, 12, 18, 422, 423, 12, 70, 63, 20, 424, 324, 94, 12, 425, 426, 12, 55, 427, 70, 242, 278, 410, 428, 306, 429, 430, 431, 235, 94, 432, 433, 12, 42, 50, 63, 434, 435, 25, 61, 436, 437, 94, 438, 408, 63, 426, 439, 129, 440, 61, 376, 196, 55, 151, 441, 442, 152, 20, 258, 443, 444, 12, 387, 386, 18, 445, 83, 228, 129, 446, 447, 448, 94, 449, 450, 46, 39, 58, 451, 10, 81, 39, 218, 66, 452, 50, 12, 453, 454, 94, 455, 18, 65, 456, 280, 25, 307, 242, 196, 457, 152, 12, 309, 12, 382, 42, 10, 61, 196, 458, 459, 70, 63, 20, 460, 94, 461, 462, 86, 463, 464, 112, 61, 465, 466, 467, 12, 368, 242, 468, 12, 23, 469, 470, 12, 471, 39, 472, 473, 474, 475, 32, 471, 32, 39, 472, 20, 476, 477, 478, 150, 479, 480, 12, 413, 481, 61, 482, 138, 483, 0, 484, 381, 382, 380, 12], [0, 485, 486, 487, 3, 488, 20, 489, 23, 490, 18, 91, 491, 26, 488, 139, 129, 61, 62, 276, 18, 492, 70, 422, 66, 20, 493, 494, 12, 495, 20, 489, 292, 293, 496, 496, 23, 497, 497, 158, 496, 23, 497, 242, 498, 23, 20, 490, 292, 293, 499, 23, 500, 158, 499, 500, 20, 501, 126, 485, 242, 502, 23, 12, 196, 209, 503, 196, 323, 112, 150, 63, 61, 379, 94, 504, 505, 12, 150, 18, 506, 63, 28, 507, 508, 509, 10, 151, 510, 23, 511, 25, 26, 66, 512, 293, 61, 179, 513, 12, 357, 61, 514, 50, 515, 415, 18, 516, 112, 58, 112, 517, 357, 151, 518, 519, 23, 520, 302, 521, 12, 98, 489, 522, 490, 12, 333, 490, 522, 489, 12, 343, 233, 523, 94, 490, 524, 489, 497, 497, 12, 12, 525, 496, 346, 233, 523, 94, 490, 526, 489, 18, 91, 491, 26, 488, 12, 527, 51, 55, 528, 529, 487, 12, 530, 26, 39, 5, 531, 139, 178, 532, 293, 12], [0, 533, 533, 534, 535, 536, 537, 3, 8, 9, 538, 10, 539, 23, 540, 307, 242, 541, 10, 538, 71, 403, 306, 542, 543, 293, 544, 8, 545, 12, 307, 376, 415, 112, 546, 50, 67, 540, 23, 539, 12, 48, 70, 63, 85, 426, 547, 538, 67, 539, 23, 540, 5, 548, 12, 42, 5, 548, 48, 85, 549, 158, 61, 71, 403, 63, 122, 12, 550, 39, 533, 551, 12], [0, 552, 553, 3, 4, 233, 554, 555, 55, 235, 12, 60, 556, 557, 558, 55, 385, 559, 560, 561, 562, 15, 63, 70, 178, 160, 112, 563, 233, 554, 555, 112, 235, 564, 565, 12, 18, 58, 566, 81, 88, 18, 567, 368, 376, 568, 129, 302, 140, 12, 569, 48, 309, 235, 357, 20, 570, 376, 66, 80, 12, 197, 571, 243, 572, 10, 573, 12, 517, 55, 574, 575, 12, 368, 242, 122, 366, 576, 12, 368, 577, 578, 311, 233, 564, 555, 67, 20, 579, 565, 12, 580, 61, 581, 582, 112, 135, 243, 583, 23, 243, 584, 585, 228, 20, 572, 63, 516, 112, 488, 473, 12, 48, 39, 242, 586, 53, 587, 588, 10, 373, 420, 197, 589, 473, 67, 407, 587, 588, 12, 589, 53, 583, 585, 112, 66, 590, 591, 23, 53, 584, 585, 112, 66, 592, 12, 39, 593, 65, 58, 112, 214, 152, 594, 595, 596, 39, 597, 66, 586, 207, 598, 357, 61, 59, 570, 12, 553, 599, 55, 600, 218, 601, 602, 50, 552, 603, 61, 604, 605, 94, 606, 238, 602, 491, 607, 608, 67, 609, 28, 610, 275, 611, 12], [0, 612, 613, 3, 614, 615, 243, 616, 617, 75, 61, 616, 75, 618, 614, 615, 61, 619, 55, 20, 620, 621, 94, 622, 623, 624, 625, 112, 626, 61, 627, 94, 61, 628, 12, 81, 621, 94, 629, 443, 630, 218, 631, 112, 632, 633, 634, 635, 636, 135, 94, 637, 638, 639, 640, 641, 642, 643, 623, 23, 569, 644, 112, 645, 646, 647, 55, 648, 23, 649, 650, 12, 651, 0, 151, 652, 653, 242, 654, 12, 615, 218, 66, 443, 655, 12, 70, 218, 66, 193, 656, 94, 657, 12, 61, 619, 658, 218, 659, 94, 151, 660, 661, 235, 646, 275, 32, 20, 662, 663, 25, 233, 140, 664, 218, 55, 665, 666, 12, 151, 619, 658, 667, 668, 242, 669, 293, 670, 73, 671, 100, 672, 673, 674, 570, 12, 658, 233, 218, 675, 67, 73, 676, 12, 151, 668, 218, 66, 677, 112, 678, 679, 680, 681, 10, 473, 112, 83, 682, 67, 55, 683, 368, 684, 315, 12, 10, 685, 686, 687, 81, 73, 659, 94, 688, 61, 689, 316, 504, 690, 691, 165, 12, 10, 692, 687, 81, 73, 176, 66, 693, 20, 694, 94, 61, 689, 23, 695, 504, 226, 696, 129, 697, 698, 238, 699, 700, 55, 61, 689, 701, 112, 66, 702, 67, 112, 61, 703, 12, 151, 704, 242, 669, 293, 672, 67, 670, 73, 705, 12, 658, 193, 63, 61, 706, 658, 23, 218, 675, 67, 73, 707, 12, 151, 687, 218, 83, 150, 368, 69, 50, 708, 112, 709, 710, 61, 689, 368, 711, 495, 586, 712, 713, 368, 684, 32, 714, 12, 432, 668, 218, 66, 669, 67, 715, 716, 672, 12, 151, 717, 218, 718, 55, 302, 658, 94, 719, 12, 112, 720, 721, 129, 61, 62, 44, 45, 47, 46, 638, 46, 722, 723, 12, 724, 723, 12, 574, 723, 12, 725, 726, 12, 727, 8, 728, 12, 12, 729, 723, 12, 570, 730, 12, 46, 61, 731, 112, 47, 732, 613, 75, 733, 607, 615, 55, 61, 3, 34, 638, 61, 616, 75, 614, 615, 734, 735, 736, 737, 738, 739, 151, 662, 668, 242, 669, 67, 670, 73, 671, 12, 740, 473, 357, 741, 94, 432, 729, 742, 238, 238, 721, 743, 668, 112, 744, 745, 47, 238, 721, 746, 112, 46, 179, 238, 747, 639, 748, 12, 48, 39, 83, 571, 53, 662, 689, 55, 39, 597, 66, 749, 112, 718, 602, 614, 750, 751, 55, 12, 307, 218, 83, 752, 753, 112, 689, 748, 10, 61, 754, 755, 12, 48, 39, 83, 58, 723, 112, 20, 722, 88, 24, 235, 307, 219, 112, 488, 12, 48, 39, 24, 243, 689, 756, 280, 23, 677, 112, 39, 757, 744, 84, 721, 243, 758, 48, 39, 759, 81, 12, 760, 731, 122, 67, 61, 62, 761, 10, 135, 94, 762, 763, 31, 32, 640, 764, 637, 765, 23, 644, 639, 766, 767, 768, 769, 770, 689, 771, 61, 616, 75, 55, 507, 612, 613, 772, 39, 773, 774, 293, 775, 12, 773, 684, 776, 734, 735, 736, 357, 61, 777, 12, 18, 376, 415, 112, 472, 0, 39, 737, 738, 152, 178, 778, 779, 39, 58, 780, 12], [0, 781, 391, 781, 3, 4, 782, 136, 55, 385, 783, 784, 785, 786, 787, 15, 788, 789, 152, 782, 136, 772, 790, 55, 28, 383, 12, 791, 63, 28, 792, 127, 94, 782, 793, 49, 794, 12, 18, 58, 42, 795, 796, 10, 797, 782, 139, 10, 473, 12, 798, 10, 314, 799, 800, 25, 782, 63, 243, 801, 802, 165, 23, 25, 406, 803, 804, 112, 61, 805, 806, 63, 807, 808, 94, 809, 810, 12, 55, 811, 349, 98, 219, 812, 55, 61, 813, 112, 572, 50, 216, 12, 699, 146, 53, 814, 12, 699, 815, 67, 816, 12, 70, 63, 817, 276, 32, 818, 50, 180, 426, 711, 819, 48, 39, 517, 820, 12, 333, 821, 782, 63, 243, 801, 802, 165, 12, 81, 63, 196, 20, 822, 12, 39, 26, 823, 824, 825, 826, 238, 83, 827, 828, 829, 830, 831, 38, 12, 832, 61, 165, 422, 833, 280, 61, 689, 420, 151, 53, 275, 376, 66, 834, 835, 12, 228, 821, 50, 836, 280, 801, 802, 668, 94, 837, 12, 782, 26, 838, 839, 802, 668, 23, 840, 473, 841, 112, 801, 842, 12, 81, 63, 20, 843, 844, 196, 20, 822, 12, 134, 376, 39, 845, 175, 828, 846, 67, 20, 839, 802, 847, 12, 134, 376, 39, 402, 848, 112, 823, 473, 849, 12, 309, 827, 829, 850, 66, 611, 851, 61, 852, 772, 278, 407, 23, 853, 258, 367, 12, 48, 39, 163, 112, 83, 828, 846, 67, 20, 839, 802, 689, 39, 24, 853, 258, 854, 852, 306, 63, 434, 122, 855, 12, 23, 856, 857, 63, 20, 858, 508, 12, 50, 26, 66, 859, 112, 178, 689, 94, 189, 802, 344, 12, 701, 81, 63, 196, 20, 822, 12, 50, 63, 20, 160, 94, 183, 860, 314, 222, 391, 781, 652, 861, 862, 574, 8, 23, 863, 864, 75, 574, 8, 865, 866, 867, 868, 869, 870, 866, 871, 12, 872, 873, 450, 874, 875, 876, 877, 878, 639, 875, 876, 877, 879, 880, 12], [0, 881, 882, 881, 3, 883, 884, 668, 431, 885, 886, 887, 18, 83, 196, 84, 48, 18, 283, 888, 61, 90, 889, 408, 238, 196, 178, 488, 55, 61, 90, 890, 63, 258, 260, 654, 12, 18, 24, 488, 891, 20, 892, 25, 218, 334, 884, 668, 642, 55, 23, 893, 293, 894, 895, 23, 665, 473, 431, 885, 896, 238, 887, 12, 18, 283, 541, 10, 20, 897, 32, 898, 112, 899, 900, 32, 518, 12, 61, 668, 218, 66, 901, 67, 20, 902, 903, 904, 905, 894, 895, 23, 218, 699, 66, 906, 55, 712, 726, 61, 907, 908, 909, 910, 911, 12, 61, 668, 218, 66, 912, 23, 913, 914, 642, 55, 129, 20, 915, 916, 722, 699, 293, 894, 895, 12, 48, 85, 850, 488, 139, 238, 569, 917, 139, 918, 919, 112, 259, 86, 850, 18, 376, 66, 278, 920, 12, 42, 48, 85, 420, 63, 175, 150, 18, 283, 921, 18, 376, 66, 922, 112, 472, 0, 39, 129, 178, 923, 39, 793, 924, 32, 112, 61, 574, 925, 39, 135, 178, 926, 238, 852, 12, 50, 927, 61, 903, 904, 376, 66, 446, 753, 928, 112, 929, 884, 668, 12, 18, 283, 921, 67, 930, 20, 903, 904, 129, 931, 94, 932, 20, 933, 934, 935, 20, 915, 916, 722, 936, 937, 938, 23, 20, 939, 940, 935, 23, 909, 20, 941, 942, 12, 18, 283, 943, 112, 944, 23, 178, 923, 376, 66, 155, 12, 882, 357, 881, 238, 12], [0, 945, 946, 947, 3, 4, 948, 949, 950, 951, 952, 15, 953, 158, 50, 954, 955, 956, 95, 50, 197, 954, 473, 55, 955, 436, 78, 12, 18, 58, 112, 957, 112, 958, 67, 81, 456, 32, 61, 959, 847, 18, 135, 112, 960, 948, 941, 112, 61, 961, 158, 948, 962, 23, 699, 963, 50, 12, 964, 570, 18, 965, 948, 61, 956, 847, 63, 966, 55, 61, 967, 961, 94, 61, 935, 948, 63, 67, 12, 197, 446, 945, 946, 947, 968, 242, 39, 541, 10, 258, 969, 970, 39, 945, 83, 196, 971, 972, 112, 150, 39, 426, 84, 12, 945, 571, 280, 94, 446, 973, 12, 974, 112, 975, 976, 12]]\n",
      "(13108,) (13108,)\n"
     ]
    }
   ],
   "source": [
    "# Preparing the data (indexed numbers)\n",
    "\n",
    "data= []\n",
    "temp_data = []\n",
    "\n",
    "for sent in e_final:\n",
    "  w=word_tokenize(sent)\n",
    "  temp_data=[]\n",
    "  for word in w:\n",
    "    temp_data.append(dictionary5[word])\n",
    "  data.append(temp_data)\n",
    "# print(data)\n",
    "print(data[0:10])\n",
    "print(np.shape(data), np.shape(groups))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "GHKBZfAk6Jgv",
    "outputId": "0dca5bd9-eede-4638-dd9e-245e0620f51e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13108, 300)\n"
     ]
    }
   ],
   "source": [
    "# Padding the inputs\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "data_pad=pad_sequences(data, maxlen=300,padding='post',truncating='post',value=0.0)\n",
    "print(np.shape(data_pad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aFO2eP3lK1Wi"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_pad, groups, test_size=0.1, random_state=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "I3gkw_gk6kKU",
    "outputId": "e6aca5c6-7e63-4cfc-9c44-907b05066960"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_31\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_31 (Embedding)     (None, 300, 32)           3181984   \n",
      "_________________________________________________________________\n",
      "gru_10 (GRU)                 (None, 500)               799500    \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 4)                 2004      \n",
      "=================================================================\n",
      "Total params: 3,983,488\n",
      "Trainable params: 3,983,488\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Model\n",
    "\n",
    "model5 = tf.keras.Sequential()\n",
    "model5.add(tf.keras.layers.Embedding(vocabsize, 32, input_length=300))\n",
    "model5.add(tf.keras.layers.GRU(500, activation='sigmoid'))\n",
    "model5.add(tf.keras.layers.Dense(4, activation='softmax'))\n",
    "\n",
    "print(model5.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bWgEl9I88gs5"
   },
   "outputs": [],
   "source": [
    "# sgd = tf.keras.optimizers.SGD(learning_rate=0.2, decay=1e-4, momentum=0.99)\n",
    "# adam = tf.keras.optimizers.Adam(learning_rate=0.01, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "model5.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "Ytmwc1Co8sAz",
    "outputId": "3b455e74-93ca-4b48-dff6-292bb7398594"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11797 samples, validate on 1311 samples\n",
      "Epoch 1/5\n",
      "11797/11797 [==============================] - 113s 10ms/sample - loss: 1.2900 - acc: 0.3849 - val_loss: 1.2830 - val_acc: 0.4066\n",
      "Epoch 2/5\n",
      "11797/11797 [==============================] - 110s 9ms/sample - loss: 1.1727 - acc: 0.4464 - val_loss: 1.0038 - val_acc: 0.5446\n",
      "Epoch 3/5\n",
      "11797/11797 [==============================] - 111s 9ms/sample - loss: 0.6159 - acc: 0.7537 - val_loss: 0.5236 - val_acc: 0.8139\n",
      "Epoch 4/5\n",
      "11797/11797 [==============================] - 111s 9ms/sample - loss: 0.2844 - acc: 0.9079 - val_loss: 0.3809 - val_acc: 0.8688\n",
      "Epoch 5/5\n",
      "11797/11797 [==============================] - 111s 9ms/sample - loss: 0.1666 - acc: 0.9496 - val_loss: 0.3097 - val_acc: 0.8963\n"
     ]
    }
   ],
   "source": [
    "q5 = model5.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=50, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TIugMFIi9F95"
   },
   "source": [
    "Validation accuracy = 89.63%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MMbUpTKxn9wH"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Assignment Three.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}