Sent_Similarity_Word2vec_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Moseli Motsoehli\n",
    "\n",
    "### Siamese LSTM for sentence similarity."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__I use the quora question pairs dataset as a test for this although the model is eventually used for text transcribed from recorded speech as a way to evaluate  public speaking__\n",
    "\n",
    "Siamese lstm to classify similar senrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/moselim/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#### Libraries\n",
    "import scipy as sc\n",
    "from scipy.spatial.distance import cosine\n",
    "import pandas as pd\n",
    "import math\n",
    "import logging\n",
    "from collections import Counter\n",
    "from  tqdm import tqdm\n",
    "from tabulate import tabulate\n",
    "from Generator import *\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.model_selection import train_test_split as tts\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "import os\n",
    "import re\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize as wt\n",
    "from nltk.tokenize import sent_tokenize as st\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import backend as k\n",
    "from keras import initializers\n",
    "from keras.optimizers import RMSprop,Adam,Adagrad\n",
    "from keras.models import Model,Sequential,load_model\n",
    "from keras.layers import Dense,LSTM,Input,Activation,Flatten,concatenate,Reshape,Embedding,Bidirectional,Dropout\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.models import load_model\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import LearningRateScheduler,EarlyStopping\n",
    "\n",
    "#model Freezing\n",
    "import tensorflow as tf\n",
    "# freeze_graph \"screenshots\" the graph\n",
    "from tensorflow.python.tools import freeze_graph\n",
    "# optimize_for_inference lib optimizes this frozen graph\n",
    "from tensorflow.python.tools import optimize_for_inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "STRING_SIZE=16\n",
    "BATCH_SIZE=16\n",
    "NUM_EPOCHS = 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of examples: 404289\n",
      "Number of examples after removing NA: 404286\n",
      "\n",
      "\n",
      "   id  qid1  qid2                                          question1  \\\n",
      "0   0     1     2  What is the step by step guide to invest in sh...   \n",
      "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
      "\n",
      "                                           question2  is_duplicate  \n",
      "0  What is the step by step guide to invest in sh...             0  \n",
      "1  What would happen if the Indian government sto...             0  \n",
      "\n",
      "\n",
      "            id    qid1    qid2  \\\n",
      "404284  404287  537928  537929   \n",
      "404285  404288  537930  537931   \n",
      "\n",
      "                                                question1  \\\n",
      "404284                                  What is one coin?   \n",
      "404285  What is the approx annual cost of living while...   \n",
      "\n",
      "                                                question2  is_duplicate  \n",
      "404284                                  What's this coin?             0  \n",
      "404285  I am having little hairfall problem but I want...             0  \n",
      "\n",
      "\n",
      "propotion of positives: 0.36920150586466016\n"
     ]
    }
   ],
   "source": [
    "datafile = \"train.csv\"\n",
    "\n",
    "data =  pd.read_csv(datafile,sep=\",\")[:][:-1]\n",
    "print(\"Number of examples: %s\"%len(data))\n",
    "data=data.dropna(subset=['question1','question2', 'is_duplicate'])\n",
    "data=data.reset_index(drop=True)\n",
    "print(\"Number of examples after removing NA: %s\"%len(data))\n",
    "print(\"\\n\")\n",
    "print(data.head(2))\n",
    "print(\"\\n\")\n",
    "print(data.tail(2))\n",
    "print(\"\\n\")\n",
    "print(\"propotion of positives: %s\"%np.mean(data[\"is_duplicate\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing addopted from my Topic Model in ICS661 AKA Thematron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_special(text):\n",
    "    special_chars = \"[~#$%^@&*&()+-_\\\",?!.:[]\\\\;><`|{}=\\\\'=»¿シし]\"\n",
    "    for k in special_chars:\n",
    "            if type(text) == str:\n",
    "                if k==\"-\" or k==\"_\":\n",
    "                    text=text.replace(k, \"\")\n",
    "                else:\n",
    "                    text=text.replace(k, \" \")       \n",
    "    return text\n",
    "\n",
    "def contractions(sent):\n",
    "    sub_pattern = [(\"will not\",\"won't\"),(\"shall not\",\"shan't\"),\n",
    "                (\" not\", \"n\\'t\"),(\" will\",\"\\'ll\"),(\" is\",\"\\'s\"),\n",
    "                   (\" am\",\"\\'m\"),(\" are\",\"\\'re\"),(\" is\",\"who\\'s\")]\n",
    "    sent2=sent.split(\" \")\n",
    "    hold = \"\"\n",
    "    for k in range(len(sent2)):\n",
    "        kk = sent2[k]\n",
    "        for rep in range(len(sub_pattern)):\n",
    "            kk = re.sub(sub_pattern[rep][1],sub_pattern[rep][0],kk)\n",
    "        hold = hold + \" \" + kk\n",
    "    return hold.lower()\n",
    "\n",
    "def remove_stop_words(text):\n",
    "    stops =  set(stopwords.words('english'))\n",
    "    stops1 = [word.lower() for word in stops]\n",
    "    punctuation = [',','.','!','?',';','-']\n",
    "    hold = []\n",
    "    if type(text) == list:\n",
    "        for word in range(len(text)):\n",
    "            if text[word].lower() in stops1 or text[word].lower() in punctuation or text[word].lower() == \"xxxxxx\":\n",
    "                continue\n",
    "            else:\n",
    "                hold.append(text[word].lower())\n",
    "    return hold\n",
    "\n",
    "def wordTok(sent):\n",
    "    tok = wt(sent)\n",
    "    return tok\n",
    "\n",
    "def pipeline(text):\n",
    "    text = contractions(text)\n",
    "    text = remove_special(text)\n",
    "    textToks = wordTok(text)\n",
    "    #textToks = remove_stop_words(textToks)\n",
    "    final = \"\"\n",
    "    for k in range(len(textToks)):\n",
    "            final = final+textToks[k]+\" \"\n",
    "    return final.strip().lower()\n",
    "\n",
    "########################################################\n",
    "########create corpus and create word vectors and training data###########\n",
    "def createCorpus(t):\n",
    "    corpus = []\n",
    "    all_sent = []\n",
    "    for k in t:\n",
    "        for p in t[k]:\n",
    "            corpus.append(st(p))\n",
    "    for sent in range(len(corpus)):\n",
    "        for k in corpus[sent]:\n",
    "            all_sent.append(k)\n",
    "    for m in range(len(all_sent)):\n",
    "        all_sent[m] = wt(all_sent[m])\n",
    "    \n",
    "    all_words=[]\n",
    "    for sent in all_sent:\n",
    "        hold=[]\n",
    "        for word in sent:\n",
    "            hold.append(word.lower())\n",
    "        all_words.append(hold)\n",
    "    return all_words\n",
    "\n",
    "def wordvecmatrix(model1,data):\n",
    "    IO_data={\"question1\":[],\"question2\":[],\"label\":[]}\n",
    "    pbar = tqdm(range(len(data[\"question1\"])))\n",
    "    for k in range(len(data[\"question1\"])):\n",
    "        q1=[]\n",
    "        q2=[]\n",
    "        label=[]\n",
    "        for word in data[\"question1\"][k]:\n",
    "            try:\n",
    "                q1.append(model1.wv.word_vec(word))\n",
    "            except Exception as e:\n",
    "                continue\n",
    "        for word in data[\"question2\"][k]:\n",
    "            try:\n",
    "                q2.append(model1.wv.word_vec(word))\n",
    "            except Exception as e:\n",
    "                continue\n",
    "        IO_data[\"label\"].append(data[\"is_duplicate\"][k])\n",
    "        IO_data[\"question1\"].append(q1) \n",
    "        IO_data[\"question2\"].append(q2)\n",
    "        pbar.update(1)\n",
    "    print('\\007')\n",
    "    pbar.close()\n",
    "    return IO_data\n",
    "\n",
    "def sequence_padding(stringlist):\n",
    "    newstring = pad_sequences(stringlist, maxlen=STRING_SIZE,\n",
    "                              dtype=object,padding='post',\n",
    "                              truncating='post', value=\"PAD\")\n",
    "    return newstring\n",
    "\n",
    "def list2string(listinput):\n",
    "    hold=\"\"\n",
    "    for k in listinput:\n",
    "        hold = hold +k.strip()+\" \"\n",
    "    return hold.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Show two comparison sentences\n",
      "--------------------------------------------\n",
      "1:  what is the story of kohinoor kohinoor diamond\n",
      "--------------------------------------------\n",
      "2:  what would happen if the indian government stole the kohinoor kohinoor diamond back\n",
      "--------------------------------------------\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "data[\"question1\"] = list(map(pipeline,data[\"question1\"]))\n",
    "data[\"question2\"] = list(map(pipeline,data[\"question2\"]))\n",
    "\n",
    "\n",
    "print(\"Show two comparison sentences\")\n",
    "print(\"--------------------------------------------\")\n",
    "print(\"1: \",data[\"question1\"][1])\n",
    "print(\"--------------------------------------------\")\n",
    "print(\"2: \",data[\"question2\"][1])\n",
    "print(\"--------------------------------------------\")\n",
    "print(data[\"is_duplicate\"][1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Corpus and Wordvectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 404286/404286 [00:34<00:00, 11820.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u0007\n",
      "['how', 'can', 'i', 'increase', 'the', 'speed', 'of', 'my', 'internet', 'connection', 'while', 'using', 'a', 'vpn', 'PAD', 'PAD']\n",
      "16\n",
      "['how', 'can', 'internet', 'speed', 'be', 'increased', 'by', 'hacking', 'through', 'dns', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD']\n",
      "16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sentences1 = list(map(wordTok,data[\"question1\"]))\n",
    "sentences2 = list(map(wordTok,data[\"question2\"]))\n",
    "\n",
    "####Truncade and Pad to N words max\n",
    "sentences1 = list(map(list,sequence_padding(sentences1)))\n",
    "sentences2 = list(map(list,sequence_padding(sentences2)))\n",
    "\n",
    "data[\"question1\"] = sentences1\n",
    "data[\"question2\"] = sentences2\n",
    "dataAll = [data[\"question1\"],data[\"question2\"]] \n",
    "dataAll = pd.concat(dataAll)\n",
    "\n",
    "model1 = Word2Vec(size = 200,sg=1,compute_loss=False,window=3,\n",
    "                 min_count=1,workers=8)\n",
    "model1.build_vocab(dataAll)  # prepare the model vocabulary\n",
    "model1.train(dataAll, total_examples=model1.corpus_count,queue_factor=5, epochs=5)\n",
    "\n",
    "final_data = wordvecmatrix(model1,data)\n",
    "print(sentences1[2])\n",
    "print(len(final_data[\"question1\"][2]))\n",
    "print(sentences2[2])\n",
    "print(len(final_data[\"question2\"][2]))\n",
    "del dataAll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           question1  \\\n",
      "0  [[-0.30817324, 0.0052596773, -0.043622255, -0....   \n",
      "1  [[-0.30817324, 0.0052596773, -0.043622255, -0....   \n",
      "2  [[-0.09129786, 0.5827426, 0.36310938, -0.44661...   \n",
      "3  [[-0.035074685, 0.062375914, 0.036061313, -0.1...   \n",
      "4  [[-0.2977956, -0.0685954, 0.136026, -0.8348036...   \n",
      "\n",
      "                                           question2  label  \n",
      "0  [[-0.30817324, 0.0052596773, -0.043622255, -0....      0  \n",
      "1  [[-0.30817324, 0.0052596773, -0.043622255, -0....      0  \n",
      "2  [[-0.09129786, 0.5827426, 0.36310938, -0.44661...      0  \n",
      "3  [[-0.005785195, 0.21426494, 0.56165993, 0.3640...      0  \n",
      "4  [[-0.2977956, -0.0685954, 0.136026, -0.8348036...      0  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4388538599014282"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(pd.DataFrame(final_data).head(5))\n",
    "cosine(model1.wv.word_vec(\"increase\"),model1.wv.word_vec(\"increased\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "split=int(0.8*len(final_data[\"label\"]))\n",
    "#x_train,x_test,y_train,y_test = tts(sentall,y,test_size=0.2)\n",
    "#x_train,x_val,y_train,y_val = tts(x_train,y_train,test_size=0.1)\n",
    "#y = to_categorical(data[\"is_duplicate\"])\n",
    "y=final_data[\"label\"]\n",
    "x1_train = np.asarray(final_data[\"question1\"][:split],dtype='float32')\n",
    "x1_val = np.asarray(final_data[\"question1\"][split:],dtype='float32')\n",
    "x2_train = np.asarray(final_data[\"question2\"][:split],dtype='float32')\n",
    "x2_val = np.asarray(final_data[\"question2\"][split:],dtype='float32')\n",
    "y_train = np.asarray(y[:split],dtype='float32')\n",
    "y_val = np.asarray(y[split:],dtype='float32')\n",
    "\n",
    "TRAIN_SIZE = len(x1_train)\n",
    "VAL_SIZE = len(x1_val)\n",
    "\n",
    "gentrain = get_samples_2_inputs(x1_train,x2_train,y_train,batch_size=BATCH_SIZE)\n",
    "gentest = get_samples_2_inputs(x1_val,x2_val,y_val,BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(323428, 16, 200)\n",
      "(323428,)\n"
     ]
    }
   ],
   "source": [
    "print(x1_train.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results of the embedding are returned as a list of sentences and their token embeddings, so each sentence is of the form<br>\n",
    "__(sentence,embeddings for each word)__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now Build an LSTM model with 2 inputs and one output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step_decay(epoch):\n",
    "\tinitial_lrate = 0.001\n",
    "\tdrop = 0.5\n",
    "\tepochs_drop = 3\n",
    "\tlrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n",
    "\treturn lrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1121 22:17:09.850288 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1121 22:17:09.870119 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1121 22:17:09.871195 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1121 22:17:09.988560 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "W1121 22:17:09.994391 47900446662336 deprecation.py:506] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W1121 22:17:10.620630 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W1121 22:17:10.624961 47900446662336 deprecation_wrapper.py:119] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3657: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "W1121 22:17:10.628459 47900446662336 deprecation.py:323] From /home/moselim/.conda/envs/tensorflow/lib/python3.7/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 340 samples, validate on 60 samples\n",
      "Epoch 1/1\n",
      "340/340 [==============================] - 5s 15ms/step - loss: 0.6830 - acc: 0.6029 - val_loss: 0.6572 - val_acc: 0.6667\n",
      "Model: \"model_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input1 (InputLayer)             (None, 16, 200)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input2 (InputLayer)             (None, 16, 200)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "LSTM1 (LSTM)                    (None, 128)          168448      input1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "LSTM2 (LSTM)                    (None, 128)          168448      input2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "merge (Concatenate)             (None, 256)          0           LSTM1[0][0]                      \n",
      "                                                                 LSTM2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense1 (Dense)                  (None, 128)          32896       merge[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 128)          0           dense1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense2 (Dense)                  (None, 64)           8256        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 1)            65          dense2[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 378,113\n",
      "Trainable params: 378,113\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model1_input = Input(shape=(x1_train.shape[1],x1_train.shape[2]),name=\"input1\")\n",
    "model1_LSTM = LSTM(128,return_sequences=False,activation='relu',\n",
    "                   dropout=0.4,recurrent_dropout=0.2,name=\"LSTM1\")(model1_input)\n",
    "model1 = Model(model1_input,model1_LSTM)\n",
    "\n",
    "model2_input = Input(shape=(x2_train.shape[1],x2_train.shape[2]),name='input2')\n",
    "model2_LSTM = LSTM(128,return_sequences=False,activation='relu',\n",
    "                   dropout=0.4,recurrent_dropout=0.2,name=\"LSTM2\")(model2_input)\n",
    "model2 = Model(model2_input,model2_LSTM)\n",
    "\n",
    "merged = concatenate([model1_LSTM,model2_LSTM],name=\"merge\")\n",
    "#model_merged_LSTM = LSTM(64,return_sequences=False,activation='relu',dropout=0.4,recurrent_dropout=0.1,name=\"LSTM_merge\")(merged)\n",
    "Dense_merged1 = Dense(128,activation=\"relu\",name=\"dense1\")(merged)\n",
    "dropout_merged =  Dropout(0.4)(Dense_merged1)\n",
    "Dense_merged2 = Dense(64,activation=\"relu\",name=\"dense2\")(dropout_merged)\n",
    "model_merged_out = Dense(1,activation=\"sigmoid\",name=\"output\")(Dense_merged2)\n",
    "model=Model([model1_input,model2_input],model_merged_out)\n",
    "\n",
    "adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.998)\n",
    "model.compile(adam, loss ='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "lrate = LearningRateScheduler(step_decay)\n",
    "earlystopper = EarlyStopping(monitor='val_loss', patience=10, verbose=1,restore_best_weights=True)\n",
    "callbacks_list = [lrate,earlystopper]\n",
    "model.fit([x1_train[:400],x2_train[:400]], y_train[:400],\n",
    "          epochs=1,\n",
    "          verbose=1,\n",
    "          batch_size=16,\n",
    "          validation_split=0.15,\n",
    "          callbacks=callbacks_list)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "20214/20214 [==============================] - 827s 41ms/step - loss: 0.5461 - acc: 0.7272 - val_loss: 0.5093 - val_acc: 0.7503\n",
      "Epoch 2/25\n",
      "20214/20214 [==============================] - 817s 40ms/step - loss: 0.5255 - acc: 0.7455 - val_loss: 0.5503 - val_acc: 0.7582\n",
      "Epoch 3/25\n",
      "20214/20214 [==============================] - 815s 40ms/step - loss: 0.5137 - acc: 0.7534 - val_loss: 0.4836 - val_acc: 0.7622\n",
      "Epoch 4/25\n",
      "20214/20214 [==============================] - 816s 40ms/step - loss: 0.5059 - acc: 0.7565 - val_loss: 0.4798 - val_acc: 0.7655\n",
      "Epoch 5/25\n",
      "20214/20214 [==============================] - 816s 40ms/step - loss: 0.5081 - acc: 0.7573 - val_loss: 0.4801 - val_acc: 0.7686\n",
      "Epoch 6/25\n",
      "20214/20214 [==============================] - 819s 41ms/step - loss: 0.5093 - acc: 0.7600 - val_loss: 0.4760 - val_acc: 0.7689\n",
      "Epoch 7/25\n",
      "20214/20214 [==============================] - 819s 41ms/step - loss: 0.5111 - acc: 0.7617 - val_loss: 0.4745 - val_acc: 0.7694\n",
      "Epoch 8/25\n",
      "20214/20214 [==============================] - 820s 41ms/step - loss: 0.5110 - acc: 0.7617 - val_loss: 0.4760 - val_acc: 0.7698\n",
      "Epoch 9/25\n",
      "20214/20214 [==============================] - 821s 41ms/step - loss: 0.5062 - acc: 0.7649 - val_loss: 0.4754 - val_acc: 0.7744\n",
      "Epoch 10/25\n",
      "20214/20214 [==============================] - 816s 40ms/step - loss: 0.5084 - acc: 0.7645 - val_loss: 0.4725 - val_acc: 0.7743\n",
      "Epoch 11/25\n",
      "20214/20214 [==============================] - 813s 40ms/step - loss: 0.5102 - acc: 0.7657 - val_loss: 0.4772 - val_acc: 0.7741\n",
      "Epoch 12/25\n",
      "20214/20214 [==============================] - 808s 40ms/step - loss: 0.5086 - acc: 0.7663 - val_loss: 0.4710 - val_acc: 0.7772\n",
      "Epoch 13/25\n",
      "20214/20214 [==============================] - 813s 40ms/step - loss: 0.5046 - acc: 0.7669 - val_loss: 0.4696 - val_acc: 0.7753\n",
      "Epoch 14/25\n",
      "20214/20214 [==============================] - 813s 40ms/step - loss: 0.5049 - acc: 0.7662 - val_loss: 0.4701 - val_acc: 0.7739\n",
      "Epoch 15/25\n",
      "20214/20214 [==============================] - 806s 40ms/step - loss: 0.5107 - acc: 0.7679 - val_loss: 0.4700 - val_acc: 0.7764\n",
      "Epoch 16/25\n",
      "20214/20214 [==============================] - 807s 40ms/step - loss: 0.5060 - acc: 0.7696 - val_loss: 0.4682 - val_acc: 0.7765\n",
      "Epoch 17/25\n",
      "20214/20214 [==============================] - 815s 40ms/step - loss: 0.5065 - acc: 0.7690 - val_loss: 0.4658 - val_acc: 0.7771\n",
      "Epoch 18/25\n",
      "20214/20214 [==============================] - 808s 40ms/step - loss: 0.5053 - acc: 0.7688 - val_loss: 0.4687 - val_acc: 0.7786\n",
      "Epoch 19/25\n",
      "20214/20214 [==============================] - 813s 40ms/step - loss: 0.5048 - acc: 0.7697 - val_loss: 0.4723 - val_acc: 0.7773\n",
      "Epoch 20/25\n",
      "20214/20214 [==============================] - 809s 40ms/step - loss: 0.5078 - acc: 0.7687 - val_loss: 0.4657 - val_acc: 0.7783\n",
      "Epoch 21/25\n",
      "20214/20214 [==============================] - 818s 40ms/step - loss: 0.5039 - acc: 0.7688 - val_loss: 0.4680 - val_acc: 0.7786\n",
      "Epoch 22/25\n",
      "20214/20214 [==============================] - 812s 40ms/step - loss: 0.5038 - acc: 0.7689 - val_loss: 0.4694 - val_acc: 0.7770\n",
      "Epoch 23/25\n",
      "20214/20214 [==============================] - 838s 41ms/step - loss: 0.5010 - acc: 0.7703 - val_loss: 0.4677 - val_acc: 0.7771\n",
      "Epoch 24/25\n",
      "20214/20214 [==============================] - 844s 42ms/step - loss: 0.5033 - acc: 0.7692 - val_loss: 0.4688 - val_acc: 0.7776\n",
      "Epoch 25/25\n",
      "20214/20214 [==============================] - 860s 43ms/step - loss: 0.5062 - acc: 0.7699 - val_loss: 0.4691 - val_acc: 0.7777\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(gentrain, \n",
    "                     steps_per_epoch=TRAIN_SIZE//BATCH_SIZE,\n",
    "                     verbose=1,\n",
    "                     validation_data=gentest,\n",
    "                     validation_steps=VAL_SIZE//BATCH_SIZE,\n",
    "                     epochs=NUM_EPOCHS,\n",
    "                     workers=8,\n",
    "                     use_multiprocessing=False,\n",
    "                     callbacks=callbacks_list)\n",
    "model.save('my_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(model.history.history['acc'])\n",
    "plt.plot(model.history.history['val_acc'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(model.history.history['loss'])\n",
    "plt.plot(model.history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = load_model('my_model_lstm256_word2vec.h5')\n",
    "model.evaluate([x1_val,x2_val],y=y_val,\n",
    "              batch_size=BATCH_SIZE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}