Amazon Text Summarizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  250M  100  250M    0     0   198M      0  0:00:01  0:00:01 --:--:--  198M\n"
     ]
    }
   ],
   "source": [
    "!curl --header 'Host: storage.googleapis.com' --user-agent 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' --header 'Accept-Language: en-US,en;q=0.5' --header 'Upgrade-Insecure-Requests: 1' 'https://storage.googleapis.com/kaggle-datasets/18/2157/amazon-fine-food-reviews.zip?GoogleAccessId=web-data@kaggle-161607.iam.gserviceaccount.com&Expires=1559384929&Signature=aB5Hh4lnq1PBPpvMX4iETE7QGjI2JW4QuwvFHovmCg2h9iT3QizWufBQyOAAGxlHo4999b%2FRjVdPk%2FcwbFuu4lwBUUWz9WJS%2FA52FmF53XhV3WhYCkRokeVUCNuvJ1e6KBHXuEp2Qg3x4JXhp6MEuKMdDddvAIpMKnABiR6%2F2k9qUYF6cXVhhabx6PQ%2FERtv3K7DOBOlx4XgUrgoLM03FhVrs41qwYv3cTv922zYE6xBV%2BN%2BH8QA7P50SLludlaJtMGnmO%2Brq11OwSsSaoqLSrAAnG5J1qNM%2BLBHQvRa9TLldThW59pj40sf3VK92X0qZ%2B1Csvs6VHC2XSwbVPdC3Q%3D%3D' --output 'amazon-fine-food-reviews.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "zip_ref = zipfile.ZipFile(\"amazon-fine-food-reviews.zip\", 'r')\n",
    "zip_ref.extractall(\"amazon\")\n",
    "zip_ref.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using SQLite Table to read data.\n",
    "con = sqlite3.connect('amazon/database.sqlite') \n",
    "\n",
    "# filtering only positive and negative reviews i.e. \n",
    "# not taking into consideration those reviews with Score=3\n",
    "# SELECT * FROM Reviews WHERE Score != 3 LIMIT 500000, will give top 500000 data points\n",
    "# you can change the number to any other number based on your computing power\n",
    "\n",
    "# filtered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews WHERE Score != 3 LIMIT 500000\"\"\", con) \n",
    "# for tsne assignment you can take 5k data points\n",
    "\n",
    "filtered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews WHERE Score != 3 LIMIT 5000\"\"\", con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sorting data according to ProductId in ascending order\n",
    "sorted_data=filtered_data.sort_values('ProductId', axis=0, ascending=True, inplace=False, kind='quicksort', na_position='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 10)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final=sorted_data.drop_duplicates(subset={\"UserId\",\"ProfileName\",\"Time\",\"Text\"}, keep='first', inplace=False)\n",
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "final=final[final.HelpfulnessNumerator<=final.HelpfulnessDenominator]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 10)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def decontracted(phrase):\n",
    "    # specific\n",
    "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
    "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "\n",
    "    # general\n",
    "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
    "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
    "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "    return phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://gist.github.com/sebleier/554280\n",
    "# we are removing the words from the stop words list: 'no', 'nor', 'not'\n",
    "# <br /><br /> ==> after the above steps, we are getting \"br br\"\n",
    "# we are including them into stop words list\n",
    "# instead of <br /> if we have <br/> these tags would have revmoved in the 1st step\n",
    "\n",
    "stopwords= set(['br', 'the', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\\\n",
    "            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \\\n",
    "            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\\\n",
    "            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', \\\n",
    "            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \\\n",
    "            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \\\n",
    "            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\\\n",
    "            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\\\n",
    "            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\\\n",
    "            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \\\n",
    "            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', \\\n",
    "            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\\\n",
    "            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\\\n",
    "            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", \\\n",
    "            'won', \"won't\", 'wouldn', \"wouldn't\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4986/4986 [00:01<00:00, 2787.08it/s]\n"
     ]
    }
   ],
   "source": [
    "# Combining all the above stundents \n",
    "from tqdm import tqdm\n",
    "preprocessed_reviews = []\n",
    "# tqdm is for printing the status bar\n",
    "for sentance in tqdm(final['Text'].values):\n",
    "    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n",
    "    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n",
    "    sentance = decontracted(sentance)\n",
    "    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n",
    "    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n",
    "    # https://gist.github.com/sebleier/554280\n",
    "    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n",
    "    preprocessed_reviews.append(sentance.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4986/4986 [00:01<00:00, 4417.01it/s]\n"
     ]
    }
   ],
   "source": [
    "preprocessed_summary = []\n",
    "# tqdm is for printing the status bar\n",
    "for sentance in tqdm(final['Summary'].values):\n",
    "    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n",
    "    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n",
    "    sentance = decontracted(sentance)\n",
    "    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n",
    "    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n",
    "    # https://gist.github.com/sebleier/554280\n",
    "    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n",
    "    preprocessed_summary.append(sentance.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, LSTM, GRU, Dense, Embedding, \\\n",
    "  Bidirectional, RepeatVector, Concatenate, Activation, Dot, Lambda\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_over_time(x):\n",
    "  assert(K.ndim(x) > 2)\n",
    "  e = K.exp(x - K.max(x, axis=1, keepdims=True))\n",
    "  s = K.sum(e, axis=1, keepdims=True)\n",
    "  return e / s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config\n",
    "BATCH_SIZE = 64\n",
    "EPOCHS = 100\n",
    "LATENT_DIM = 256\n",
    "LATENT_DIM_DECODER = 256 # idea: make it different to ensure things all fit together properly!\n",
    "NUM_SAMPLES = 4986\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "MAX_NUM_WORDS = 20000\n",
    "EMBEDDING_DIM = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_texts = [] # sentence in target language\n",
    "target_texts_inputs = [] # sentence in target language offset by 1\n",
    "\n",
    "for i in preprocessed_summary:\n",
    "    \n",
    "    target_text = i + ' <eos>'\n",
    "    target_text_input = '<sos> ' + i\n",
    "    target_texts.append(target_text)\n",
    "    \n",
    "    target_texts_inputs.append(target_text_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_texts = preprocessed_reviews.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 13015 unique input tokens.\n"
     ]
    }
   ],
   "source": [
    "# tokenize the inputs\n",
    "tokenizer_inputs = Tokenizer(num_words=MAX_NUM_WORDS)\n",
    "tokenizer_inputs.fit_on_texts(input_texts)\n",
    "input_sequences = tokenizer_inputs.texts_to_sequences(input_texts)\n",
    "\n",
    "# get the word to index mapping for input language\n",
    "word2idx_inputs = tokenizer_inputs.word_index\n",
    "print('Found %s unique input tokens.' % len(word2idx_inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# determine maximum length input sequence\n",
    "max_len_input = max(len(s) for s in input_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2971 unique output tokens.\n"
     ]
    }
   ],
   "source": [
    "# tokenize the outputs\n",
    "# don't filter out special characters\n",
    "# otherwise <sos> and <eos> won't appear\n",
    "tokenizer_outputs = Tokenizer(num_words=MAX_NUM_WORDS, filters='')\n",
    "tokenizer_outputs.fit_on_texts(target_texts + target_texts_inputs) # inefficient, oh well\n",
    "target_sequences = tokenizer_outputs.texts_to_sequences(target_texts)\n",
    "target_sequences_inputs = tokenizer_outputs.texts_to_sequences(target_texts_inputs)\n",
    "\n",
    "# get the word to index mapping for output language\n",
    "word2idx_outputs = tokenizer_outputs.word_index\n",
    "print('Found %s unique output tokens.' % len(word2idx_outputs))\n",
    "\n",
    "# store number of output words for later\n",
    "# remember to add 1 since indexing starts at 1\n",
    "num_words_output = len(word2idx_outputs) + 1\n",
    "\n",
    "# determine maximum length output sequence\n",
    "max_len_target = max(len(s) for s in target_sequences)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoder_data.shape: (4986, 486)\n",
      "encoder_data[0]: [   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    7  240 5417\n",
      " 5418 5419  414 1147 3818 7211  144 2418   86 1898]\n",
      "decoder_data[0]: [   2 1309 1310    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0]\n",
      "decoder_data.shape: (4986, 20)\n"
     ]
    }
   ],
   "source": [
    "# pad the sequences\n",
    "encoder_inputs = pad_sequences(input_sequences, maxlen=max_len_input)\n",
    "print(\"encoder_data.shape:\", encoder_inputs.shape)\n",
    "print(\"encoder_data[0]:\", encoder_inputs[0])\n",
    "\n",
    "decoder_inputs = pad_sequences(target_sequences_inputs, maxlen=max_len_target, padding='post')\n",
    "print(\"decoder_data[0]:\", decoder_inputs[0])\n",
    "print(\"decoder_data.shape:\", decoder_inputs.shape)\n",
    "\n",
    "decoder_targets = pad_sequences(target_sequences, maxlen=max_len_target, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading word vectors...\n",
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "# store all the pre-trained word vectors\n",
    "print('Loading word vectors...')\n",
    "word2vec = {}\n",
    "with open(\"glove/glove.6B.100d.txt\") as f:\n",
    "  # is just a space-separated text file in the format:\n",
    "  # word vec[0] vec[1] vec[2] ...\n",
    "  for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    vec = np.asarray(values[1:], dtype='float32')\n",
    "    word2vec[word] = vec\n",
    "print('Found %s word vectors.' % len(word2vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filling pre-trained embeddings...\n"
     ]
    }
   ],
   "source": [
    "# prepare embedding matrix\n",
    "print('Filling pre-trained embeddings...')\n",
    "num_words = min(MAX_NUM_WORDS, len(word2idx_inputs) + 1)\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word2idx_inputs.items():\n",
    "  if i < MAX_NUM_WORDS:\n",
    "    embedding_vector = word2vec.get(word)\n",
    "    if embedding_vector is not None:\n",
    "      # words not found in embedding index will be all zeros.\n",
    "      embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create embedding layer\n",
    "embedding_layer = Embedding(\n",
    "  num_words,\n",
    "  EMBEDDING_DIM,\n",
    "  weights=[embedding_matrix],\n",
    "  input_length=max_len_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create targets, since we cannot use sparse\n",
    "# categorical cross entropy when we have sequences\n",
    "decoder_targets_one_hot = np.zeros(\n",
    "  (\n",
    "    len(input_texts),\n",
    "    max_len_target,\n",
    "    num_words_output\n",
    "  ),\n",
    "  dtype='float32'\n",
    ")\n",
    "\n",
    "# assign the values\n",
    "for i, d in enumerate(decoder_targets):\n",
    "  for t, word in enumerate(d):\n",
    "    decoder_targets_one_hot[i, t, word] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "##### build the model #####\n",
    "\n",
    "# Set up the encoder - simple!\n",
    "encoder_inputs_placeholder = Input(shape=(max_len_input,))\n",
    "x = embedding_layer(encoder_inputs_placeholder)\n",
    "encoder = Bidirectional(LSTM(\n",
    "  LATENT_DIM,\n",
    "  return_sequences=True,dropout=0.5,\n",
    "  \n",
    "))\n",
    "encoder_outputs = encoder(x)\n",
    "\n",
    "\n",
    "# Set up the decoder - not so simple\n",
    "decoder_inputs_placeholder = Input(shape=(max_len_target,))\n",
    "\n",
    "# this word embedding will not use pre-trained vectors\n",
    "# although you could\n",
    "decoder_embedding = Embedding(num_words_output, EMBEDDING_DIM)\n",
    "decoder_inputs_x = decoder_embedding(decoder_inputs_placeholder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "######### Attention #########\n",
    "# Attention layers need to be global because\n",
    "# they will be repeated Ty times at the decoder\n",
    "attn_repeat_layer = RepeatVector(max_len_input)\n",
    "attn_concat_layer = Concatenate(axis=-1)\n",
    "attn_dense1 = Dense(10, activation='tanh')\n",
    "attn_dense2 = Dense(1, activation=softmax_over_time)\n",
    "attn_dot = Dot(axes=1) # to perform the weighted sum of alpha[t] * h[t]\n",
    "\n",
    "def one_step_attention(h, st_1):\n",
    "  # h = h(1), ..., h(Tx), shape = (Tx, LATENT_DIM * 2)\n",
    "  # st_1 = s(t-1), shape = (LATENT_DIM_DECODER,)\n",
    " \n",
    "  # copy s(t-1) Tx times\n",
    "  # now shape = (Tx, LATENT_DIM_DECODER)\n",
    "  st_1 = attn_repeat_layer(st_1)\n",
    "\n",
    "  # Concatenate all h(t)'s with s(t-1)\n",
    "  # Now of shape (Tx, LATENT_DIM_DECODER + LATENT_DIM * 2)\n",
    "  x = attn_concat_layer([h, st_1])\n",
    "\n",
    "  # Neural net first layer\n",
    "  x = attn_dense1(x)\n",
    "\n",
    "  # Neural net second layer with special softmax over time\n",
    "  alphas = attn_dense2(x)\n",
    "\n",
    "  # \"Dot\" the alphas and the h's\n",
    "  # Remember a.dot(b) = sum over a[t] * b[t]\n",
    "  context = attn_dot([alphas, h])\n",
    "\n",
    "  return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the rest of the decoder (after attention)\n",
    "decoder_lstm = LSTM(LATENT_DIM_DECODER, return_state=True,dropout=0.5)\n",
    "decoder_dense = Dense(num_words_output, activation='softmax')\n",
    "\n",
    "initial_s = Input(shape=(LATENT_DIM_DECODER,), name='s0')\n",
    "initial_c = Input(shape=(LATENT_DIM_DECODER,), name='c0')\n",
    "context_last_word_concat_layer = Concatenate(axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unlike previous seq2seq, we cannot get the output\n",
    "# all in one step\n",
    "# Instead we need to do Ty steps\n",
    "# And in each of those steps, we need to consider\n",
    "# all Tx h's\n",
    "\n",
    "# s, c will be re-assigned in each iteration of the loop\n",
    "s = initial_s\n",
    "c = initial_c\n",
    "\n",
    "# collect outputs in a list at first\n",
    "outputs = []\n",
    "for t in range(max_len_target): # Ty times\n",
    "  # get the context using attention\n",
    "  context = one_step_attention(encoder_outputs, s)\n",
    "\n",
    "  # we need a different layer for each time step\n",
    "  selector = Lambda(lambda x: x[:, t:t+1])\n",
    "  xt = selector(decoder_inputs_x)\n",
    "  \n",
    "  # combine \n",
    "  decoder_lstm_input = context_last_word_concat_layer([context, xt])\n",
    "\n",
    "  # pass the combined [context, last word] into the LSTM\n",
    "  # along with [s, c]\n",
    "  # get the new [s, c] and output\n",
    "  o, s, c = decoder_lstm(decoder_lstm_input, initial_state=[s, c])\n",
    "\n",
    "  # final dense layer to get next word prediction\n",
    "  decoder_outputs = decoder_dense(o)\n",
    "  outputs.append(decoder_outputs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'outputs' is now a list of length Ty\n",
    "# each element is of shape (batch size, output vocab size)\n",
    "# therefore if we simply stack all the outputs into 1 tensor\n",
    "# it would be of shape T x N x D\n",
    "# we would like it to be of shape N x T x D\n",
    "\n",
    "def stack_and_transpose(x):\n",
    "  # x is a list of length T, each element is a batch_size x output_vocab_size tensor\n",
    "  x = K.stack(x) # is now T x batch_size x output_vocab_size tensor\n",
    "  x = K.permute_dimensions(x, pattern=(1, 0, 2)) # is now batch_size x T x output_vocab_size\n",
    "  return x\n",
    "\n",
    "# make it a layer\n",
    "stacker = Lambda(stack_and_transpose)\n",
    "outputs = stacker(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_weights_path = 'best_modelAmazon.h5'\n",
    "\n",
    "callbacks_list = [\n",
    "    ModelCheckpoint(final_weights_path, monitor='val_loss', verbose=1, save_best_only=True),\n",
    "    EarlyStopping(monitor='val_loss', patience=10, verbose=1),TensorBoard(log_dir='Graph', histogram_freq=1, write_graph=True, write_images=True)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the model\n",
    "model = Model(\n",
    "  inputs=[\n",
    "    encoder_inputs_placeholder,\n",
    "    decoder_inputs_placeholder,\n",
    "    initial_s, \n",
    "    initial_c,\n",
    "  ],\n",
    "  outputs=outputs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3988 samples, validate on 998 samples\n",
      "Epoch 1/100\n",
      "3988/3988 [==============================] - 218s 55ms/step - loss: 1.6189 - val_loss: 1.1929\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.19285, saving model to best_modelAmazon.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 's0:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'c0:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_1/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_1/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_2/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_2/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_3/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_3/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_4/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_4/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_5/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_5/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_6/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_6/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_7/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_7/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_8/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_8/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_9/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_9/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_10/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_10/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_11/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_11/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_12/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_12/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_13/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_13/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_14/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_14/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_15/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_15/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_16/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_16/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_17/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_17/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n",
      "/usr/local/lib/python3.5/dist-packages/keras/engine/network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'lstm_2_18/while/Exit_3:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'lstm_2_18/while/Exit_4:0' shape=(?, 256) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/100\n",
      "3988/3988 [==============================] - 173s 43ms/step - loss: 1.1240 - val_loss: 1.1187\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.19285 to 1.11870, saving model to best_modelAmazon.h5\n",
      "Epoch 3/100\n",
      "3988/3988 [==============================] - 174s 44ms/step - loss: 1.0395 - val_loss: 1.1237\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 1.11870\n",
      "Epoch 4/100\n",
      "3988/3988 [==============================] - 173s 43ms/step - loss: 1.0164 - val_loss: 1.1143\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.11870 to 1.11427, saving model to best_modelAmazon.h5\n",
      "Epoch 5/100\n",
      "3988/3988 [==============================] - 175s 44ms/step - loss: 0.9973 - val_loss: 1.1241\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 1.11427\n",
      "Epoch 6/100\n",
      "3988/3988 [==============================] - 177s 44ms/step - loss: 0.9827 - val_loss: 1.1027\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.11427 to 1.10270, saving model to best_modelAmazon.h5\n",
      "Epoch 7/100\n",
      "3988/3988 [==============================] - 176s 44ms/step - loss: 0.9682 - val_loss: 1.1005\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.10270 to 1.10048, saving model to best_modelAmazon.h5\n",
      "Epoch 8/100\n",
      "3988/3988 [==============================] - 176s 44ms/step - loss: 0.9549 - val_loss: 1.0973\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.10048 to 1.09727, saving model to best_modelAmazon.h5\n",
      "Epoch 9/100\n",
      "3988/3988 [==============================] - 176s 44ms/step - loss: 0.9377 - val_loss: 1.1049\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.09727\n",
      "Epoch 10/100\n",
      "3988/3988 [==============================] - 174s 44ms/step - loss: 0.9185 - val_loss: 1.1055\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.09727\n",
      "Epoch 11/100\n",
      "3988/3988 [==============================] - 176s 44ms/step - loss: 0.9013 - val_loss: 1.1030\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.09727\n",
      "Epoch 12/100\n",
      "3988/3988 [==============================] - 176s 44ms/step - loss: 0.8829 - val_loss: 1.1149\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.09727\n",
      "Epoch 13/100\n",
      "3988/3988 [==============================] - 174s 44ms/step - loss: 0.8647 - val_loss: 1.1216\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 1.09727\n",
      "Epoch 14/100\n",
      "3988/3988 [==============================] - 174s 44ms/step - loss: 0.8485 - val_loss: 1.1123\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 1.09727\n",
      "Epoch 15/100\n",
      "3988/3988 [==============================] - 173s 43ms/step - loss: 0.8324 - val_loss: 1.1246\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 1.09727\n",
      "Epoch 16/100\n",
      "3988/3988 [==============================] - 175s 44ms/step - loss: 0.8176 - val_loss: 1.1181\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 1.09727\n",
      "Epoch 17/100\n",
      "3988/3988 [==============================] - 172s 43ms/step - loss: 0.8011 - val_loss: 1.1312\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 1.09727\n",
      "Epoch 18/100\n",
      "3988/3988 [==============================] - 174s 44ms/step - loss: 0.7857 - val_loss: 1.1189\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 1.09727\n",
      "Epoch 00018: early stopping\n"
     ]
    }
   ],
   "source": [
    "# compile the model\n",
    "model.compile(optimizer=\"rmsprop\", loss='categorical_crossentropy')\n",
    "\n",
    "# train the model\n",
    "z = np.zeros((NUM_SAMPLES, LATENT_DIM_DECODER)) # initial [s, c]\n",
    "r = model.fit(\n",
    "  [encoder_inputs, decoder_inputs, z, z], decoder_targets_one_hot,\n",
    "  batch_size=64,\n",
    "  epochs=EPOCHS,\n",
    "  validation_split=0.2,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4XNWd5vHvKam0lXapJFmSJVnGeJPBxgt4uk3YQmgSYMiCQ0hYQmCyQZLOMOHJyqTTTxamk+6eTkMITVjC5gS6m2kgLIHgLDbGNt43sGzJkm3tuyxrO/PHvZLKsla7pFJVvZ/nqaeWe6vqV1elt06de+pcY61FREQikyfUBYiIyNRRyIuIRDCFvIhIBFPIi4hEMIW8iEgEU8iLiEQwhbyISARTyIuIRDCFvIhIBIsN1RNnZ2fbkpKSUD29iEhY2rJlS7211j/R9UMW8iUlJWzevDlUTy8iEpaMMRWTWV/dNSIiEUwhLyISwRTyIiIRLGR98iISnXp6eqiqqqKrqyvUpcxoCQkJFBYW4vV6z+pxFPIiMq2qqqpISUmhpKQEY0yoy5mRrLU0NDRQVVXFnDlzzuqx1F0jItOqq6uLrKwsBfwYjDFkZWUF5duOQl5Epp0CfnzB2kZhF/L7jrfyo5f30drVE+pSRERmvLAL+cqGTh586yAHa9tDXYqIhKnk5ORQlzBtxg15Y8wjxphaY8yuMda5xBizzRiz2xjzVnBLPNXcHOePU17XMZVPIyISESbSkn8UuGq0hcaYdOBfgWuttYuBTwSntJEVZSYR6zGU16slLyJnx1rLPffcQ1lZGUuWLOHZZ58F4NixY1x88cUsXbqUsrIy/vjHP9LX18ett946uO7PfvazEFc/MeMOobTWrjfGlIyxyqeA5621le76tcEpbWTeGA9FmUlqyYtEgP/9/3az52hrUB9zUX4q37tm8YTWff7559m2bRvbt2+nvr6elStXcvHFF/PUU0/xoQ99iG9961v09fXR2dnJtm3bqK6uZtcup1Ojubk5qHVPlWD0yZ8LZBhj/mCM2WKMuTkIjzmmUr9PIS8iZ+1Pf/oTN954IzExMeTm5vKBD3yAd955h5UrV/KrX/2K++67j507d5KSkkJpaSnl5eXcdddd/O53vyM1NTXU5U9IMH4MFQssBy4HEoENxpiN1toDw1c0xtwJ3AlQVFR0xk9Y6k9m/Xv19PVbYjwaiiUSriba4p5uF198MevXr+fFF1/k1ltv5W//9m+5+eab2b59O6+88goPPvgg69at45FHHgl1qeMKRku+CnjFWtthra0H1gPnj7SitfYha+0Ka+0Kv3/C0yGfpjTbR3dvP9VNJ874MURE1qxZw7PPPktfXx91dXWsX7+eVatWUVFRQW5uLnfccQef+9zn2Lp1K/X19fT39/Oxj32MH/zgB2zdujXU5U9IMFry/wn8izEmFogDLgSmdI9Eqd8ZYXOwvp2irKSpfCoRiWDXX389GzZs4Pzzz8cYw09+8hPy8vJ47LHHuP/++/F6vSQnJ/P4449TXV3NbbfdRn9/PwA//OEPQ1z9xIwb8saYp4FLgGxjTBXwPcALYK190Fq71xjzO2AH0A88bK0ddbhlMJT6fYAzjPLS+VP5TCISidrbndF5xhjuv/9+7r///lOW33LLLdxyyy2n3S9cWu+BJjK65sYJrHM/cP946wVLli+O1IRYyus0jFJEZCxh94tXcD59S/3JGmEjIjKOsAx5cIdR6gdRIiJjCtuQn+tPpqb1JO0ne0NdiojIjBW2IV+a7ex8PaQuGxGRUYVvyLvDKNVlIyIyurAN+eKsJIyBg2rJi4iMKmxDPsEbQ2FGooZRisiUGmvu+cOHD1NWVjaN1Uxe2IY8QGm2hlGKiIwlGNMahEyp38emQ43091s8mqhMJPy8fC8c3xncx8xbAn/zo1EX33vvvcyePZsvfelLANx3333Exsby5ptv0tTURE9PDz/4wQ+47rrrJvW0XV1dfOELX2Dz5s3Exsby05/+lEsvvZTdu3dz22230d3dTX9/P8899xz5+fnccMMNVFVV0dfXx3e+8x3Wrl17Vi97NGEe8smc6OnjeGsX+emJoS5HRMLA2rVr+epXvzoY8uvWreOVV17h7rvvJjU1lfr6ei666CKuvfbaSR1M++c//znGGHbu3Mm+ffu48sorOXDgAA8++CBf+cpXuOmmm+ju7qavr4+XXnqJ/Px8XnzxRQBaWlqm5LVCmIf83OyhOWwU8iJhaIwW91RZtmwZtbW1HD16lLq6OjIyMsjLy+NrX/sa69evx+PxUF1dTU1NDXl5eRN+3D/96U/cddddACxYsIDi4mIOHDjA6tWr+fu//3uqqqr46Ec/yrx581iyZAlf//rX+cY3vsFHPvIR1qxZM1UvN8z75DWMUkTOwCc+8Ql++9vf8uyzz7J27VqefPJJ6urq2LJlC9u2bSM3N5eurq6gPNenPvUpXnjhBRITE7n66qt54403OPfcc9m6dStLlizh29/+Nt///veD8lwjCeuWfG5qPL64GO18FZFJWbt2LXfccQf19fW89dZbrFu3jpycHLxeL2+++SYVFRWTfsw1a9bw5JNPctlll3HgwAEqKyuZP38+5eXllJaWcvfdd1NZWcmOHTtYsGABmZmZfPrTnyY9PZ2HH354Cl6lI6xD3hjDHL+PgxpGKSKTsHjxYtra2igoKGDWrFncdNNNXHPNNSxZsoQVK1awYMGCST/mF7/4Rb7whS+wZMkSYmNjefTRR4mPj2fdunU88cQTeL1e8vLy+OY3v8k777zDPffcg8fjwev18sADD0zBq3QYa+2UPfhYVqxYYTdv3nzWj3P30++ypaKJP997WRCqEpGptnfvXhYuXBjqMsLCSNvKGLPFWrtioo8R1n3y4ExUdrTlBF09faEuRURkxgnr7hpwxspbC4fqO1g4KzyOni4i4WXnzp185jOfOeW2+Ph43n777RBVNHEREfLgDKNUyIuEB2vtpMagh9qSJUvYtm3btD5nsLrSw767Zs7gWHntfBUJBwkJCTQ0NAQtxCKRtZaGhgYSEhLO+rHCviWfFBdLfloC5fUaRikSDgoLC6mqqqKuri7UpcxoCQkJFBYWnvXjhH3IA+7xXtWSFwkHXq+XOXPmhLqMqBH23TXgHu+1rkNf/0REhomMkM/20Xayl7r2k6EuRURkRomMkHfnsDlYq355EZFAERLy7ggbTVQmInKKiAj5/LREErweTVQmIjJMRIS8x2MoyfJphI2IyDAREfLgzGGjsfIiIqeKmJAv9fs40tjJyV5NVCYiMiCiQr7fQmVDZ6hLERGZMSIn5LPdYZTa+SoiMihyQl7DKEVEThMxIZ+S4MWfEq9hlCIiASIm5MGZ3kDDKEVEhkRWyGsYpYjIKSIq5Of6fTR39tDY0R3qUkREZoSICvmhQwGqy0ZEBCIt5N1hlNr5KiLiiKiQL8xIxBtjOKhhlCIiwARC3hjziDGm1hiza5z1Vhpjeo0xHw9eeZMTG+NxJypTS15EBCbWkn8UuGqsFYwxMcCPgVeDUNNZcQ4FqJa8iAhMIOStteuBxnFWuwt4DqgNRlFno9SfTGVjJ719/aEuRUQk5M66T94YUwBcDzwwgXXvNMZsNsZsrqurO9unHlFpto+ePsuRphNT8vgiIuEkGDte/xH4hrV23KaztfYha+0Ka+0Kv98fhKc+3cDxXtVlIyICsUF4jBXAM8YYgGzgamNMr7X2P4Lw2JM2d3CsfAeXLwxFBSIiM8dZh7y1ds7AZWPMo8B/hSrgAdKT4sj0xWk2ShERJhDyxpingUuAbGNMFfA9wAtgrX1wSqs7Q6XZPs0rLyLCBELeWnvjRB/MWnvrWVUTJKV+H2/sm5oduyIi4SSifvE6oNSfTH37SVpO9IS6FBGRkIrMkM/WRGUiIhCpIe/XRGUiIhChIV+UmUSMx2iEjYhEvYgM+bhYD0WZSWrJi0jUi8iQh4HjvSrkRSS6RW7I+30cauigr9+GuhQRkZCJ4JBPpru3n6PNmqhMRKJX5Ia8O4zyoIZRikgUi9yQ1zBKEZHIDfns5DhSEmI1jFJEolrEhrwxhlJ/slryIhLVIjbkAeZqGKWIRLmIDvlSv4/jrV10nOwNdSkiIiER4SHv7Hw9VK/WvIhEp4gO+bluyGsYpYhEq4gO+eKsJIzRMEoRiV4RHfIJ3hgKMxIpV3eNiESpiA55gNLsZB08RESiVuSHvN/HofoOrNVEZSISfaIg5JPp7O7jeGtXqEsREZl2ER/ycweP96p+eRGJPhEf8kMTlalfXkSiT8SHfG5qPL64GA6qJS8iUSjiQ94Ywxy/T8MoRSQqRXzIg4ZRikj0io6Q9/uobj5BV09fqEsREZlWURLyyVgLhxvUZSMi0SU6Qn7geK+1CnkRiS7REfL+gbHy6pcXkegSFSGfFBfLrLQEjbARkagTFSEPTmteLXkRiTbRE/LZzkG9NVGZiEST6Al5v4+2k73UtZ8MdSkiItMmikJ+YA4b9cuLSPSInpDXbJQiEoWiJuQL0hOJj/Vo56uIRJWoCXmPxzAnWxOViUh0GTfkjTGPGGNqjTG7Rll+kzFmhzFmpzHmL8aY84NfZnBoGKWIRJuJtOQfBa4aY/kh4APW2iXA3wEPBaGuKTHXn8yRphN09/aHuhQRkWkxbshba9cDjWMs/4u1tsm9uhEoDFJtQVfq99HXb6lsVJeNiESHYPfJ3w68HOTHDJrSbGcYpY4SJSLRImghb4y5FCfkvzHGOncaYzYbYzbX1dWd2RN1tcJb90Nf76TvOjRRmUJeRKJDUELeGHMe8DBwnbW2YbT1rLUPWWtXWGtX+P3+M3uy/S/Dmz+A5++YdNCnJHjxp8Rr56uIRI3Ys30AY0wR8DzwGWvtgbMvaRznr4X2GnjtO9DfCx9/BGK8E757qYZRikgUmcgQyqeBDcB8Y0yVMeZ2Y8znjTGfd1f5LpAF/KsxZpsxZvMU1uv4q7vhQz+EvS/Ab26F3u4J37XUr+O9ikj0GLclb629cZzlnwM+F7SKJmr1F8ETCy/fA+tuhhseg9j4ce821++jqbOHpo5uMnxx01CoiEjohPcvXi+8Ez78UzjwMjxzE/R0jXuXwZ2v9WrNi0jkC++QB1h5O1zzz/D+6/DMjdBzYszVNYxSRKJJ+Ic8wPJb4Lqfw8E34akboHv0AC/MSMQbYzSMUkSiQmSEPMCym+D6X8DhP8GTN8DJkbtjYmM8FGdpDhsRiQ6RE/LgDK/86C+hcgM8+XE42TbiahpGKSLRIrJCHmDJx+Hj/wZV78ATH4WultNWKfUnU9HQQW+fJioTkcgWeSEPsPh6+MSjcHQrPHE9nGg+ZXGp30dPn6WqaeydtCIi4S4yQx5g4TVwwxNwbAc8fh10Dk2kOdcdRnlQ/fIiEuEiN+QBFlwNn3wKavfC49dChzOtzsAwSo2wEZFIF9khD3DulXDjU1D/Hjx2DXTUk+GLIyPJqx9EiUjEO+sJysLCOVfAjc/A0zfCox+BW16g1J+sH0SJyOhOtkHtPqjd7fQG1O51pk7xz4fs+eBfAP5zISEt1JWOKTpCHmDupXDTb5wfSz36Yc7P/BEvlNuzf1xrnZNnhnwp6mp15vSJSwp1JSJnrq8HGsuhux18fufkTZya5+o96XzTr907FOg1e6Clcmgdr88J984GKP8D9AVMipgyKyD4B04LwJc9NfVOUvSEPMCcNfDp5+DXH+fujq/xYvs9bKloZHlRhvNmOtEMXc3ueUvA5THOu1rA9kNqAaQXQXqxex5wSpkFMUHa1L3d0HIEmg5BUwU0VzjnTYedyyeaICYOilbDvA8632L8C8CY4Dy/RIbebuiog45a6KiH9lrncu9JSCuEtNnOeze1AGKncCK/ni5oeB/q9kH9Aee8br9zW/+w40XEJTvB6ctxgz/bOU/OGbrs8zvLEzNOb3j190PzYSfAAwM98Lk8sZB9Lsxe5fySPmcR5C6CtKKhx+vrdf7X6vafWve7v4aegN6BpKxhwe9+EKTmT+v/o7E2CK3ZM7BixQq7efPUz0o8osqN2F9/jI7ufrqJIcNzAjP8DRXIeJyvZAnpkJjunCekDV02BlqqoLnSObUdG3b/GEgrGPkDIL0IUvKHPgT6+6H9eECAHz71cutRIOBvFhPnPEZGifP4GcXOP+97r0PdXmed1EI453In9Od8ABJSg7ctZebo7nSCur1uKMDbB85r3dvqnMtdzeM/HgDGCaX0oqHgTy+C9NnO+y2tcEKzv9Ld4bSWB4Jx4LzpkNNIAuf/LGOO2w3ihmJ8KnTWu7XXB7wG97yzfuj+p5TtgaSBDwG/0xir2w89nUPrZJQ4IZ6zCHIWOudZ55z5h5q1Tg7U73df3/6h1xm4veNSYM3XYM3Xz+hpjDFbrLUrJrx+VIY8QPVW2v/4AC/vb6HTk8xH/6qMlPTsgCAPCPW4lMl1x/R0QWu1E8wDwR94GulDYKDF1HwE+k4GLnS+CWSUOAEeGOYZJZCcN3ptLVXOxG3vvw4H/wDdbU5LZfZFMO8Kp5WfW6ZWfjjp6YLGg0541L/ntCLr90PjYefvO5KEtKEWbrJ/qCU8cHmwJZzjNBpaq533acuRgPete7m1GmzfqY+fnBcQ/O6HgMcbEHb7nPsO8MQ6YTrQrTHQws06B7wJk9se/f3Ot9eOulO/mQxed7+leBMhd/FQqPvnQ3zy5J7rTFnr1BLY8i9ZA4uuPaOHU8hP0o6qZj750EaKs3ys+x8XkZIw8aNMnbHek27Lv+LUf6DerqEwTy9xA332xFpK4+nrgSNvO4H/3utQs9O5PTnPCft5V0Dppc6HmoReZ6Mb4AdODfTmioCWq3ECNftcyJoLybluYLuhneyGeTDePwP6eqHt6Knv25aAD4KWKujvcdaNiXdqCwxz/3zILJ3U0dzkVAr5M/DWgTpuf/QdLizN5Fe3riIudobsRJ1Krcfg4O/hvdeg/E3n66yJgcKVQ638nEVOyy5cW/rdHdB23D0dc87ba5wPPNsfcOoLuGyHLRt+ClgeE+e0EGPjIdY9H7yeMHTyJox+HZwdjMPDvLN+6HXExEP2PCcws891LvvnQ+bcmbeDvb/P2c59J51vnJ6YUFcUcRTyZ+i5LVV8/Tfbufb8fP5x7VI8njANtjPR1wvVm91W/mtwbNupy2Pi3eBywysmzg2puGHXA9aLiR+6zZsI3iQnkLw+5/rA5VPO3dN4faLdnc5+i8HwrhkK8bZjTpC3HYeTraffd6AuY5x+24GTJybg+rBlp50MYJwRFr1dThdKb8DpTCVmul0X85zui+xznSF6abMVljJosiEfXaNrxvCx5YXUtHXxk9/tJy8tgW9evTDUJU2fmFgousg5XfZtZ6fWwTegtcrpWho49QVc7u0aCrneLuebwGnruOsNfH2fKE/sqcE/cH6i2Qn3ESadIyYeUvKc/Rc5C2HuZUPXk3Od85Q8p396Kr+ZWOtsl54T7ut3z4dfH/hwsH3Ozsbsc8GXNXV1SdRSyAf4wgfmcryli4fWl5ObmsDtfz0n1CWFRrLfmbY5WPp6nVENPZ1OF0pPp9Ma7+lwzwduPzHCbZ1D1/3zofQSSAkI7YEQT8yYGd1Kxgx9mxGZARTyAYwxfO+axdS1neTv/msPOSnxXHN+fqjLCn8xsRCTqqGbIiEQBXsYJyfGY/jZ2qWsKsnk6+u285eD9ePfSURkhlLIjyDBG8Mvb15BcVYS/+PxLew9NsIOPBGRMKCQH0VakpfHPrsKX3wst/5qE9XNOsCIiIQfhfwY8tMTeeyzq+js7uOWRzbR3Nk9/p1ERGYQhfw45uel8MubV1DZ0MnnHttMV0/f+HcSEZkhFPITcFFpFj9bu5QtlU3c/fS79PWH5gdkIiKTpZCfoA+fN4vvfmQRr+6p4Xsv7CJUvxQWEZkMjZOfhNv+ag7HW7v4xVvl5KUm8OXL5oW6JBGRMSnkJ+kbH1pAbetJ/s+rB8hNTeATK2aHuiQRkVEp5CfJ4zH8+GPnUd9+knuf30l2SjyXzs8JdVkiIiNSn/wZiIv18MCnl7MgL4Uv/nor249M9Cg7IiLTSyF/hpLjY/nVbSvJTonjE7/YwO2PvsMzmyqpbTuLqWZFRIJM3TVnISclgafvuIiH/3iI1/fW8Pt9tRgDS2enc8XCXK5clMs5OcmYmTA7oohEJR00JEisteyvaeO13TW8vreG7VXOnOfFWUlcsTCXDy7KZUVxBrEx+vIkImdOR4aaIY63dPH7fTW8tqeGv7zfQHdfP+lJXi6bn8MVi3K5+Fw/yfH6IiUik6OQn4HaT/byxwN1vLa3hjf21dLc2UNcjIfVc7O4YlEuH1yYS17aJI9SLyJRSSE/w/X29bOloonX9tTw2t4aKho6AVhSkMZlC3JYPTeLZUXpxMfqmJ4icrqgh7wx5hHgI0CttbZshOUG+CfgaqATuNVau3W8J47WkA9kreVgXTuv7nG6dbYdacZaiI/1sLw4g9WlWayem8V5henExaovX0SmJuQvBtqBx0cJ+auBu3BC/kLgn6y1F473xAr507Wc6GHToUY2HGxgQ3nD4MFKEr0xrCjJ4KKB0C9I0w5ckSg12ZAfd8+ftXa9MaZkjFWuw/kAsMBGY0y6MWaWtfbYRIsQR1qilw8uckbiADR1dPP2oQY2ljvBf/8r+wHwxcWwck7mYEt/cX4aMR4N0xSR0wVjeEcBcCTgepV7m0L+LGX44riqbBZXlc0CoL79JG+XN7KhvJ4NBxv4w/46AFLiY1k1J5PVc7O4qDSLRbNS8Sj0RYRp/jGUMeZO4E6AoqKi6XzqiJCdHM+Hz5vFh89zQr+2tYsN5U5Lf2N5A7/fVws4Lf1F+akszk9jcX4qZQVpnJOTjFddPCJRJxghXw0ETsVY6N52GmvtQ8BD4PTJB+G5o1pOagLXLS3guqUFABxrOcHG8ga2VTaz+2gr6zYfobPbOZJVXKyHBXkpLA4I/4WzUknwahSPSCQLRsi/AHzZGPMMzo7XFvXHh8astESuX1bI9csKAejrtxyq72D30RZ2H21l99EWXtp5nKc3Ob1rMR7DXL+Psvw0Fhc4wb8oP5XUBG8oX4aIBNG4IW+MeRq4BMg2xlQB3wO8ANbaB4GXcEbWvI8zhPK2qSpWJifGYzgnJ5lzcpIHW/vWWqqbT7CrupU9R1vYdbSVPx+s5/l3h758FWclUZaf5nb5OC1/f0p8qF6GiJwF/RhKAKhrOznY4t9V3cKuoy0caTwxuDwnJf6Urp7F+WnMzkzU5Gsi0yzoQyglOvhT4rlkfg6XBBwApeVED3uOtrLnmNPVs+doK+vfqx88kHlKQiyLZgUEf0Eqc/3awSsykyjkZVRpiV5Wz3XG4g/o6unjQE3bYB//7qOtPLWpgq6efuDUHbyL3PBfkJdCUpzeaiKhoP88mZQEbwznFaZzXmH64G3ODt52N/id8H9519AOXmOgKDOJ+bkpLMhLYX5eKgtmpVCS5dOPuESmmEJezpqzgzeFc3JSTtnBe7Sli93VLew73sb+423sO97K63trcHt7iI/1MC83mfm5qW74Ox8C/pR49fWLBIlCXqaEMYaC9EQK0hO5cnHe4O1dPX28X9vuBn8r+4638cf36nhua9XgOpm+OObnDoX+/LwUzs1Nwaf590UmTf81Mq0SvDGUFaRRVpB2yu2NHd3sO97K/sFWf9spP+YyBublJLO8OIPlxZksL86gJCtJLX6RcWgIpcxY/f2WqqYT7D3eyr5jbbx7pImtFU20dvUCkOWL44LiDJYXZ7CiOIOygjT9glcinoZQSsTweAxFWUkUZSXxIbfLp7/f8n5dO1sqmth8uIktFY28tqcGgLgYD2UFqae09vUjLol2aslL2KtvP8nWiia2uKcdVS109zlDOouzklhelMHyEqfFPy8nRSN6JKzp8H8S9U729rGrupUtFY2DwV/f3g04P+BaWTI0F//CWakKfQkr6q6RqBcfG+N22WQAznDOysZONh9uYnNFIxvLG3nDnZY5NSGWC0uzBkN/fm6K5uKXiKKQl4hnjKE4y0dxlo+PLXdm6Dze0jV48JWN5UP9+hlJ3sHDLK4uzeKcnGSN4JGwpu4aEaCqqXPwMIsbyxuobnYmZ8tOjuei0qGjbpVm+xT6ElLqkxc5S9ZajjSeGGzpbyhvoKb1JAC5qfFOS780ixUlmcz1K/RleinkRYLMWufgKxvLG9lQ3sCGgw3Utzuhn5Hk5YKB0TtFGZw/O11j9WVKacerSJAZYyj1J1PqT+ZTFxZhreVgXcfg6J3NFU2Dx9eN9RgWF6Sxwt3xu7w4g9zUhBC/AolmasmLBEFjRzfvVjqBv6Wiie1HmjnZ64zVL8xIHPxV7gXFGSzI07BNOXNqyYuEQKYvjssX5nL5wlwAunv72XOslc2HG9la2cSGgw3857ajAPjiYlhW5AT+QPAna/I1mSJ6Z4lMgbhYD0tnp7N0tjPvvrXOPDwDP87aXNHEv7zxHv3Wmaq5LD+VC0uzWFWSycqSTNKSdDB1CQ5114iESFtXD+9WNrPpUCObDjWy7Ugz3X39GAML8lK5cE4mF87JZOWcTLKTNQePODS6RiRMdfX0se2IE/pvH2pgS0XT4GEVz8lJZpUb+hfOySIvTTtzo5VCXiRCdPf2s7O6xW3pN7D5cBNtJ51plosyk7hwTiar5mRyUWkWhRmJGq8fJRTyIhGqr9+y91grG8sbnOA/3EhzZw8As9ISWDXH6c+/cE6mpmOIYAp5kSjR3295r7adTYca2HiokXcONVLb5vxIK9MXx4riDFa5rf1Fs1KJjfGEuGIJBg2hFIkSHo9hvnsM3M+sLsFaS0VDJ5sONw7uzH3VnXjNFxfDBcUZbhdPFucV6iha0UIteZEIdryli02HnVb+pkON7K9pA5yjaC2dnc7KORmsmpPFco3VDxvqrhGRUTV1dLO5oolNhxrYdLiJXdUt9PVbPAYW56exsiSTpUXpLJudrp25M5RCXkQmrONkL1srm3jnUCNvu2P1B6ZjyPLFDf6ga2lROucVppOWqB9phZr65EWaOjSxAAAJ/klEQVRkwnzxsayZ52fNPD8APX397D/exrtHmtlW2cy2I0OTrwHM9ftYOjtjsLU/Py8Fr3bozmhqyYvImFpO9LCjaiD0nVNDh3PM3PhYD0sK0gZb+0tnp1OQrm6eqaTuGhGZUgPz8AS29ncdbaXb7ebJTo5n6ex0lhWlc35hOufNTiM1Qd08waLuGhGZUsYYZmcmMTsziWvPzwecX+fuO97qtPTdFv/re2vc9WGuP3mof1/dPNNKLXkRmRItnT1srxrq4tl2pJlGt5snweuhLH+om+f8Qo3mmSh114jIjDSxbh43+GdnqJtnFOquEZEZabRunv3H29h2pMkJ/yPNvL53aDRPqd9HWX4aZQWplOWnsTg/TXPtT5Ja8iIyowSO5tlZ3cLuo61UN58YXD47M9EN/jQW56dSVpAWVfPtqyUvImEtLdF7yth9cI6hu/toixP61a3sOtrCy7uODy7PS02grCCVxW74lxWkkpeaoD5+FPIiEgYyfXGnBX/LiR72HG1l99EWdlW3sOtoK7/fV8tA50SWL47FBWmU5adyXmE6FxSlk5MafQdbUciLSFhKS/Syem4Wq+dmDd7W2d3L3mOt7KpuHQz+h9aX09vvJH9BeuLgr3UvKM5gcX4q8bGRPRvnhELeGHMV8E9ADPCwtfZHw5YXAY8B6e4691prXwpyrSIiY0qKi2V5cSbLizMHb+vq6WPPsVberWzm3com3q1s5sUdxwBnNs5F+aksK0rngqIMlhVF3i92x93xaoyJAQ4AHwSqgHeAG621ewLWeQh411r7gDFmEfCStbZkrMfVjlcRCZWa1i4n9I84ob+jqnnweLr+lHiWzU5nWVEGFxSls6QwjaS4mdPpMRU7XlcB71try90neAa4DtgTsI4FUt3LacDRiRYgIjLdclMTuKosj6vK8oCAidnclv67R5oHD7gS4zEsyEthWZE7fr8wjbn+ZGI84dHan0jIFwBHAq5XARcOW+c+4FVjzF2AD7giKNWJiEwDb4zHHZWTxmdWO7c1dnQ74/crm3m3spn/ePcov95YCUBSXAyL81NZUpDO+bPTWFKQRkmWD88MDP5gfQe5EXjUWvsPxpjVwBPGmDJrbX/gSsaYO4E7AYqKioL01CIiwZfpi+OyBblctiAXcI6pW17fwc7qZrYfcYZzPrWpgkf+7MRcSnwsZQVpnFeYxpLCtBkzVcNEQr4amB1wvdC9LdDtwFUA1toNxpgEIBuoDVzJWvsQ8BA4ffJnWLOIyLTzeAzn5CRzTk4y1y8rBKC3r5/369rZUdXCjqpmdla18Ks/H6a7zwn+9CQvSwaCvyCd8wrTmJU2veP3JxLy7wDzjDFzcML9k8Cnhq1TCVwOPGqMWQgkAHXBLFREZKaJjfGwIC+VBXmp3LDCaQt39/ZzoKaNHVUtg63+X7w1NIwzOzmez3+glM+tKZ2eGsdbwVrba4z5MvAKzvDIR6y1u40x3wc2W2tfAL4O/NIY8zWcnbC32lDNlyAiEkJxsUP9++B0S3f19LH3WCs7q1vYUdWCP2X6pmHQ3DUiImFkskMoNWu/iEgEU8iLiEQwhbyISARTyIuIRDCFvIhIBFPIi4hEMIW8iEgEU8iLiESwkP0YyhhTB1Sc4d2zgfogljMdVPP0CLeaw61eUM3TZbSai621/hFuH1HIQv5sGGM2T+YXXzOBap4e4VZzuNULqnm6BKtmddeIiEQwhbyISAQL15B/KNQFnAHVPD3CreZwqxdU83QJSs1h2ScvIiITE64teRERmYAZHfLGmKuMMfuNMe8bY+4dYXm8MeZZd/nbxpiS6a/ylHpmG2PeNMbsMcbsNsZ8ZYR1LjHGtBhjtrmn74ai1mE1HTbG7HTrOW2Sf+P4Z3c77zDGXBCKOgPqmR+w/bYZY1qNMV8dtk7It7Mx5hFjTK0xZlfAbZnGmNeMMe+55xmj3PcWd533jDG3hLDe+40x+9y/+78bY9JHue+Y76Fprvk+Y0x1wN/+6lHuO2a+THPNzwbUe9gYs22U+05+O1trZ+QJ5yhUB4FSIA7YDiwats4XgQfdy58Eng1xzbOAC9zLKcCBEWq+BPivUG/fYTUdBrLHWH418DJggIuAt0Nd87D3yXGcscMzajsDFwMXALsCbvsJcK97+V7gxyPcLxMod88z3MsZIar3SiDWvfzjkeqdyHtommu+D/ifE3jfjJkv01nzsOX/AHw3WNt5JrfkVwHvW2vLrbXdwDPAdcPWuQ54zL38W+ByE8JDo1trj1lrt7qX24C9QEGo6gmi64DHrWMjkG6MmRXqolyXAwettWf6w7opY61dDzQOuznwPfsY8N9HuOuHgNestY3W2ibgNeCqKSvUNVK91tpXrbW97tWNQOFU1zEZo2zjiZhIvkyJsWp28+sG4OlgPd9MDvkC4EjA9SpOD8zBddw3YguQNS3VjcPtOloGvD3C4tXGmO3GmJeNMYuntbCRWeBVY8wWY8ydIyyfyN8iVD7J6P8QM207A+Raa4+5l48DuSOsM1O392dxvtGNZLz30HT7stvF9MgoXWIzdRuvAWqste+NsnzS23kmh3zYMsYkA88BX7XWtg5bvBWna+F84P8C/zHd9Y3gr621FwB/A3zJGHNxqAuaCGNMHHAt8JsRFs/E7XwK63z/DovhbcaYbwG9wJOjrDKT3kMPAHOBpcAxnO6PcHEjY7fiJ72dZ3LIVwOzA64XureNuI4xJhZIAxqmpbpRGGO8OAH/pLX2+eHLrbWt1tp29/JLgNcYkz3NZQ6vqdo9rwX+HeerbKCJ/C1C4W+ArdbamuELZuJ2dtUMdHW557UjrDOjtrcx5lbgI8BN7gfTaSbwHpo21toaa22ftbYf+OUotcyobQyDGfZR4NnR1jmT7TyTQ/4dYJ4xZo7bYvsk8MKwdV4ABkYefBx4Y7Q34XRw+9P+Ddhrrf3pKOvkDew3MMaswvkbhOyDyRjjM8akDFzG2dG2a9hqLwA3u6NsLgJaArocQmnUVs9M284BAt+ztwD/OcI6rwBXGmMy3K6GK93bpp0x5irgfwHXWms7R1lnIu+haTNsf9H1o9QykXyZblcA+6y1VSMtPOPtPB17k89iL/TVOCNUDgLfcm/7Ps4bDiAB56v6+8AmoDTE9f41ztfvHcA293Q18Hng8+46XwZ24+zN3wj8txDXXOrWst2ta2A7B9ZsgJ+7f4edwIoZ8N7w4YR2WsBtM2o743wAHQN6cPp8b8fZZ/R74D3gdSDTXXcF8HDAfT/rvq/fB24LYb3v4/RdD7yfB0az5QMvjfUeCmHNT7jv0x04wT1reM3u9dPyJVQ1u7c/OvD+DVj3rLezfvEqIhLBZnJ3jYiInCWFvIhIBFPIi4hEMIW8iEgEU8iLiEQwhbyISARTyIuIRDCFvIhIBPv/kkE1PVywgrwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot some data\n",
    "plt.plot(r.history['loss'], label='loss')\n",
    "plt.plot(r.history['val_loss'], label='val_loss')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Make predictions #####\n",
    "# As with the poetry example, we need to create another model\n",
    "# that can take in the RNN state and previous word as input\n",
    "# and accept a T=1 sequence.\n",
    "\n",
    "# The encoder will be stand-alone\n",
    "# From this we will get our initial decoder hidden state\n",
    "# i.e. h(1), ..., h(Tx)\n",
    "encoder_model = Model(encoder_inputs_placeholder, encoder_outputs)\n",
    "\n",
    "# next we define a T=1 decoder model\n",
    "encoder_outputs_as_input = Input(shape=(max_len_input, LATENT_DIM * 2,))\n",
    "decoder_inputs_single = Input(shape=(1,))\n",
    "decoder_inputs_single_x = decoder_embedding(decoder_inputs_single)\n",
    "\n",
    "# no need to loop over attention steps this time because there is only one step\n",
    "context = one_step_attention(encoder_outputs_as_input, initial_s)\n",
    "\n",
    "# combine context with last word\n",
    "decoder_lstm_input = context_last_word_concat_layer([context, decoder_inputs_single_x])\n",
    "\n",
    "# lstm and final dense\n",
    "o, s, c = decoder_lstm(decoder_lstm_input, initial_state=[initial_s, initial_c])\n",
    "decoder_outputs = decoder_dense(o)\n",
    "\n",
    "\n",
    "# note: we don't really need the final stack and tranpose\n",
    "# because there's only 1 output\n",
    "# it is already of size N x D\n",
    "# no need to make it 1 x N x D --> N x 1 x D\n",
    "\n",
    "\n",
    "\n",
    "# create the model object\n",
    "decoder_model = Model(\n",
    "  inputs=[\n",
    "    decoder_inputs_single,\n",
    "    encoder_outputs_as_input,\n",
    "    initial_s, \n",
    "    initial_c\n",
    "  ],\n",
    "  outputs=[decoder_outputs, s, c]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map indexes back into real words\n",
    "# so we can view the results\n",
    "idx2word_eng = {v:k for k, v in word2idx_inputs.items()}\n",
    "idx2word_trans = {v:k for k, v in word2idx_outputs.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "  # Encode the input as state vectors.\n",
    "  enc_out = encoder_model.predict(input_seq)\n",
    "\n",
    "  # Generate empty target sequence of length 1.\n",
    "  target_seq = np.zeros((1, 1))\n",
    "  \n",
    "  # Populate the first character of target sequence with the start character.\n",
    "  # NOTE: tokenizer lower-cases all words\n",
    "  target_seq[0, 0] = word2idx_outputs['<sos>']\n",
    "\n",
    "  # if we get this we break\n",
    "  eos = word2idx_outputs['<eos>']\n",
    "\n",
    "\n",
    "  # [s, c] will be updated in each loop iteration\n",
    "  s = np.zeros((1, LATENT_DIM_DECODER))\n",
    "  c = np.zeros((1, LATENT_DIM_DECODER))\n",
    "\n",
    "\n",
    "  # Create the translation\n",
    "  output_sentence = []\n",
    "  for _ in range(max_len_target):\n",
    "    o, s, c = decoder_model.predict([target_seq, enc_out, s, c])\n",
    "        \n",
    "\n",
    "    # Get next word\n",
    "    idx = np.argmax(o.flatten())\n",
    "\n",
    "    # End sentence of EOS\n",
    "    if eos == idx:\n",
    "      break\n",
    "\n",
    "    word = ''\n",
    "    if idx > 0:\n",
    "      word = idx2word_trans[idx]\n",
    "      output_sentence.append(word)\n",
    "\n",
    "    # Update the decoder input\n",
    "    # which is just the word just generated\n",
    "    target_seq[0, 0] = idx\n",
    "\n",
    "  return ' '.join(output_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-\n",
      "Input sentence: pleasantly surprised find item delivered day order speediest delivery date probably helps location near everything looks described pictured happy done business seller gingerbread house carefully packaged not harm fragile pieces inside excited give item gift\n",
      "Predicted translation: not dop\n",
      "Actual translation: rudolph gingerbread house <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: month old male shih tzu brought home breeder kept pro plan puppy tiny dry kibble bag almost finished cups left bag began walking away bowl soon put test rolled morsels across tile floor kitchen see would happen would eat pieces go bring one squeaker toys play happy not want play food game get eat went got blue buffalo dry puppy food lamb oatmeal formula bigger size kibbles seemed liked bigger sized items chew harder antioxidant bits since teething chewing dentabones venison antler lamb lung chips comfort got quarter food left bag would walk away bowl went almost days still small bowel movements urinating drinking water playing running continuing successful training exercises called vet wanted give wet venison formula science diet veterinarian diet exam showed nothing physically wrong one vet tech said shih tzu tend picky eaters not pick food check counter left began research breed natural diet know outdoor excursions attracted road kill sniff air want lunge would probably eat left devices not allow remember kid late early friends dogs fed choices available back gravy train alpo mixed meat drippings meat ate bones etc lived late teens one friends small mutt lived years old never saw dogs itching digging usual ears flea problem since grew rural area often common occurrance well read natural diets decided get sojos ground beef boneless chicken thighs breasts weston not big fan totally raw meat browned put toaster oven add tsp missing link meat portion soaked sojos original eats gusto early days would play take food away not food aggressive eats recommended portion size walks away done weston wants play hard train hard take nice walk nap coat soft easy keep brushed groomed droppings nice sized easy clean neutered days ago ate sojos mix gusto even e collar challenge fan finished second lb bag ordered another one hope helps someone picky eater know another reviewer mentioned shih tzu not liking shows dogs certain tastes different much like us humans\n",
      "Predicted translation: great dog food\n",
      "Actual translation: picky puppy <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: product gift far distant relative arrived right schedule perfect condition loved choices inside basket cannot attest food personally loved\n",
      "Predicted translation: great product\n",
      "Actual translation: perfect gift <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: fan full bodied coffees give one shot taste vast majority k cups watery weak even small cup setting gravitate towards extra bold varieties previously enjoying newman extra bold gm dark magic reading favorable comparisons mahogany decided give chance find slightly less bitter not bitter bad newman eb dark magic little smoother also think full bodied without burned bean flavor tried mahogany larger cup setting actually still liked something never said\n",
      "Predicted translation: great product\n",
      "Actual translation: best k cup date <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: banana pancakes not sure tastes without fruit best pancake long time thank making gluten free bisquick try biscuits next\n",
      "Predicted translation: great pancakes\n",
      "Actual translation: best gf pancake mix <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: since gluten free tried types gf breads expensive knew brazilians bread game lock inexpensive tapioca version cheese bread custom probably skinny lol loved ordering whole lot\n",
      "Predicted translation: great pancakes\n",
      "Actual translation: able eat bread <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: ok bought bisquick gf first thing tried make drop biscuits recipe listed box not kind rolled cut drop biscuits results tasty taste gf bisquick much like original taste not tart original want similar buttermilk taste need use buttermilk instead milk biscuit recipe noticed something else odd might problem anyone new cooking seems amount mix called not quite right read side box claims cups mix box servings cup biscuit recipe called cups mix measured heaping teaspoonfuls measuring cup not even cup mix left bag know mix not wheat flour maybe kind flour measures differently maybe dug bag cup would used less recipe came right must misprint box assuming measured properly biscuits dipped gravy smothered butter jam sliced make yummy mcdonald breakfast sandwiches following instructions lot biscuits single cooking two might want cut recipe half still plenty not go box quickly mix cost around not enough left make anything else without buying another box biscuits really really good still watching budget not prepare need biscuits yummy fairly golden recipe calls eggs sliced crumbly edges like glutenous ones without eggs probably not get nice eye appealing color even though drop biscuits still cut shortening margarine not cutter need use long fork difficult celiac craving biscuits buy cutter cost much box mix biscuit recipe also calls milk want nice tart taste reminiscent original bisquick need use buttermilk mix made rice flour sugar leavening potato starch salt xanthum gum many mixes goodies made rice flour taste grainy bisquick done fantastic job rice flour not grainy repeat not grainy new celiac cooking using starch corn potato tapioca normal adds lightness baked item would not possible rice flour alone leavening simply baking soda powder necessary rising sugar salt flavor consistency yes necessary xanthum gum perfectly natural ingredient gives celiac safe baked goods bit elasticity acting replacement gluten find xanthum gum every baked goodie made people celiac disease make mix spend less money sure difficult time finding rice flour finely ground rice flour predominant main ingredient mix recipes listed box pancakes waffles strawberry shortcakes pizza crust ultimate chicken fingers oven baked chicken find recipes website recommend product taste wish mix box price\n",
      "Predicted translation: great product\n",
      "Actual translation: tasty biscuits <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: product must celiacs garlic biscuits pizza crusts make amazing\n",
      "Predicted translation: great product\n",
      "Actual translation: yummm <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: arrived slightly thawed parents would not accept however company helpful issued full refund\n",
      "Predicted translation: great product\n",
      "Actual translation: great support <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: course not taste quite good regular brownies hello healthy said fantastic eat crap good alternatives\n",
      "Predicted translation: great product\n",
      "Actual translation: grrrrrrrreat <eos>\n",
      "Continue? [Y/n]Y\n",
      "-\n",
      "Input sentence: chips nasty thought someone spilled drink bag no chips soaked grease nasty\n",
      "Predicted translation: not good\n",
      "Actual translation: disgusting <eos>\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "  # Do some test translations\n",
    "  i = np.random.choice(len(input_texts))\n",
    "  input_seq = encoder_inputs[i:i+1]\n",
    "  translation = decode_sequence(input_seq)\n",
    "  print('-')\n",
    "  print('Input sentence:', input_texts[i])\n",
    "  print('Predicted translation:', translation)\n",
    "  print('Actual translation:', target_texts[i])\n",
    "\n",
    "  ans = input(\"Continue? [Y/n]\")\n",
    "  if ans and ans.lower().startswith('n'):\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}