lstm_keras_text_gen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Small experiment to train LSTM model and let it generate automatic text generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import random\n",
    "from numpy import array\n",
    "from matplotlib import pyplot\n",
    "from matplotlib.patches import PathPatch\n",
    "from matplotlib.path import Path\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load ascii text and covert to lowercase\n",
    "filename = \"wonderland.txt\"\n",
    "raw_text = open(filename).read()\n",
    "raw_text = raw_text.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create mapping of unique chars to integers\n",
    "chars = sorted(list(set(raw_text)))\n",
    "char_to_int = dict((c, i) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Characters:  163815\n",
      "Total Vocab:  60\n"
     ]
    }
   ],
   "source": [
    "n_chars = len(raw_text)\n",
    "n_vocab = len(chars)\n",
    "print (\"Total Characters: \", n_chars)\n",
    "print (\"Total Vocab: \", n_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Patterns:  163715\n"
     ]
    }
   ],
   "source": [
    "# prepare the dataset of input to output pairs encoded as integers\n",
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in range(0, n_chars - seq_length, 1):\n",
    "\tseq_in = raw_text[i:i + seq_length]\n",
    "\tseq_out = raw_text[i + seq_length]\n",
    "\tdataX.append([char_to_int[char] for char in seq_in])\n",
    "\tdataY.append(char_to_int[seq_out])\n",
    "n_patterns = len(dataX)\n",
    "print (\"Total Patterns: \", n_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reshape X to be [samples, time steps, features]\n",
    "X = numpy.reshape(dataX, (n_patterns, seq_length, 1))\n",
    "# normalize\n",
    "X = X / float(n_vocab)\n",
    "# one hot encode the output variable\n",
    "y = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the checkpoint\n",
    "filepath=\"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.5625Epoch 00000: loss improved from 2.62014 to 2.56247, saving model to weights-improvement-00-2.5625.hdf5\n",
      "163715/163715 [==============================] - 902s - loss: 2.5625   \n",
      "Epoch 2/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.5063Epoch 00001: loss improved from 2.56247 to 2.50628, saving model to weights-improvement-01-2.5063.hdf5\n",
      "163715/163715 [==============================] - 1187s - loss: 2.5063  \n",
      "Epoch 3/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.4582Epoch 00002: loss improved from 2.50628 to 2.45821, saving model to weights-improvement-02-2.4582.hdf5\n",
      "163715/163715 [==============================] - 1316s - loss: 2.4582  \n",
      "Epoch 4/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.4127Epoch 00003: loss improved from 2.45821 to 2.41269, saving model to weights-improvement-03-2.4127.hdf5\n",
      "163715/163715 [==============================] - 1304s - loss: 2.4127  \n",
      "Epoch 5/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.3700Epoch 00004: loss improved from 2.41269 to 2.36999, saving model to weights-improvement-04-2.3700.hdf5\n",
      "163715/163715 [==============================] - 1328s - loss: 2.3700  \n",
      "Epoch 6/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.3300Epoch 00005: loss improved from 2.36999 to 2.32999, saving model to weights-improvement-05-2.3300.hdf5\n",
      "163715/163715 [==============================] - 1350s - loss: 2.3300  \n",
      "Epoch 7/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.2918Epoch 00006: loss improved from 2.32999 to 2.29182, saving model to weights-improvement-06-2.2918.hdf5\n",
      "163715/163715 [==============================] - 1435s - loss: 2.2918  \n",
      "Epoch 8/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.2561Epoch 00007: loss improved from 2.29182 to 2.25616, saving model to weights-improvement-07-2.2562.hdf5\n",
      "163715/163715 [==============================] - 1277s - loss: 2.2562  \n",
      "Epoch 9/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.2233Epoch 00008: loss improved from 2.25616 to 2.22329, saving model to weights-improvement-08-2.2233.hdf5\n",
      "163715/163715 [==============================] - 1327s - loss: 2.2233  \n",
      "Epoch 10/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.1911Epoch 00009: loss improved from 2.22329 to 2.19113, saving model to weights-improvement-09-2.1911.hdf5\n",
      "163715/163715 [==============================] - 1490s - loss: 2.1911  \n",
      "Epoch 11/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.1618Epoch 00010: loss improved from 2.19113 to 2.16184, saving model to weights-improvement-10-2.1618.hdf5\n",
      "163715/163715 [==============================] - 1476s - loss: 2.1618  \n",
      "Epoch 12/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.1339Epoch 00011: loss improved from 2.16184 to 2.13394, saving model to weights-improvement-11-2.1339.hdf5\n",
      "163715/163715 [==============================] - 1367s - loss: 2.1339  \n",
      "Epoch 13/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.1083Epoch 00012: loss improved from 2.13394 to 2.10832, saving model to weights-improvement-12-2.1083.hdf5\n",
      "163715/163715 [==============================] - 1349s - loss: 2.1083  \n",
      "Epoch 14/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.0835Epoch 00013: loss improved from 2.10832 to 2.08350, saving model to weights-improvement-13-2.0835.hdf5\n",
      "163715/163715 [==============================] - 1344s - loss: 2.0835  \n",
      "Epoch 15/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.0583Epoch 00014: loss improved from 2.08350 to 2.05830, saving model to weights-improvement-14-2.0583.hdf5\n",
      "163715/163715 [==============================] - 1296s - loss: 2.0583  \n",
      "Epoch 16/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.0315Epoch 00015: loss improved from 2.05830 to 2.03151, saving model to weights-improvement-15-2.0315.hdf5\n",
      "163715/163715 [==============================] - 1288s - loss: 2.0315  \n",
      "Epoch 17/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 2.0137Epoch 00016: loss improved from 2.03151 to 2.01370, saving model to weights-improvement-16-2.0137.hdf5\n",
      "163715/163715 [==============================] - 1322s - loss: 2.0137  \n",
      "Epoch 18/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 1.9928Epoch 00017: loss improved from 2.01370 to 1.99279, saving model to weights-improvement-17-1.9928.hdf5\n",
      "163715/163715 [==============================] - 1349s - loss: 1.9928  \n",
      "Epoch 19/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 1.9731Epoch 00018: loss improved from 1.99279 to 1.97314, saving model to weights-improvement-18-1.9731.hdf5\n",
      "163715/163715 [==============================] - 1390s - loss: 1.9731  \n",
      "Epoch 20/20\n",
      "163712/163715 [============================>.] - ETA: 0s - loss: 1.9582Epoch 00019: loss improved from 1.97314 to 1.95825, saving model to weights-improvement-19-1.9582.hdf5\n",
      "163715/163715 [==============================] - 1424s - loss: 1.9582  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x19d7facfda0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if 'session' in locals() and session is not None:\n",
    "    print('Close interactive session')\n",
    "    session.close()\n",
    "model.fit(X, y, epochs=20, batch_size=128, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Now that the LSTM model has been trained let's try to see how it generates text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Randomly choose a starting text pattern:\n",
      "\"  them hit her in the face. ‘i’ll put a stop to this,’\n",
      "she said to herself, and shouted out, ‘you’d b \"\n",
      "eteer to tel the bormouse of the latter of the care tith the corrd, and she san oo anice, and she tent hnt lo the would bell aoong the was oo the tan  the wondd oo the was oow io whsh then a lottle saaten thee the was soink the was oo the tooe of the while \n",
      "a  ‘hh poen tie whst shael to tey an thee, and then soen io the would bell ao thle to an all dorn th the tond, bad aelen to tei toen i shilld toiee to soonk the was oo the tand oi the thnt of the harte haree herden the was oo the table, \n",
      "‘hele is as thl sieeg that she sas,’ the macc tirhle  she houkd aolng to the thitg sabbit, and she was notting an the could, and soon the harter sam the was oot in she woudd fo would hot theer hor hoad to the kaster, and the whnt hnrdle an the could, and soon th the woodd aod lottle thin she was oo the tand oi the thnt of the harte haree seid the whst of the garee harde herden she had soe tas go the douse, and saed  she hound she looke oo the goophsn shate ths oo the was oo tae she tai  the hound do\n",
      " Text generation done.\n"
     ]
    }
   ],
   "source": [
    "# start with random text\n",
    "import sys\n",
    "int_to_char = dict((i, c) for i, c in enumerate(chars))\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print (\"Randomly choose a starting text pattern:\")\n",
    "print (\"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
    "\n",
    "# LSTM model taking the random seed text now starting to generate the text\n",
    "for i in range(1000):\n",
    "\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "\tx = x / float(n_vocab)\n",
    "\tprediction = model.predict(x, verbose=0)\n",
    "\tindex = numpy.argmax(prediction)\n",
    "\tresult = int_to_char[index]\n",
    "\tseq_in = [int_to_char[value] for value in pattern]\n",
    "\tsys.stdout.write(result)\n",
    "\tpattern.append(index)\n",
    "\tpattern = pattern[1:len(pattern)]\n",
    "print(\"\\n Text generation done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The above model took almost 12 hours to be trained and the text generation as can be seen above is not quite that correct. This model can be trained more than 20 epochs \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n",
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "#save the model and weights in Yaml format\n",
    "from keras.models import model_from_json\n",
    "from keras.models import model_from_yaml\n",
    "import numpy\n",
    "import os\n",
    "\n",
    "numpy.random.seed(12345)\n",
    "\n",
    "model.save('I:\\\\LSTM\\\\wonderland_model.h5')\n",
    "model.save_weights('I:\\\\LSTM\\\\weights_model_yaml.h5')\n",
    "# serialize model to YAML\n",
    "model_yaml = model.to_yaml()\n",
    "with open(\"I:\\\\LSTM\\\\wonderland_model.yaml\", \"w\") as yaml_file:\n",
    "    yaml_file.write(model_yaml)\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"I:\\\\LSTM\\\\weights_model_yaml.h5\")\n",
    "print(\"Saved model to disk\")\n",
    "\n",
    "# load YAML and create model\n",
    "yaml_file = open('I:\\\\LSTM\\\\wonderland_model.yaml', 'r')\n",
    "loaded_model_yaml = yaml_file.read()\n",
    "yaml_file.close()\n",
    "loaded_model = model_from_yaml(loaded_model_yaml)\n",
    "# load weights into new model\n",
    "loaded_model.load_weights(\"I:\\\\LSTM\\\\weights_model_yaml.h5\")\n",
    "print(\"Loaded model from disk\")\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n",
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "#save the model and weights in json format\n",
    "model_json = model.to_json()\n",
    "with open(\"d:\\\\LSTM\\\\wonderland_model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"d:\\\\LSTM\\\\wonderland_weigths_json.h5\")\n",
    "print(\"Saved model to disk\")\n",
    "\n",
    "\n",
    "json_file = open('d:\\\\LSTM\\\\wonderland_model.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "loaded_model = model_from_json(loaded_model_json)\n",
    "# load weights into new model\n",
    "loaded_model.load_weights(\"d:\\\\LSTM\\\\wonderland_weigths_json.h5\")\n",
    "print(\"Loaded model from disk\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}