Multi-channel Convolutional Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "from os import listdir\n",
    "from nltk.corpus import stopwords\n",
    "from pickle import dump\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load doc into memory\n",
    "def load_doc(filename):\n",
    "    file = open(filename, 'r')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc(doc):\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    tokens = [word for word in tokens if word.isalpha()]\n",
    "    # filter out stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    tokens = [w for w in tokens if not w in stop_words]\n",
    "    # filter out short tokens\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load all docs in a directory\n",
    "def process_docs(directory, is_trian):\n",
    "    documents = list()\n",
    "    # walk through all files in the folder\n",
    "    for filename in listdir(directory):\n",
    "        # skip any reviews in the test set\n",
    "        if is_trian and filename.startswith('cv9'):\n",
    "            continue\n",
    "        if not is_trian and not filename.startswith('cv9'):\n",
    "            continue\n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # load the doc\n",
    "        doc = load_doc(path)\n",
    "        # clean doc\n",
    "        tokens = clean_doc(doc)\n",
    "        # add to list\n",
    "        documents.append(tokens)\n",
    "    return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save a dataset to file\n",
    "def save_dataset(dataset, filename):\n",
    "    dump(dataset, open(filename, 'wb'))\n",
    "    print('Saved: %s' % filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved: models/imdb_train.pkl\n"
     ]
    }
   ],
   "source": [
    "# load all training reviews\n",
    "negative_docs = process_docs('datasets/review_polarity/txt_sentoken/neg', True)\n",
    "positive_docs = process_docs('datasets/review_polarity/txt_sentoken/pos', True)\n",
    "trainX = negative_docs + positive_docs\n",
    "trainy = [0 for _ in range(900)] + [1 for _ in range(900)]\n",
    "save_dataset([trainX,trainy], 'models/imdb_train.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainy[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved: models/imdb_test.pkl\n"
     ]
    }
   ],
   "source": [
    "# load all test reviews\n",
    "negative_docs = process_docs('datasets/review_polarity/txt_sentoken/neg', False)\n",
    "positive_docs = process_docs('datasets/review_polarity/txt_sentoken/pos', False)\n",
    "testX = negative_docs + positive_docs\n",
    "testY = [0 for _ in range(100)] + [1 for _ in range(100)]\n",
    "save_dataset([testX,testY], 'models/imdb_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pickle import load\n",
    "from numpy import array\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.models import Model\n",
    "from keras.layers import Input\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Embedding\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load a clean dataset\n",
    "def load_dataset(filename):\n",
    "    return load(open(filename, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fit a tokenizer\n",
    "def create_tokenizer(lines):\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(lines)\n",
    "    return tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# calculate the maximum document length\n",
    "def max_length(lines):\n",
    "    return max([len(s) for s in lines])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# encode a list of lines\n",
    "def encode_text(tokenizer, lines, length):\n",
    "    # integer encode\n",
    "    encoded = tokenizer.texts_to_sequences(lines)\n",
    "    # pad encoded sequences\n",
    "    padded = pad_sequences(encoded, maxlen=length, padding='post')\n",
    "    return padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define the model\n",
    "def define_model(length, vocab_size):\n",
    "    # channel 1\n",
    "    inputs1 = Input(shape=(length,))\n",
    "    embedding1 = Embedding(vocab_size, 100)(inputs1)\n",
    "    conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding1)\n",
    "    drop1 = Dropout(0.5)(conv1)\n",
    "    pool1 = MaxPooling1D(pool_size=2)(drop1)\n",
    "    flat1 = Flatten()(pool1)\n",
    "    # channel 2\n",
    "    inputs2 = Input(shape=(length,))\n",
    "    embedding2 = Embedding(vocab_size, 100)(inputs2)\n",
    "    conv2 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding2)\n",
    "    drop2 = Dropout(0.5)(conv2)\n",
    "    pool2 = MaxPooling1D(pool_size=2)(drop2)\n",
    "    flat2 = Flatten()(pool2)\n",
    "    # channel 3\n",
    "    inputs3 = Input(shape=(length,))\n",
    "    embedding3 = Embedding(vocab_size, 100)(inputs3)\n",
    "    conv3 = Conv1D(filters=32, kernel_size=8, activation='relu')(embedding3)\n",
    "    drop3 = Dropout(0.5)(conv3)\n",
    "    pool3 = MaxPooling1D(pool_size=2)(drop3)\n",
    "    flat3 = Flatten()(pool3)\n",
    "    # merge\n",
    "    merged = concatenate([flat1, flat2, flat3])\n",
    "    # interpretation\n",
    "    dense1 = Dense(10, activation='relu')(merged)\n",
    "    outputs = Dense(1, activation='sigmoid')(dense1)\n",
    "    model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)\n",
    "    # compile\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    # summarize\n",
    "    print(model.summary())\n",
    "    plot_model(model, show_shapes=True, to_file='models/multichannel.png')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load training dataset\n",
    "trainLines, trainLabels = load_dataset('models/imdb_train.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create tokenizer\n",
    "tokenizer = create_tokenizer(trainLines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max document length: 1380\n",
      "Vocabulary size: 44277\n"
     ]
    }
   ],
   "source": [
    "# calculate max document length\n",
    "length = max_length(trainLines)\n",
    "# calculate vocabulary size\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "print('Max document length: %d' % length)\n",
    "print('Vocabulary size: %d' % vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1800, 1380)\n"
     ]
    }
   ],
   "source": [
    "# encode data\n",
    "trainX = encode_text(tokenizer, trainLines, length)\n",
    "print(trainX.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 1380)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 1380)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 1380)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 1380, 100)    4427700     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1380, 100)    4427700     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1380, 100)    4427700     input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 1377, 32)     12832       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 1375, 32)     19232       embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 1373, 32)     25632       embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1377, 32)     0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 1375, 32)     0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 1373, 32)     0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 688, 32)      0           dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 687, 32)      0           dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)  (None, 686, 32)      0           dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 22016)        0           max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 21984)        0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 21952)        0           max_pooling1d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 65952)        0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           659530      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            11          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 14,000,337\n",
      "Trainable params: 14,000,337\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = define_model(length, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1800/1800 [==============================] - 43s 24ms/step - loss: 0.6941 - acc: 0.5444\n",
      "Epoch 2/10\n",
      "1800/1800 [==============================] - 44s 24ms/step - loss: 0.5008 - acc: 0.7667\n",
      "Epoch 3/10\n",
      "1800/1800 [==============================] - 42s 24ms/step - loss: 0.0899 - acc: 0.9689\n",
      "Epoch 4/10\n",
      "1800/1800 [==============================] - 41s 23ms/step - loss: 0.0063 - acc: 1.0000\n",
      "Epoch 5/10\n",
      "1800/1800 [==============================] - 41s 23ms/step - loss: 0.0019 - acc: 1.0000\n",
      "Epoch 6/10\n",
      "1800/1800 [==============================] - 41s 23ms/step - loss: 0.0012 - acc: 1.0000\n",
      "Epoch 7/10\n",
      "1800/1800 [==============================] - 42s 23ms/step - loss: 8.8242e-04 - acc: 1.0000\n",
      "Epoch 8/10\n",
      "1800/1800 [==============================] - 44s 24ms/step - loss: 6.5699e-04 - acc: 1.0000\n",
      "Epoch 9/10\n",
      "1800/1800 [==============================] - 43s 24ms/step - loss: 5.3815e-04 - acc: 1.0000\n",
      "Epoch 10/10\n",
      "1800/1800 [==============================] - 44s 24ms/step - loss: 4.3466e-04 - acc: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x10f58df98>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit model\n",
    "model.fit([trainX,trainX,trainX], array(trainLabels), epochs=10, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save the model\n",
    "model.save('models/imdb_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1800, 1380) (200, 1380)\n"
     ]
    }
   ],
   "source": [
    "testLines, testLabels = load_dataset('models/imdb_test.pkl')\n",
    "testX = encode_text(tokenizer, testLines, length)\n",
    "print(trainX.shape, testX.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model = load_model('models/imdb_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 100.000000\n",
      "Test Accuracy: 84.000000\n"
     ]
    }
   ],
   "source": [
    "# evaluate model on training dataset\n",
    "loss, acc = model.evaluate([trainX,trainX,trainX], array(trainLabels), verbose=0)\n",
    "print('Train Accuracy: %f' % (acc*100))\n",
    " \n",
    "# evaluate model on test dataset dataset\n",
    "loss, acc = model.evaluate([testX,testX,testX],array(testLabels), verbose=0)\n",
    "print('Test Accuracy: %f' % (acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}