Skip-gram.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(13)\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Reshape\n",
    "from IPython.display import SVG\n",
    "from keras.utils import np_utils\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.preprocessing.text import Tokenizer, base_filter\n",
    "from keras.utils.visualize_util import model_to_dot, plot\n",
    "from gensim.models.doc2vec import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = get_file('alice.txt', origin=\"http://www.gutenberg.org/cache/epub/11/pg11.txt\")\n",
    "corpus = open(path).readlines()[0:200]\n",
    "\n",
    "corpus = [sentence for sentence in corpus if sentence.count(\" \") >= 2]\n",
    "\n",
    "tokenizer = Tokenizer(filters=base_filter()+\"'\")\n",
    "tokenizer.fit_on_texts(corpus)\n",
    "corpus = tokenizer.texts_to_sequences(corpus)\n",
    "nb_samples = sum(len(s) for s in corpus)\n",
    "V = len(tokenizer.word_index) + 1\n",
    "dim = 100\n",
    "window_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_data(corpus, window_size, V):\n",
    "    maxlen = window_size*2\n",
    "    for words in corpus:\n",
    "        L = len(words)\n",
    "        for index, word in enumerate(words):\n",
    "            s = index-window_size\n",
    "            e = index+window_size+1\n",
    "                    \n",
    "            in_words = []\n",
    "            labels = []\n",
    "            for i in range(s, e):\n",
    "                if i != index and 0 <= i < L:\n",
    "                    in_words.append([word] )\n",
    "                    labels.append(words[i])\n",
    "\n",
    "            x = np.array(in_words,dtype=np.int32)\n",
    "            y = np_utils.to_categorical(labels, V)\n",
    "            yield (x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"296pt\" viewBox=\"0.00 0.00 336.32 296.00\" width=\"336pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 292)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-292 332.321,-292 332.321,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 4567794968 -->\n",
       "<g class=\"node\" id=\"node1\"><title>4567794968</title>\n",
       "<polygon fill=\"none\" points=\"0,-243.5 0,-287.5 328.321,-287.5 328.321,-243.5 0,-243.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"101.502\" y=\"-261.3\">embedding_input_3 (InputLayer)</text>\n",
       "<polyline fill=\"none\" points=\"203.004,-243.5 203.004,-287.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"230.838\" y=\"-272.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"203.004,-265.5 258.673,-265.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"230.838\" y=\"-250.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"258.673,-243.5 258.673,-287.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"293.497\" y=\"-272.3\">(None, 1)</text>\n",
       "<polyline fill=\"none\" points=\"258.673,-265.5 328.321,-265.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"293.497\" y=\"-250.3\">(None, 1)</text>\n",
       "</g>\n",
       "<!-- 4567794856 -->\n",
       "<g class=\"node\" id=\"node2\"><title>4567794856</title>\n",
       "<polygon fill=\"none\" points=\"2.71387,-162.5 2.71387,-206.5 325.607,-206.5 325.607,-162.5 2.71387,-162.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"87.502\" y=\"-180.3\">embedding_3 (Embedding)</text>\n",
       "<polyline fill=\"none\" points=\"172.29,-162.5 172.29,-206.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"200.125\" y=\"-191.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"172.29,-184.5 227.959,-184.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"200.125\" y=\"-169.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.959,-162.5 227.959,-206.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"276.783\" y=\"-191.3\">(None, 1)</text>\n",
       "<polyline fill=\"none\" points=\"227.959,-184.5 325.607,-184.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"276.783\" y=\"-169.3\">(None, 1, 100)</text>\n",
       "</g>\n",
       "<!-- 4567794968&#45;&gt;4567794856 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>4567794968-&gt;4567794856</title>\n",
       "<path d=\"M164.161,-243.329C164.161,-235.183 164.161,-225.699 164.161,-216.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"167.661,-216.729 164.161,-206.729 160.661,-216.729 167.661,-216.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 4567794352 -->\n",
       "<g class=\"node\" id=\"node3\"><title>4567794352</title>\n",
       "<polygon fill=\"none\" points=\"21,-81.5 21,-125.5 307.321,-125.5 307.321,-81.5 21,-81.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"87.502\" y=\"-99.3\">reshape_3 (Reshape)</text>\n",
       "<polyline fill=\"none\" points=\"154.004,-81.5 154.004,-125.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181.838\" y=\"-110.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"154.004,-103.5 209.673,-103.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181.838\" y=\"-88.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"209.673,-81.5 209.673,-125.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258.497\" y=\"-110.3\">(None, 1, 100)</text>\n",
       "<polyline fill=\"none\" points=\"209.673,-103.5 307.321,-103.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258.497\" y=\"-88.3\">(None, 100)</text>\n",
       "</g>\n",
       "<!-- 4567794856&#45;&gt;4567794352 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>4567794856-&gt;4567794352</title>\n",
       "<path d=\"M164.161,-162.329C164.161,-154.183 164.161,-144.699 164.161,-135.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"167.661,-135.729 164.161,-125.729 160.661,-135.729 167.661,-135.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 4567757152 -->\n",
       "<g class=\"node\" id=\"node4\"><title>4567757152</title>\n",
       "<polygon fill=\"none\" points=\"39.6587,-0.5 39.6587,-44.5 288.663,-44.5 288.663,-0.5 39.6587,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"94.502\" y=\"-18.3\">dense_3 (Dense)</text>\n",
       "<polyline fill=\"none\" points=\"149.345,-0.5 149.345,-44.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"177.18\" y=\"-29.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"149.345,-22.5 205.014,-22.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"177.18\" y=\"-7.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"205.014,-0.5 205.014,-44.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"246.838\" y=\"-29.3\">(None, 100)</text>\n",
       "<polyline fill=\"none\" points=\"205.014,-22.5 288.663,-22.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"246.838\" y=\"-7.3\">(None, 573)</text>\n",
       "</g>\n",
       "<!-- 4567794352&#45;&gt;4567757152 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>4567794352-&gt;4567757152</title>\n",
       "<path d=\"M164.161,-81.3294C164.161,-73.1826 164.161,-63.6991 164.161,-54.7971\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"167.661,-54.729 164.161,-44.729 160.661,-54.729 167.661,-54.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skipgram = Sequential()\n",
    "skipgram.add(Embedding(input_dim=V, output_dim=dim, init='glorot_uniform', input_length=1))\n",
    "skipgram.add(Reshape((dim, )))\n",
    "skipgram.add(Dense(input_dim=dim, output_dim=V, activation='softmax'))\n",
    "SVG(model_to_dot(skipgram, show_shapes=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "skipgram.compile(loss='categorical_crossentropy', optimizer=\"adadelta\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 10880.5715966\n",
      "1 10205.9945803\n",
      "2 10074.3077066\n",
      "3 9965.59868503\n",
      "4 9854.57731271\n",
      "5 9750.60724521\n",
      "6 9657.09119558\n",
      "7 9572.85030174\n",
      "8 9495.95489979\n",
      "9 9424.25686908\n"
     ]
    }
   ],
   "source": [
    "for ite in range(10):\n",
    "    loss = 0.\n",
    "    for x, y in generate_data(corpus, window_size, V):\n",
    "        loss += skipgram.train_on_batch(x, y)\n",
    "\n",
    "    print(ite, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = open('vectors.txt' ,'w')\n",
    "f.write(\" \".join([str(V-1),str(dim)]))\n",
    "f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectors = skipgram.get_weights()[0]\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    f.write(word)\n",
    "    f.write(\" \")\n",
    "    f.write(\" \".join(map(str, list(vectors[i,:]))))\n",
    "    f.write(\"\\n\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v = Word2Vec.load_word2vec_format('./vectors.txt', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('s', 0.42469197511672974),\n",
       " ('too', 0.4138217866420746),\n",
       " ('whether', 0.3842674493789673),\n",
       " ('wonderland', 0.368587851524353),\n",
       " ('in', 0.36720913648605347),\n",
       " ('that', 0.3651423156261444),\n",
       " ('adventures', 0.3534693419933319),\n",
       " ('labelled', 0.31480032205581665),\n",
       " ('yes', 0.30488651990890503),\n",
       " ('got', 0.29601991176605225)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['alice'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}