pos_blstm_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pos_blstm_cnn.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHI_vtGDuKt9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip uninstall tensorflow\n",
        "!pip install tensorflow-gpu==1.15\n",
        "!pip install git+https://github.com/guillaumegenthial/tf_metrics.git\n",
        "\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZJ7tDnc7s76q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Config:\n",
        "    def __init__(self):\n",
        "        # directories\n",
        "        self.train_data_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/data/bijankhan_corpus.tsv'\n",
        "        self.model_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/model_2'\n",
        "        self.we_model_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/data/cc.fa.300.vec'\n",
        "        self.we_pickled_model_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/data/cc.fa.300.pickle'\n",
        "\n",
        "        # general\n",
        "        self.data_split = .1\n",
        "        self.num_epochs = 25\n",
        "        self.batch_size = 4\n",
        "        self.shuffle_buffer = 320000\n",
        "        self.num_tags = 15\n",
        "        self.num_pos_tags = 14\n",
        "        self.word_max_len = 30\n",
        "        self.learning_rate = 1e-3\n",
        "        self.max_len = 1276\n",
        "\n",
        "        # embeddings\n",
        "        self.num_words = 100000\n",
        "        self.word_embed_dim = 300\n",
        "        self.num_chars = 256  # number of most frequent characters to be kept\n",
        "        self.char_embed_dim = 32\n",
        "        self.pos_embed_dim = 16\n",
        "\n",
        "        # lstm variables\n",
        "        self.lstm_units = 256  # number of hidden units in the RNN\n",
        "        self.dropout = .5  # keeping probability"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GarAaRl8lvft",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import random\n",
        "import pickle\n",
        "from collections import Counter\n",
        "import sys\n",
        "\n",
        "import numpy as np\n",
        "from gensim.models import KeyedVectors\n",
        "\n",
        "\n",
        "cfg = Config()\n",
        "\n",
        "class DataLoader:\n",
        "    def __init__(self):\n",
        "        # loading word embedding model\n",
        "        try:\n",
        "            with open(cfg.we_pickled_model_dir, 'rb') as handle:\n",
        "                self.word_embedding_model = pickle.load(handle)\n",
        "        except FileNotFoundError:\n",
        "            self.word_embedding_model = KeyedVectors.load_word2vec_format(cfg.we_model_dir, binary=False)\n",
        "            with open(cfg.we_pickled_model_dir, 'wb') as handle:\n",
        "                pickle.dump(self.word_embedding_model, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "        sents, all_pos_tags, all_ezafe_tags = self._data_reader(cfg.train_data_dir)\n",
        "\n",
        "        sents_shuf = []\n",
        "        all_pos_tags_shuf = []\n",
        "        all_ezafe_tags_shuf = []\n",
        "        index_shuf = list(range(len(sents)))\n",
        "\n",
        "        for i in index_shuf:\n",
        "            sents_shuf.append(sents[i])\n",
        "            all_pos_tags_shuf.append(all_pos_tags[i])\n",
        "            all_ezafe_tags_shuf.append(all_ezafe_tags[i])\n",
        "\n",
        "        random.seed(17)\n",
        "        random.shuffle(index_shuf)\n",
        "        data_split_1 = int(len(sents_shuf) * .1)\n",
        "        data_split_2 = int(len(sents_shuf) * .2)\n",
        "        \n",
        "        self.test_data = sents_shuf[:data_split_1], all_pos_tags_shuf[:data_split_1], all_ezafe_tags_shuf[:data_split_1]\n",
        "        self.valid_data = sents_shuf[data_split_1:data_split_2], all_pos_tags_shuf[data_split_1:data_split_2], all_ezafe_tags_shuf[data_split_1:data_split_2]\n",
        "        self.train_data = sents_shuf[data_split_2:], all_pos_tags_shuf[data_split_2:], all_ezafe_tags_shuf[data_split_2:]            \n",
        "        \n",
        "        print('train data:', len(self.train_data[0]))\n",
        "        print('validation data:', len(self.valid_data[0]))\n",
        "        print('test data:', len(self.test_data[0]))\n",
        "        \n",
        "        try:\n",
        "            with open('/content/drive/My Drive/Colab Notebooks/ezafe/data/indices.pickle', 'rb') as handle:\n",
        "                self.char_to_index, self.word_to_index, self.pos_tag_to_index, self.ezafe_tag_to_index = pickle.load(handle)\n",
        "            \n",
        "            print(self.pos_tag_to_index)\n",
        "\n",
        "            self.index_to_word = {i: key for key, i in self.word_to_index.items()}\n",
        "            self.index_to_ezafe_tag = {i: key for key, i in self.ezafe_tag_to_index.items()}\n",
        "        \n",
        "            sents, all_pos_tags, all_ezafe_tags = self._data_reader(cfg.train_data_dir)\n",
        "\n",
        "        except FileNotFoundError:\n",
        "            print('Building vocabulary...')\n",
        "\n",
        "            vocab_list = []\n",
        "            char_list = []\n",
        "            for sent in self.train_data[0]:\n",
        "                for word in sent:\n",
        "                    vocab_list.append(word)\n",
        "                    for char in word:\n",
        "                        char_list.append(char)\n",
        "            \n",
        "            most_common_words = Counter(vocab_list).most_common(cfg.num_words)\n",
        "            most_common_chars = Counter(char_list).most_common(cfg.num_chars)\n",
        "            \n",
        "            self.word_to_index = {}\n",
        "            for i, pair in enumerate([('<PAD>', 0)] + most_common_words):\n",
        "                self.word_to_index[pair[0]] = i + 1\n",
        "\n",
        "            self.char_to_index = {}\n",
        "            for i, pair in enumerate([('<PAD>', 0), ('<UNK>', 1)] + most_common_chars):\n",
        "                self.char_to_index[pair[0]] = i + 1\n",
        "            \n",
        "            self.pos_tag_to_index = {}\n",
        "            for i, tag in enumerate(set(x for y in self.train_data[1] for x in y)):\n",
        "                self.pos_tag_to_index[tag] = i \n",
        "\n",
        "            self.ezafe_tag_to_index = {'0': 0, '1': 1}\n",
        "\n",
        "            self.index_to_word = {i: key for key, i in self.word_to_index.items()}\n",
        "            self.index_to_ezafe_tag = {i: key for key, i in self.ezafe_tag_to_index.items()}\n",
        "\n",
        "            # saving the tokenizers\n",
        "            with open('/content/drive/My Drive/Colab Notebooks/ezafe/data/indices.pickle', 'wb') as handle:\n",
        "                indices = self.char_to_index, self.word_to_index, self.pos_tag_to_index, self.ezafe_tag_to_index\n",
        "                pickle.dump(indices, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "\n",
        "    def _data_reader(self, directory):\n",
        "        sents, sent = [], []\n",
        "        all_ezafe_tags, ezafe_tags = [], []\n",
        "        all_pos_tags, pos_tags = [], []\n",
        "        with open(directory) as bijankhan_corpus:\n",
        "            for line in bijankhan_corpus:\n",
        "                if line != '\\n':\n",
        "                    word, pos_tag, ezafe_tag = line.strip().split('\\t')\n",
        "                    sent.append(word.replace('ي', 'ی').replace('ك', 'ک').replace('ة', 'ه'))\n",
        "                    pos_tags.append(pos_tag)\n",
        "                    ezafe_tags.append(ezafe_tag)\n",
        "                else:\n",
        "                    sents.append(sent)\n",
        "                    all_pos_tags.append(pos_tags)\n",
        "                    all_ezafe_tags.append(ezafe_tags)\n",
        "                     \n",
        "                    sent = []\n",
        "                    pos_tags = []\n",
        "                    ezafe_tags = []\n",
        "\n",
        "        return sents, all_pos_tags, all_ezafe_tags\n",
        "\t\n",
        "\n",
        "    def _pad(self, word):\n",
        "        for _ in range(cfg.word_max_len - len(word)):\n",
        "            word.append(0)\n",
        "        return word\n",
        "    \n",
        "\n",
        "    def _sent_to_index(self, sentence, mode='word'):\n",
        "        if mode is 'word':\n",
        "            return [self.word_to_index.get(word, 1) for word in sentence]\n",
        "        elif mode is 'char':\n",
        "            indexed_sentence = []\n",
        "            for word in sentence:\n",
        "                indexed_word = []\n",
        "                for char in word:\n",
        "                    indexed_word.append(self.char_to_index.get(word, 1))\n",
        "                indexed_sentence.append(self._pad(indexed_word))\n",
        "            return indexed_sentence\n",
        "\n",
        "\n",
        "    def _sent_to_embed(self, sentence):\n",
        "        embed_sent = []\n",
        "        for word in sentence:\n",
        "            try:\n",
        "                embed_sent.append(self.word_embedding_model[word])\n",
        "            except KeyError:\n",
        "                embed_sent.append([0 for _ in range(cfg.word_embed_dim)])\n",
        "        return embed_sent\n",
        "\n",
        "    \n",
        "    def _pos_tags_to_index(self, tags):\n",
        "        return [self.pos_tag_to_index[tag] for tag in tags]\n",
        "\n",
        "    \n",
        "    def _ezafe_tags_to_index(self, tags):\n",
        "        return [self.ezafe_tag_to_index[tag] for tag in tags]\n",
        "\n",
        "\n",
        "    def data_generator(self, mode=None, char=False, pos=None):\n",
        "        if mode is 'train':\n",
        "            sents, pos_tags, ezafe_tags = self.train_data\n",
        "        elif mode is 'eval': \n",
        "            sents, pos_tags, ezafe_tags = self.valid_data\n",
        "        else:\n",
        "            raise ArgumentError(\"Invalid argument. 'mode' must be either 'train', 'eval', or 'pred'.\")\n",
        "    \n",
        "        for sent, pos_tag, ezafe_tag in zip(sents, pos_tags, ezafe_tags):\n",
        "            sent_char = self._sent_to_index(sent, mode='char')\n",
        "            # sent_word = self._sent_to_index(sent)\n",
        "            sent_word = self._sent_to_embed(sent)\n",
        "            length = [1 for _ in range(len(sent))]\n",
        "            pos_tag = self._pos_tags_to_index(pos_tag)\n",
        "            tag = self._ezafe_tags_to_index(ezafe_tag)\n",
        "            weights = [1. if x == 0 else 1.5 for x in tag]\n",
        "            \n",
        "            if char:\n",
        "                yield (np.array(sent_word), np.array(sent_char), np.array(length)), np.array(pos_tag)\n",
        "            elif pos is 'cposi':\n",
        "                yield (np.array(sent_word), np.array(sent_char), np.array(pos_tag), np.array(length), np.array(weights)), np.array(ezafe_tag)\n",
        "            else:\n",
        "                yield np.array(sent_word), np.array(tag)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fmuk5Gc9lyrC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import sys\n",
        "import logging\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "\n",
        "from tensorflow.contrib import layers\n",
        "from pathlib import Path\n",
        "from tf_metrics import precision, recall, f1\n",
        "\n",
        "\n",
        "data_loader = DataLoader()\n",
        "\n",
        "\n",
        "def model_fn(mode, features, labels):\n",
        "    # Logging\n",
        "    Path('results').mkdir(exist_ok=True)\n",
        "    tf.logging.set_verbosity(logging.INFO)\n",
        "    handlers = [logging.FileHandler('results/main.log'),\n",
        "                logging.StreamHandler(sys.stdout)]\n",
        "    logging.getLogger('tensorflow').handlers = handlers\n",
        "    \n",
        "    word_inputs, char_inputs, length = features\n",
        "\n",
        "    training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
        "\n",
        "    batch_size = tf.shape(word_inputs)[0]\n",
        "    # input_lengths = tf.count_nonzero(word_inputs, 1, dtype=tf.int32)\n",
        "    input_lengths = tf.count_nonzero(length, 1, dtype=tf.int32)\n",
        "\n",
        "    # Char Embeddings\n",
        "    char_embeddings = tf.get_variable('char_embeddings', [cfg.num_chars + 2, cfg.char_embed_dim])\n",
        "    embedded_chars = tf.nn.embedding_lookup(char_embeddings, char_inputs)\n",
        "    # embedded_chars = tf.layers.dropout(embedded_chars, rate=.5, training=training)\n",
        "    \n",
        "    # Reshaping for CNN\n",
        "    output = tf.reshape(embedded_chars, [-1, tf.shape(char_inputs)[2], cfg.char_embed_dim])\n",
        "\n",
        "    # CNN\n",
        "    output = tf.layers.conv1d(output, filters=64, kernel_size=2, strides=1, padding=\"same\", activation=tf.nn.relu)\n",
        "    output = tf.layers.max_pooling1d(output, pool_size=2, strides=2)\n",
        "    output = tf.layers.conv1d(output, filters=128, kernel_size=2, strides=1, padding=\"same\", activation=tf.nn.relu)\n",
        "    output = tf.layers.max_pooling1d(output, pool_size=2, strides=2)\n",
        "\n",
        "    cnn_output = tf.layers.dropout(output, rate=.5, training=training)\n",
        "    cnn_output = tf.layers.flatten(cnn_output)\n",
        "\n",
        "    # Word Embeddings\n",
        "    # word_embeddings = tf.get_variable('word_embeddings', [cfg.num_words + 2, cfg.word_embed_dim])\n",
        "    # embedded_words = tf.nn.embedding_lookup(word_embeddings, word_inputs)\n",
        "    # word_inputs = tf.layers.dropout(word_inputs, rate=.5, training=training)\n",
        "    \n",
        "    # Reshaping CNN and concatenating for LSTM\n",
        "    cnn_output = tf.reshape(cnn_output, [-1, tf.shape(char_inputs)[1], 128 * int(cfg.word_max_len / 4)])\n",
        "    lstm_inputs = tf.concat([word_inputs, cnn_output], axis=-1) \n",
        "\n",
        "    # LSTM\n",
        "    transposed_emb = tf.transpose(lstm_inputs, perm=[1, 0, 2])\n",
        "    fw_cell = tf.contrib.rnn.LSTMBlockFusedCell(cfg.lstm_units)\n",
        "    bw_cell = tf.contrib.rnn.TimeReversedFusedRNN(tf.contrib.rnn.LSTMBlockFusedCell(cfg.lstm_units))\n",
        "    output_fw, _ = fw_cell(transposed_emb, dtype=tf.float32, sequence_length=input_lengths)\n",
        "    output_bw, _ = bw_cell(transposed_emb, dtype=tf.float32, sequence_length=input_lengths)\n",
        "    output = tf.concat([output_fw, output_bw], axis=-1)\n",
        "    output = tf.transpose(output, perm=[1, 0, 2])\n",
        "    lstm_output = tf.layers.dropout(output, rate=.5, training=training)\n",
        "\n",
        "    # Dense\n",
        "    output = tf.reshape(lstm_output, [-1, 2 * cfg.lstm_units])\n",
        "    logits = tf.layers.dense(output, cfg.num_tags)\n",
        "    pred = tf.reshape(logits, [-1, tf.shape(word_inputs)[1], cfg.num_tags])\n",
        "    pred_ids = tf.cast(tf.argmax(pred, axis=-1), tf.int32)\n",
        "    \n",
        "    # Loss\n",
        "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=pred))\n",
        "\n",
        "    # Metrics\n",
        "    weights = tf.to_float(tf.sign(length))\n",
        "    indices = list(range(cfg.num_tags))\n",
        "    metrics = [('acc', tf.metrics.accuracy(labels, pred_ids, weights))]\n",
        "    tags_ = list(range(cfg.num_tags))\n",
        "    tags_.remove(4)\n",
        "    for i in tags_:\n",
        "        metrics.extend([('precision_' + str(i), precision(labels, pred_ids, cfg.num_tags, [i], weights)),\n",
        "                        ('recall_' + str(i), recall(labels, pred_ids, cfg.num_tags, [i], weights)),\n",
        "                        ('f1_' + str(i), f1(labels, pred_ids, cfg.num_tags, [i], weights))])\n",
        "        \n",
        "    metrics.extend([('precision', precision(labels, pred_ids, cfg.num_tags, tags_, weights, average='macro')),\n",
        "                    ('recall', recall(labels, pred_ids, cfg.num_tags, tags_, weights, average='macro')),\n",
        "                    ('f1', f1(labels, pred_ids, cfg.num_tags, tags_, weights, average='macro'))])\n",
        "    \n",
        "    metrics = {x: y for x, y in metrics}\n",
        "    \n",
        "    for metric_name, op in metrics.items():\n",
        "        tf.summary.scalar(metric_name, op[1])\n",
        "    \n",
        "    if mode == tf.estimator.ModeKeys.EVAL:\n",
        "        return tf.estimator.EstimatorSpec(mode, loss=loss, \n",
        "                                          eval_metric_ops=metrics)\n",
        "\n",
        "    elif mode == tf.estimator.ModeKeys.TRAIN:\n",
        "        train_op = tf.train.AdamOptimizer().minimize(loss, \n",
        "                                                     global_step=tf.train.get_or_create_global_step())\n",
        "        return tf.estimator.EstimatorSpec(mode, \n",
        "                                          loss=loss, \n",
        "                                          train_op=train_op)\n",
        "\n",
        "def input_fn(mode=None):\n",
        "    data_generator = lambda: data_loader.data_generator(mode=mode, char=True)\n",
        "\n",
        "    dataset = tf.data.Dataset.from_generator(data_generator, \n",
        "                                             output_types=((tf.float32, tf.int32, tf.int32), tf.int32),\n",
        "                                             output_shapes=(([None, cfg.word_embed_dim], [None, None], [None]), [None]))\n",
        "\n",
        "    if mode is 'train':\n",
        "        dataset = dataset.shuffle(cfg.shuffle_buffer).repeat(cfg.num_epochs)\n",
        "        \n",
        "    dataset = dataset.padded_batch(cfg.batch_size, padded_shapes=(([None, cfg.word_embed_dim], [None, None], [None]), [None]))\n",
        "        \n",
        "    return dataset\n",
        " \n",
        "\n",
        "def train():\n",
        "    train_input_func = lambda: input_fn(mode='train')\n",
        "    eval_input_func = lambda: input_fn(mode='eval')\n",
        "    \n",
        "    est_conf = tf.estimator.RunConfig(cfg.model_dir, save_checkpoints_secs=60)\n",
        "    estimator = tf.estimator.Estimator(model_fn, cfg.model_dir, est_conf)\n",
        "    \n",
        "    Path(estimator.eval_dir()).mkdir(parents=True, exist_ok=True)\n",
        "    \n",
        "    train_spec = tf.estimator.TrainSpec(input_fn=train_input_func)\n",
        "    eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_func, throttle_secs=60)\n",
        "\n",
        "    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)\n",
        "  \n",
        "if __name__ == '__main__':\n",
        "    train()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}