groupby_parse_tokenize.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "preprocess_data_.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdOFa-qsHWbo",
        "colab_type": "code",
        "outputId": "62ebc297-d26b-4b3e-e2fe-55b75ce4a5eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "import pandas as pd\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSHEF3ymHcDv",
        "colab_type": "code",
        "outputId": "e96b6130-f4fc-4aba-cddb-8da735d9abbb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "!ls \"/content/drive/My Drive/WashingtonPost.v2/data/parquet\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ContentsType-animated_video.parquet  ContentsType-kicker.parquet\n",
            "ContentsType-ar-wikitude.parquet     ContentsType-list.parquet\n",
            "ContentsType-author_info.parquet     ContentsType-pull_quote.parquet\n",
            "ContentsType-byline.parquet\t     ContentsType-sanitized_html.parquet\n",
            "ContentsType-date.parquet\t     ContentsType-title.parquet\n",
            "ContentsType-deck.parquet\t     ContentsType-top_deck.parquet\n",
            "ContentsType-gallery.parquet\t     ContentsType-tweet.parquet\n",
            "ContentsType-image.parquet\t     ContentsType-video.parquet\n",
            "ContentsType-inline_story.parquet    documents.parquet\n",
            "ContentsType-instagram.parquet\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BxPUe_wyhzKs",
        "colab_type": "text"
      },
      "source": [
        "# Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1CBKcudWuNLa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath = \"/content/drive/My Drive/WashingtonPost.v2/data/parquet/ContentsType-sanitized_html.parquet\"\n",
        "doc_df = pd.read_parquet(filepath)\n",
        "print(doc_df.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fYuu7xATc8Pr",
        "colab_type": "text"
      },
      "source": [
        "#Group by Document ID"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "35wV8I4G0B51",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Append all text from a document together\n",
        "doc_df_grouped = doc_df.groupby('document_id')['content'].apply(lambda x: \" \".join(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9MQSudGMD4-A",
        "outputId": "1251ca4d-8dbd-4649-c9fe-a12c790eb147",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "filepath_data_grouped = \"/content/drive/My Drive/WashingtonPost.v2/data/doc_grouped.csv\"\n",
        "doc_df_grouped.to_csv(filepath_data_grouped)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: The signature of `Series.to_csv` was aligned to that of `DataFrame.to_csv`, and argument 'header' will change its default value from False to True: please pass an explicit value to suppress this warning.\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EbgYUsSe0iK2",
        "colab_type": "code",
        "outputId": "0a2cac13-b0e0-48ba-b2c6-2e0e8e119112",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "doc_df_grouped.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "document_id\n",
              "0000107b869682826003b04a40e63948        If Senate Democrats needed a miracle not to lo...\n",
              "00002df2-867b-11e1-abb4-839961045500    <span class=\"dateline\">CHEYENNE, Wyo. —</span>...\n",
              "000045ed8eabe76d1049d21dfc08abbd        <strong>ON THE DOCKET: </strong>A three-judge ...\n",
              "0000559e4f5786355e61755bd3bac097        The last few years have not been kind to the V...\n",
              "000058a975a5d3bceec214776c48d7a2        Kris Bryant, the second overall pick two years...\n",
              "Name: content, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3-iwwqbaj5B1",
        "colab_type": "text"
      },
      "source": [
        "# Load Grouped Documents"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hfqzrZ7_Hgj8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath = \"/content/drive/My Drive/WashingtonPost.v2/data/doc_grouped.csv\"\n",
        "doc_df = pd.read_csv(filepath, header=None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBqFEbapIRa2",
        "colab_type": "code",
        "outputId": "2c42ae07-1f5f-4743-8436-9212d0fe5de2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "print(doc_df.shape)\n",
        "print(len(doc_df))\n",
        "print(doc_df.head())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(592341, 2)\n",
            "592341\n",
            "                                      0                                                  1\n",
            "0      0000107b869682826003b04a40e63948  If Senate Democrats needed a miracle not to lo...\n",
            "1  00002df2-867b-11e1-abb4-839961045500  <span class=\"dateline\">CHEYENNE, Wyo. —</span>...\n",
            "2      000045ed8eabe76d1049d21dfc08abbd  <strong>ON THE DOCKET: </strong>A three-judge ...\n",
            "3      0000559e4f5786355e61755bd3bac097  The last few years have not been kind to the V...\n",
            "4      000058a975a5d3bceec214776c48d7a2  Kris Bryant, the second overall pick two years...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZhlbdOjEgcz",
        "colab_type": "code",
        "outputId": "01fc5ea2-1f59-489b-8d63-274443c96d9a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "!pip install rank_bm25"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting rank_bm25\n",
            "  Downloading https://files.pythonhosted.org/packages/d2/e4/38d03d6d5e2deae8d2838b81d6ba2742475ced42045f5c46aeb00c5fb79c/rank_bm25-0.2.tar.gz\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from rank_bm25) (1.17.4)\n",
            "Building wheels for collected packages: rank-bm25\n",
            "  Building wheel for rank-bm25 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for rank-bm25: filename=rank_bm25-0.2-cp36-none-any.whl size=4162 sha256=75017f9c968d75b84b4474588b3f6bc0deb9c696ae28642b86e3d2120d848df0\n",
            "  Stored in directory: /root/.cache/pip/wheels/6f/0c/1f/78945dd6a5478bbcdb50d73ac96ae5af2ffcdfcd374fd9b1bf\n",
            "Successfully built rank-bm25\n",
            "Installing collected packages: rank-bm25\n",
            "Successfully installed rank-bm25-0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m1n-JsA8kEOp",
        "colab_type": "text"
      },
      "source": [
        "# Preprocess Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z-H82JnqkUBb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Tokenize:\n",
        "import numpy as np\n",
        "from rank_bm25 import BM25Okapi\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlhrcwqFJueC",
        "colab_type": "code",
        "outputId": "1356a845-8c60-47bb-9c0f-219b576d01da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# Tokenize sentence. Remove punctuation marks.\n",
        "# Converts all words to lowercase\n",
        "def tokenize(sentence):\n",
        "    sentence = sentence.replace('’','')\n",
        "    words = nltk.word_tokenize(sentence)\n",
        "    # Remove single character punctuation words\n",
        "    words_cleaned = [ word.lower() for word in words if word.isalnum() or not len(word) == 1 ]\n",
        "    return words_cleaned"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2tm8uytHnn0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath_tokenized = \"/content/drive/My Drive/WashingtonPost.v2/data/tokenized_data.csv\"\n",
        "\n",
        "index = 0\n",
        "datasize = len(doc_df)\n",
        "\n",
        "while index < datasize:\n",
        "    new_index = index + 2000\n",
        "    print(\"Tokenized slice \", new_index)\n",
        "    print(100*index/datasize, \"%\")\n",
        "\n",
        "    # select slice from data\n",
        "    df_slice = doc_df.iloc[index:new_index]\n",
        "    index = new_index\n",
        "    \n",
        "    id_series = df_slice.iloc[:,0]\n",
        "    text_series = df_slice.iloc[:,1]\n",
        "\n",
        "    # Remove html tags from documents\n",
        "    s_parsed = text_series.map(lambda x: BeautifulSoup(x).get_text())\n",
        "    print(\"Parsed\")\n",
        "    # Tokenize the corpus\n",
        "    tokenized_corpus_slice = s_parsed.map(lambda x: tokenize(x))\n",
        "    print(\"Tokenized\")\n",
        "    # Append slice to csv file\n",
        "    slice_df = pd.DataFrame(dict(s1 = id_series, s2 = tokenized_corpus_slice))\n",
        "    slice_df.to_csv(filepath_tokenized, header = False, mode=\"a\")\n",
        "    print(\"Saved slice\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}