Tweet_Dataset-Download.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "#1-Tweet-Dataset",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AmbiTyga/ASL-Classifier/blob/master/Tweet_Dataset-Download.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NCAiXeZ6tyLf"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/AmbiTyga/ProjectNemo/blob/main/Data-collection/Tweet_collections.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bda5pFIyPMSi"
      },
      "source": [
        "||consumer-key|consumer-secret|access-token|access-token-secret|\r\n",
        "|-|------------|---------------|------------|-------------------|\r\n",
        "|1.|'7yyyT8SL3ZhbZnO2SDAbc95mY'|'2VL25ca3ckKZzv89Efe57qJC67rzFsyovV9V4JELqWAA5H3O9f'|'983655149049528321-RrTBmkk67qpAgOeiBPNjrSIvgi77Vtm'|'0zeSsJmvuPIGQz60PH8fyxw6StMswhmJvqMBJHyC4RGa8'|\r\n",
        "|2.|'afSquKSqkrs8eqrNr8aJ2gZTo'|'0JEyc80fzjo9BT6K99wfaJGZQJWabWgIVyo1vnNNju8Cmlz8R8'|'983655149049528321-gaHXrE4TYGjRNGFbuuhBWT6uoeDqP85'|'tupkn43qGuGoVbIr4SkxLbOitClYJpHRB7guUGkWKeFHV'|\r\n",
        "|3.|'U5B8didckAvhEYbglEGjQZFKy'|'kKYy8kk5Uz6L7Sf81F3Yl2WLCOcVUmfLpoVHaE2nZ9oMFd9Geg'|'983655149049528321-Qfg97mu8ayy4vnPdTbhZ9G5iNEz1QOx'|'lkBYzpG2xJtqoBGBbAhVkfNwHSv4Uws582FfPY0AFMHdr'|\r\n",
        "|4.|'HWXlr3Kvay09d0U2IqBsWM8ml'|'r0418axVb8pTt0NTHq3SX2g5S00aSlSAILstPEMJyXg0P1ZYNS'|'983655149049528321-JhgRvArPCnUEp7aHiDG1mQic2bTRjf5'|'ZfEzo8G70IpgcvcpaQfbMXYgwmCI3tiwKzsCPmAfqWjo2'|\r\n",
        "|5.|'NxxrPJ8RIE6TA7sNw0Gj5p98D'|'cOieHviP3bzkQ9SjVKII5jWR7RWVDQYGRdtXDY9OwYWSjclEBq'|'3178807178-1VS1Mdi7qYoQbuPPRRG5Fa75WmPn2q9hoyxrQBc'|'c0bonCEZUBauXLYqI1I2nouAX250NBjRLn5CYDHkrYTkb'|\r\n",
        "|6.|'UJQWbFqI7cl4Q01SzeGUbKUp5'|'BPpQikbmAsrWmeVRNYbprWe84cYVpFXnMWJyFOh1dxkoo7dHoG'|'983655149049528321-UC9KApH4Fhph2rWLZDJqlsxsHDcb1wx'|'ADfG5LGfqu0jG6Yqn0kqrr1FmUuQlkrfVJ3F4N2Mg5mSS'|"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a59Y5tjeenHv",
        "cellView": "form"
      },
      "source": [
        "#@title Download Dataset\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/verified-2019/verified-2019.tar.gz -q\r\n",
        "!mkdir verified\r\n",
        "!tar xzf verified-2019.tar.gz -C verified\r\n",
        "!rm verified-2019.tar.gz\r\n",
        "print(\"Verified-2019 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/midterm-2018/midterm-2018.tar.gz -q\r\n",
        "!mkdir midterm\r\n",
        "!tar xzf midterm-2018.tar.gz -C midterm\r\n",
        "!rm midterm-2018.tar.gz\r\n",
        "print(\"Midterm-2018 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/astroturf/astroturf.tar.gz -q\r\n",
        "!mkdir astro\r\n",
        "!tar xzf astroturf.tar.gz -C astro\r\n",
        "!rm astroturf.tar.gz\r\n",
        "print(\"AstroTurf downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/botwiki-2019/botwiki-2019.tar.gz -q\r\n",
        "!mkdir botwiki\r\n",
        "!tar xzf botwiki-2019.tar.gz -C botwiki\r\n",
        "!rm botwiki-2019.tar.gz\r\n",
        "print(\"Botwiki-2019 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/cresci-stock-2018/cresci-stock-2018.tar.gz -q\r\n",
        "!mkdir stock\r\n",
        "!tar xzf cresci-stock-2018.tar.gz -C stock\r\n",
        "!rm cresci-stock-2018.tar.gz\r\n",
        "print(\"Cresci Stock downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/cresci-rtbust-2019/cresci-rtbust-2019.tar.gz -q\r\n",
        "!mkdir cresci-rtbust\r\n",
        "!tar xzf cresci-rtbust-2019.tar.gz -C cresci-rtbust\r\n",
        "!rm cresci-rtbust-2019.tar.gz\r\n",
        "print(\"Cresci Rtbust downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/gilani-2017/gilani-2017.tar.gz -q\r\n",
        "!mkdir gilani\r\n",
        "!tar xzf gilani-2017.tar.gz -C gilani\r\n",
        "!rm gilani-2017.tar.gz\r\n",
        "print(\"Gilani-17 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/political-bots-2019/political-bots-2019.tar.gz -q\r\n",
        "!mkdir political\r\n",
        "!tar xzf political-bots-2019.tar.gz -C political\r\n",
        "!rm political-bots-2019.tar.gz\r\n",
        "print(\"Political-bots downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/botometer-feedback-2019/botometer-feedback-2019.tar.gz -q\r\n",
        "!mkdir botometer\r\n",
        "!tar xzf botometer-feedback-2019.tar.gz -C botometer\r\n",
        "!rm botometer-feedback-2019.tar.gz\r\n",
        "print(\"Botometer Feedback downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/vendor-purchased-2019/vendor-purchased-2019.tar.gz -q\r\n",
        "!mkdir vendor\r\n",
        "!tar xzf vendor-purchased-2019.tar.gz -C vendor\r\n",
        "!rm vendor-purchased-2019.tar.gz\r\n",
        "print(\"Vendor Purchased 2019 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/celebrity-2019/celebrity-2019.tar.gz -q\r\n",
        "!mkdir celebrity\r\n",
        "!tar xzf celebrity-2019.tar.gz -C celebrity\r\n",
        "!rm celebrity-2019.tar.gz\r\n",
        "print(\"Celebrity-2019 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/pronbots-2019/pronbots-2019.tar.gz -q\r\n",
        "!mkdir pronbots\r\n",
        "!tar xvzf pronbots-2019.tar.gz -C pronbots\r\n",
        "!rm pronbots-2019.tar.gz\r\n",
        "print(\"Pronbots-2019 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/cresci-2017/cresci-2017.csv.zip -q\r\n",
        "!unzip -q cresci-2017.csv.zip\r\n",
        "!mkdir cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/crowdflower_results.csv.zip -d cresci-17 \r\n",
        "!unzip -q /content/datasets_full.csv/fake_followers.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/genuine_accounts.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/social_spambots_1.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/social_spambots_2.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/social_spambots_3.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/traditional_spambots_1.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/traditional_spambots_2.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/traditional_spambots_3.csv.zip -d cresci-17\r\n",
        "!unzip -q /content/datasets_full.csv/traditional_spambots_4.csv.zip -d cresci-17\r\n",
        "!rm cresci-2017.csv.zip\r\n",
        "!rm datasets_full.csv -r\r\n",
        "!rm /content/cresci-17/__MACOSX -r\r\n",
        "print(\"Cresci-17 downloaded and extracted\")\r\n",
        "\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/caverlee-2011/caverlee-2011.zip -q\r\n",
        "!mkdir caverlee\r\n",
        "!unzip -q /content/caverlee-2011.zip -d caverlee \r\n",
        "!rm caverlee-2011.zip\r\n",
        "print(\"Caverlee-11 downloaded and extracted\")\r\n",
        "\r\n",
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/varol-2017/varol-2017.dat.gz -q\r\n",
        "!mkdir varol\r\n",
        "!gunzip -dc < varol-2017.dat.gz > varol/varol-2017.dat\r\n",
        "!rm varol-2017.dat.gz\r\n",
        "print(\"Varol-17 downloaded and extracted\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xMxWrkWs0iUy",
        "outputId": "d0d4f27b-597c-4add-8bd9-dfcf572d9381"
      },
      "source": [
        "!wget https://botometer.osome.iu.edu/bot-repository/datasets/pronbots-2019/pronbots-2019.tar.gz -q\r\n",
        "!mkdir pronbots\r\n",
        "!tar xvzf pronbots-2019.tar.gz -C pronbots\r\n",
        "!rm pronbots-2019.tar.gz\r\n",
        "print(\"Pronbots-2019 downloaded and extracted\")"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "pronbots-2019.tsv\n",
            "pronbots-2019_tweets.json\n",
            "Pronbots-2019 downloaded and extracted\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YxQQv0UTfHBq"
      },
      "source": [
        "import pandas as pd\n",
        "import pickle as pkl\n",
        "from tqdm import tqdm\n",
        "import tweepy, time"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JqlE9tqsp8aH"
      },
      "source": [
        "# assign the values accordingly \n",
        "consumer_key = '7yyyT8SL3ZhbZnO2SDAbc95mY'\t\t\t\n",
        "consumer_secret = '2VL25ca3ckKZzv89Efe57qJC67rzFsyovV9V4JELqWAA5H3O9f'\n",
        "access_token= '983655149049528321-RrTBmkk67qpAgOeiBPNjrSIvgi77Vtm'\n",
        "access_token_secret = '0zeSsJmvuPIGQz60PH8fyxw6StMswhmJvqMBJHyC4RGa8'\n",
        "  \n",
        "# authorization of consumer key and consumer secret \n",
        "auth = tweepy.OAuthHandler(consumer_key, consumer_secret) \n",
        "  \n",
        "# set access to user's access key and access secret  \n",
        "auth.set_access_token(access_token, access_token_secret) \n",
        "  \n",
        "# calling the api  \n",
        "api = tweepy.API(auth) "
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ahqQm4la0kcT"
      },
      "source": [
        "data = pd.read_csv(\"/content/pronbots/pronbots-2019.tsv\",header = None,sep = '\\t')\r\n",
        "bot_ids = data[0].values[:2000]"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bCIyGsaheVMs",
        "outputId": "fb3f1a68-5c5a-4576-eb26-5cc6ed037b01"
      },
      "source": [
        "failed = []\r\n",
        "done = []\r\n",
        "for x in tqdm(bot_ids):\r\n",
        "  try:\r\n",
        "    done.append(api.user_timeline(x,count = 500))\r\n",
        "  except tweepy.RateLimitError:\r\n",
        "    print('\\nRate Exceeded')\r\n",
        "    time.sleep(15*60)\r\n",
        "    print(\"\\nStarted\")\r\n",
        "  except Exception as e:\r\n",
        "    failed.append(x)\r\n",
        "  time.sleep(1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 34%|███▍      | 677/2000 [13:03<25:12,  1.14s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Rate Exceeded\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VydXxBKb17Ci",
        "outputId": "334780a8-db96-43fe-e929-486b62ac5725"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imw8XwcYPZKQ"
      },
      "source": [
        "len(done), len(failed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yg2rE9RNUbSg"
      },
      "source": [
        "with open(\"bot-2000#1.pkl\", 'wb') as f:\r\n",
        "  pkl.dump(done,f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jh2EE3XfUe09"
      },
      "source": [
        "!mv \"/content/bot-2000#1.pkl\" \"/content/drive/MyDrive/Pronbots\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "_irZ7rv1qxW0",
        "outputId": "e3af4b3d-ad23-4e37-80ed-ba0097803cc9"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1111770704</td>\n",
              "      <td>bot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>422386111</td>\n",
              "      <td>bot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>94059036</td>\n",
              "      <td>bot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>419403479</td>\n",
              "      <td>bot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3141909615</td>\n",
              "      <td>bot</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            0    1\n",
              "0  1111770704  bot\n",
              "1   422386111  bot\n",
              "2    94059036  bot\n",
              "3   419403479  bot\n",
              "4  3141909615  bot"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbaNsN9-qzfo",
        "outputId": "d84068cb-951d-4039-e11a-ab4ea7a26912"
      },
      "source": [
        "data[1].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "bot    21964\n",
              "Name: 1, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "369B6JmkrJLM",
        "outputId": "8929587a-877b-4360-b4ad-4ff244ff978e"
      },
      "source": [
        "data[0].max(),data[3].max(),data[4].max(),data[5].max(),data[6].max(),data[7].max(),"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(173766965, 109388, 4309347, 148047, 15, 474)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AFizDWXBrcBz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}