parsing_bl_pandas.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "parsing_bl_pandas",
      "provenance": [],
      "authorship_tag": "ABX9TyN6NXSPk+kn+LGE1ynbUJgK",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DmitryKutsev/NIS_SentiFrame/blob/master/parsing_bl_pandas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BjOpLg87c_h4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install pymorphy2[fast]\n",
        "import pandas as pd\n",
        "import json\n",
        "from collections import Counter\n",
        "# from pandas.io.json import json_normalize\n",
        "from pymorphy2 import MorphAnalyzer\n",
        "morph = MorphAnalyzer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5mBGZ4prROq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ! wget https://raw.githubusercontent.com/nicolay-r/RuSentiFrames/master/collection.json\n",
        "with open ('collection.json') as jf:\n",
        "  bl_data = json.load(jf)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6rB3R6xGOQKj",
        "colab_type": "code",
        "outputId": "98526f9c-ae92-4d3c-b24d-b138c06eab25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# задаём названия колонок, хотя этого можно и не делать, наверное\n",
        "cols = ['title', 'variants', 'comment', 'roles.a0', 'roles.a1', 'roles.a2', 'roles.a3',\n",
        "       'frames.polarity', 'frames.effect', 'frames.state', 'variant', 'key']\n",
        "\n",
        "# создаём шаблон итогового датафрейма с нужными колонками (см. выше)\n",
        "# к нему будем приклеивать датафреймы, в которых строки будут различаться лишь\n",
        "# колонкой вариант\n",
        "bl_df = pd.DataFrame(columns=cols)\n",
        "\n",
        "# для каждого фрейма\n",
        "for fr in bl_data:\n",
        "  # достаём фрейм\n",
        "  frame = bl_data[fr]\n",
        "  # создаём второй шаблон, к которому будем приклеивать строки для каждого варианта\n",
        "  # когда наполним этот второй шаблон, он отправится в итоговый дф\n",
        "  df = pd.DataFrame(columns=cols)\n",
        "\n",
        "\n",
        "  # для каждого варианта предиката\n",
        "  for variant in frame['variants']:\n",
        "    # переносим джейсон фрейма, с которым работаем, в датафрейм из одной строки\n",
        "    # max_level=1 потому что дальше фрейм не разворачивается\n",
        "    base_df = pd.json_normalize(frame, max_level=1)\n",
        "    # в новую колонку кладём вариант\n",
        "    base_df['variant'] = [variant]\n",
        "    # ко второму дф приклеиваем дф-строку с вариантом, индекс сбрасываем\n",
        "    # не сортируем, чтоб нимношк быстрее было\n",
        "    df = df.append(base_df, ignore_index=True, sort=False)\n",
        "\n",
        "\n",
        "  # на этом этапе у нас df для фрейма наполнился одинаковыми строками\n",
        "  # где только варианты различаются\n",
        "  # ключ, соответствующий фрейму, лежит в fr\n",
        "  # до кучи положим его туда же, в каждую строку\n",
        "  df['key'] = [fr for i in range(0,len(frame['variants']))]\n",
        "  # получившийся дф для фрейма приклеиваем к итоговому\n",
        "  bl_df = bl_df.append(df, ignore_index=True, sort=False)\n",
        "  # идём в следующий фрейм\n",
        "\n",
        "# смотрим, сколько строчек вышло в дф и не потеряли ли мы что-то\n",
        "len(bl_df)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6247"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q6Z6YkqDMdWp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# дропнем лишние колонки, но две колонки комментариев хорошо бы было совместить конечно и оставить\n",
        "bl_df.drop(['comment', 'comments', 'frames.value', 'title', 'variants'], axis=1, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYv4wSF69DIE",
        "colab_type": "code",
        "outputId": "f5843f6a-8c70-4209-e537-84bdae8ff13a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "# посмотрим, сколько всего повторяющихся вариантов предикатов\n",
        "# их 196\n",
        "\n",
        "bl_df.drop_duplicates(subset=['variant'], keep=False, inplace=True, ignore_index=True)\n",
        "bl_df.variant.value_counts().head(196)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "испытать благодарность      1\n",
              "проявить сочувствие         1\n",
              "нанести серьезный удар      1\n",
              "стать дороже                1\n",
              "раскручивание               1\n",
              "                           ..\n",
              "доверие                     1\n",
              "атака войск                 1\n",
              "внести предложение          1\n",
              "выступать с предложением    1\n",
              "покаяние                    1\n",
              "Name: variant, Length: 196, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "meYG12KpkkMC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ну, терь самое главное (типа)\n",
        "\n",
        "# идем по строкам\n",
        "for row in bl_df.itertuples():\n",
        "  # достаём индес строки\n",
        "  row_id = row[0]\n",
        "  # и достаём из неё лист полярностей\n",
        "  polarities_list = row[5]\n",
        "\n",
        "  # пытаемся\n",
        "  try:\n",
        "    # для отдельной полярности\n",
        "    for polarity in polarities_list:\n",
        "\n",
        "      # обращаемся к элементам полярности (которая сама список) по индексу\n",
        "      # элемент с индексом 0 - это тот, кто относится\n",
        "      # элемент  индексом 1 - тот, к кому относятся\n",
        "      # создаём название колонки формата 'КтоОтносится_ККому'\n",
        "      col = '{}_{}'.format(polarity[0], polarity[1])\n",
        "\n",
        "      #  обращаемся к ячейке по адресу [индекс строки, название новой колонки]\n",
        "      # заполняем эту ячейку меткой (метка - это элемент полярности с индексом 2)\n",
        "      # ну, метка бывает pos или neg\n",
        "      # степень уверенности не берём с собой\n",
        "      bl_df.loc[row_id, col] = polarity[2]\n",
        "\n",
        "\n",
        "      # чтоб сохранить меру уверенности, можно её приклеить к метке,\n",
        "      # если заменить строку выше на строку ниже\n",
        "      # но в данный момент неясно, зачем она нужна, так что не будем\n",
        "      # bl_df.loc[row_id, col] = '{} {}'.format(polarity[2], polarity[3])\n",
        "  \n",
        "  # если попытка не удалась, значит в полярности NaN\n",
        "  except:\n",
        "    pass\n",
        "  # идём в следующую строку"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hh_mrzAGhEN0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bl_df['len'] = bl_df['variant'].apply(lambda x: len(x.split()))\n",
        "bl_df['tag'] = bl_df['variant'].apply(lambda x: morph.parse(x)[0].tag.POS)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CS036YsqOOTB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# то же самое для эффекта\n",
        "# добавляем колонки 'effect_a1', 'effect_a0', 'effect_a2'\n",
        "\n",
        "for row in bl_df.itertuples():\n",
        "  row_id = row[0]\n",
        "  effect_list = row[6]\n",
        "  try:\n",
        "    for effect in effect_list:\n",
        "      col = 'effect_{}'.format(effect[0])\n",
        "      bl_df.loc[row_id, col] = effect[1]\n",
        "  except:\n",
        "    pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ulVhow15QXi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# то же самое для  state\n",
        "# добавляем колонки 'state_a1', 'state_a0', 'state_a2'\n",
        "\n",
        "for row in bl_df.itertuples():\n",
        "  row_id = row[0]\n",
        "  # семь - это номер колонки 'frames.state'\n",
        "  state_list = row[7]\n",
        "  try:\n",
        "    for state in state_list:\n",
        "      col = 'state_{}'.format(state[0])\n",
        "      bl_df.loc[row_id, col] = state[1]\n",
        "  except:\n",
        "    pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nB2bSodz6T1w",
        "colab_type": "code",
        "outputId": "54dd0d6a-0297-4e5d-eb9c-c530b1584e8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "bl_df.state_a2.value_counts()\n",
        "\n",
        "# a0\n",
        "# pos    1785\n",
        "# neg    1062"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "neg    225\n",
              "pos     39\n",
              "Name: state_a2, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6v1S8C_i63mV",
        "colab_type": "code",
        "outputId": "50b43d05-c83c-41cf-f938-c1176b3fa5c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "bl_df.effect_a2.value_counts()\n",
        "# a0\n",
        "# +    935\n",
        "# -    556"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-    1160\n",
              "+     775\n",
              "Name: effect_a2, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxB7APJuJoYK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bl_df.drop(['frames.polarity', 'frames.effect', 'frames.state'], axis=1, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orjNZ2VZeGo-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        },
        "outputId": "74d909f9-380a-4b66-db68-71024d2e69dc"
      },
      "source": [
        "bl_df.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>roles.a3</th>\n",
              "      <th>variant</th>\n",
              "      <th>key</th>\n",
              "      <th>a0_a1</th>\n",
              "      <th>a1_a2</th>\n",
              "      <th>a2_a1</th>\n",
              "      <th>author_a1</th>\n",
              "      <th>author_a2</th>\n",
              "      <th>a1_a0</th>\n",
              "      <th>a0_a2</th>\n",
              "      <th>author_a0</th>\n",
              "      <th>a2_a0</th>\n",
              "      <th>a0_a3</th>\n",
              "      <th>a1_a3</th>\n",
              "      <th>author_a3</th>\n",
              "      <th>a2_a2</th>\n",
              "      <th>a2_a3</th>\n",
              "      <th>effect_a1</th>\n",
              "      <th>effect_a0</th>\n",
              "      <th>effect_a2</th>\n",
              "      <th>effect_a3</th>\n",
              "      <th>state_a0</th>\n",
              "      <th>state_a1</th>\n",
              "      <th>state_a2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>тот, кто надеется</td>\n",
              "      <td>то, на что надеются</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>не терять надежду</td>\n",
              "      <td>0_0</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>+</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>тот, кто надеется</td>\n",
              "      <td>то, на что надеются</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ждать</td>\n",
              "      <td>0_0</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>+</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>тот, кто надеется</td>\n",
              "      <td>то, на что надеются</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>чаять</td>\n",
              "      <td>0_0</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>+</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>тот, кто надеется</td>\n",
              "      <td>то, на что надеются</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>тешить себя надеждой</td>\n",
              "      <td>0_0</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>+</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>тот, кто надеется</td>\n",
              "      <td>то, на что надеются</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>питать надежду</td>\n",
              "      <td>0_0</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>+</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>pos</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            roles.a0             roles.a1 roles.a2  ... state_a0 state_a1 state_a2\n",
              "0  тот, кто надеется  то, на что надеются      NaN  ...      pos      NaN      NaN\n",
              "1  тот, кто надеется  то, на что надеются      NaN  ...      pos      NaN      NaN\n",
              "2  тот, кто надеется  то, на что надеются      NaN  ...      pos      NaN      NaN\n",
              "3  тот, кто надеется  то, на что надеются      NaN  ...      pos      NaN      NaN\n",
              "4  тот, кто надеется  то, на что надеются      NaN  ...      pos      NaN      NaN\n",
              "\n",
              "[5 rows x 27 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pvNPiT9GDwXC",
        "colab_type": "text"
      },
      "source": [
        "# Странные effect и state"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhEmZQa_5uhu",
        "colab_type": "code",
        "outputId": "ee3abfdb-f886-4871-af37-104065d2adfe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        }
      },
      "source": [
        "bl_df[\n",
        "      (bl_df.effect_a1 == '+')\n",
        "      &\n",
        "      (bl_df.state_a1 == 'neg')\n",
        "][['state_a1', 'effect_a1', 'variant', 'roles.a0', 'roles.a1', 'roles.a2', 'key']].head(1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>state_a1</th>\n",
              "      <th>effect_a1</th>\n",
              "      <th>variant</th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>key</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5973</th>\n",
              "      <td>neg</td>\n",
              "      <td>+</td>\n",
              "      <td>выражать соболезнования</td>\n",
              "      <td>тот, кто сочувствует</td>\n",
              "      <td>тот, кому сочувствует a0</td>\n",
              "      <td>то, в чем и из-за чего сочувствует a0</td>\n",
              "      <td>1_104</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     state_a1 effect_a1  ...                               roles.a2    key\n",
              "5973      neg         +  ...  то, в чем и из-за чего сочувствует a0  1_104\n",
              "\n",
              "[1 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MG_tPke9DpKg",
        "colab_type": "code",
        "outputId": "1de7de93-c31f-4036-cb0c-b23b64929dd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 692
        }
      },
      "source": [
        "bl_data['1_104']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'comment': '',\n",
              " 'frames': {'effect': [['a1', '+', 1.0], ['a2', '-', 1.0]],\n",
              "  'polarity': [['a0', 'a1', 'pos', 1.0],\n",
              "   ['a0', 'a2', 'neg', 1.0],\n",
              "   ['a1', 'a0', 'pos', 1.0],\n",
              "   ['a1', 'a2', 'neg', 1.0],\n",
              "   ['author', 'a0', 'pos', 1.0],\n",
              "   ['author', 'a1', 'pos', 1.0],\n",
              "   ['author', 'a2', 'neg', 1.0]],\n",
              "  'state': [['a1', 'neg', 0.7]]},\n",
              " 'roles': {'a0': 'тот, кто сочувствует',\n",
              "  'a1': 'тот, кому сочувствует a0',\n",
              "  'a2': 'то, в чем и из-за чего сочувствует a0'},\n",
              " 'title': ['сочувствовать'],\n",
              " 'variants': ['выражать соболезнования',\n",
              "  'выражать сочувствие',\n",
              "  'выразить соболезнования',\n",
              "  'выразить сочувствие',\n",
              "  'испытать сопереживание',\n",
              "  'испытать сочувствие',\n",
              "  'испытывать сопереживание',\n",
              "  'испытывать сочувствие',\n",
              "  'отнестись с сочувствием',\n",
              "  'отнестись с участием',\n",
              "  'относиться с сочувствием',\n",
              "  'относиться с участием',\n",
              "  'посочувствовать',\n",
              "  'проявить сочувствие',\n",
              "  'проявить участие',\n",
              "  'проявлять сочувствие',\n",
              "  'проявлять участие',\n",
              "  'соболезнование',\n",
              "  'соболезновать',\n",
              "  'сопереживание',\n",
              "  'сопереживать',\n",
              "  'сочувствие',\n",
              "  'сочувствовать',\n",
              "  'участие']}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MISe-bd6DKgK",
        "colab_type": "code",
        "outputId": "1bdbfac2-3fae-4526-8ba6-ec5d8270d4c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        }
      },
      "source": [
        "bl_df[\n",
        "      (bl_df.effect_a1 == '-')\n",
        "      &\n",
        "      (bl_df.state_a1 == 'pos')\n",
        "][['state_a1', 'effect_a1', 'variant', 'roles.a0', 'roles.a1', 'roles.a2', 'key']].head(1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>state_a1</th>\n",
              "      <th>effect_a1</th>\n",
              "      <th>variant</th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>key</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5128</th>\n",
              "      <td>pos</td>\n",
              "      <td>-</td>\n",
              "      <td>заставать врасплох</td>\n",
              "      <td>тот, кто застал и подловил</td>\n",
              "      <td>тот, кого застали и подловили</td>\n",
              "      <td>то, где или за чем застали и подловили</td>\n",
              "      <td>1_68</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     state_a1 effect_a1  ...                                roles.a2   key\n",
              "5128      pos         -  ...  то, где или за чем застали и подловили  1_68\n",
              "\n",
              "[1 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IxXNJ_BODbLN",
        "colab_type": "code",
        "outputId": "b18acbbf-e69a-42b1-dc2d-72c53c5a92b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 727
        }
      },
      "source": [
        "bl_data['1_68']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'comment': '',\n",
              " 'frames': {'effect': [['a1', '-', 1.0], ['a2', '-', 1.0]],\n",
              "  'polarity': [['a0', 'a1', 'neg', 1.0],\n",
              "   ['a1', 'a0', 'neg', 1.0],\n",
              "   ['a0', 'a2', 'neg', 1.0],\n",
              "   ['a1', 'a2', 'pos', 0.7],\n",
              "   ['author', 'a0', 'pos', 0.7],\n",
              "   ['author', 'a1', 'neg', 0.7],\n",
              "   ['author', 'a2', 'neg', 0.7]],\n",
              "  'state': [['a0', 'pos', 1.0], ['a1', 'pos', 1.0]]},\n",
              " 'roles': {'a0': 'тот, кто застал и подловил',\n",
              "  'a1': 'тот, кого застали и подловили',\n",
              "  'a2': 'то, где или за чем застали и подловили'},\n",
              " 'title': ['застать', 'подловить'],\n",
              " 'variants': ['заставать врасплох',\n",
              "  'заставать на месте преступления',\n",
              "  'застать',\n",
              "  'застать врасплох',\n",
              "  'застать на месте преступления',\n",
              "  'застигать',\n",
              "  'застигать врасплох',\n",
              "  'застигнуть врасплох',\n",
              "  'застичь врасплох',\n",
              "  'застукать',\n",
              "  'ловить на месте преступления',\n",
              "  'подлавливание',\n",
              "  'подлавливать',\n",
              "  'подловить',\n",
              "  'поймать на месте преступления',\n",
              "  'прихватить',\n",
              "  'прихватить за жабры',\n",
              "  'прихватывать',\n",
              "  'прихватывать за жабры',\n",
              "  'прищучивать',\n",
              "  'прищучить',\n",
              "  'схватить за жабры',\n",
              "  'схватывать за жабры',\n",
              "  'ухватить за жабры',\n",
              "  'ухватывать за жабры',\n",
              "  'хватать за жабры']}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dF4-0R1PDZoR",
        "colab_type": "text"
      },
      "source": [
        "#Достаём списком"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NiehcfKmS8l8",
        "colab_type": "code",
        "outputId": "630b3204-4131-4eeb-b1db-4b177f45aeab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "bl_df.columns"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['roles.a0', 'roles.a1', 'roles.a2', 'roles.a3', 'frames.polarity',\n",
              "       'frames.effect', 'frames.state', 'variant', 'key', 'n_polarities',\n",
              "       'to_whom', 'score_mean', 'a0_a1', 'a1_a2', 'a2_a1', 'author_a1',\n",
              "       'author_a2', 'a1_a0', 'a0_a2', 'a0_a3', 'a1_a3', 'author_a0', 'a2_a0',\n",
              "       'author_a3', 'a2_a2', 'a2_a3', 'effect_a1', 'effect_a0', 'effect_a2',\n",
              "       'effect_a3', 'state_a0', 'state_a1', 'state_a2'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-EzNV8g2Ku0o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# тут по условию достаётся список глаголов\n",
        "# раскаментим одно из двух условий ниже\n",
        "\n",
        "# condition = 'pos'\n",
        "condition = 'neg'\n",
        "\n",
        "# по условиям достаём варианты\n",
        "variants = bl_df[\n",
        "      \n",
        "      # первое условие\n",
        "      (bl_df.a2_a1 == condition)\n",
        "      \n",
        "      # логическое И\n",
        "      &\n",
        "      \n",
        "      # второе условие\n",
        "      (bl_df.a1_a2 == condition)\n",
        "\n",
        "      # можно добавить ещё миллион каких-нить условий вот в таком формате\n",
        "      # &\n",
        "      # (bl_df.a1_a2 == condition)\n",
        "\n",
        "]['variant'].values\n",
        "\n",
        "# нам интересны только те предикаты, которым пайморфи даёт pos-тег инфинитива\n",
        "# или финитного глагола (на всякий)\n",
        "# ну и чтобы предикат был одним словом, никаких пробелов\n",
        "candidates = [variant for variant in variants \n",
        "              if \n",
        "              (' ' not in variant) \n",
        "              and \n",
        "              (('INFN' or 'VERB') in morph.parse(variant)[0].tag)\n",
        "              ]\n",
        "# имя файла будет с интересующей нас меткой в начале\n",
        "fname = \"{}_candidates_list.json\".format(condition)\n",
        "\n",
        "# сбрасываем лист кандидатов туда\n",
        "with open(fname, \"w\") as write_file:\n",
        "    json.dump(candidates, write_file, \n",
        "              # это чтоб нормально кириллица записалась\n",
        "              ensure_ascii=False\n",
        "              )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1g3u_Hcjf4Hf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# делаем допущение, что так у нас получится вытащить глаголы с аргументом в дативе\n",
        "# и второе допущение - что такие аргументы лежат в а1\n",
        "# на самом деле мы поэкспериментировали с а0 и а2 и a3 тоже\n",
        "# эксперименты показали, что лучше их в a1 искать\n",
        "\n",
        "# заводим новую колонку 'to_whom'\n",
        "# в ней будем хранить тру и фолс\n",
        "# тру - мб датив есть\n",
        "# фолс - мб его нету\n",
        "\n",
        "# смотрим на описание роли a1\n",
        "# если в нём есть ', кому' ИЛИ ', чему', лямбда-функция возвращает True\n",
        "# и кладёт его в новую колонку\n",
        "# ну и кладёт фолс если нету ни одной из этих строк в описании\n",
        "# такого аргумента может и не быть, поэтому на всякий конвертируем его в строку\n",
        "# эксепшены я не умею делать! научите...\n",
        "bl_df['to_whom'] = bl_df['roles.a1'].apply(lambda x: (', кому' or ', чему') in str(x))\n",
        "\n",
        "# в список кандидатов на датив кладём только нужные\n",
        "# отсеиваем нужные по условиям\n",
        "dative_candidates = [variant for variant\n",
        "                    # во-первых, чтобы тру лежало в 'to_whom'\n",
        "                     in bl_df[bl_df['to_whom'] == True]['variant'].values \n",
        "                    # во-вторых, чтоб это был глагол\n",
        "                     if (' ' not in variant) and (('INFN' or 'VERB') in morph.parse(variant)[0].tag)]\n",
        "\n",
        "# сбрасываем получившийся список в файл\n",
        "with open(\"dative_candidates_list.json\", \"w\") as write_file:\n",
        "    json.dump(dative_candidates, write_file, ensure_ascii=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GAUkWwzfDhcL",
        "colab_type": "text"
      },
      "source": [
        "#Достаём словарём"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hiV1mneNQMGg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# both positive and both negative\n",
        "\n",
        "# достаём словарь вида {'глагол':'список полярностей'}\n",
        "# если попадаются дубликаты, записывается полярность последнего\n",
        "\n",
        "condition = 'pos'\n",
        "# condition = 'neg'\n",
        "\n",
        "conditioned_df = bl_df[\n",
        "      (bl_df.a0_a1 == condition)\n",
        "      &\n",
        "      (bl_df.a1_a0 == condition)\n",
        "      # можно добавить ещё миллион каких-нить условий вот в таком формате\n",
        "      # &\n",
        "      # (bl_df.a1_a0 == condition)\n",
        "]\n",
        "\n",
        "variants = conditioned_df['variant'].values\n",
        "pols = conditioned_df['frames.polarity'].values\n",
        "\n",
        "candidates = {k:v for (k,v) in dict(zip(variants, pols)).items() \n",
        "              if (' ' not in k)\n",
        "              and (('INFN' or 'VERB') in morph.parse(k)[0].tag)\n",
        "              }\n",
        "\n",
        "fname = \"{}_candidates_dict.json\".format(condition)\n",
        "\n",
        "with open(fname, \"w\") as write_file:\n",
        "    json.dump(candidates, write_file, ensure_ascii=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsU0E-K9QUEg",
        "colab_type": "code",
        "outputId": "866d5cbc-e200-4226-a849-f6f21a6d801b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# opposite polarities\n",
        "# arguments\n",
        "\n",
        "# достаём словарь вида {'глагол':'список полярностей'}\n",
        "# если попадаются дубликаты, записывается полярность последнего\n",
        "\n",
        "def conditioner(arg1, arg2, condition='opp', output_format='l', author=False):\n",
        "  '''\n",
        "  output_format: str, 'd' dictionary, 'l' list\n",
        "\n",
        "  If author==True arg2 is destination of both author's and arg1's attitudes.\n",
        "  '''\n",
        "  if author==True:\n",
        "    conditioned_df = bl_df[(bl_df['author_{}'.format(arg2)] != bl_df['{}_{}'.format(arg1, arg2)])\n",
        "                          & (bl_df['author_{}'.format(arg2)].notnull())\n",
        "                          & (bl_df['{}_{}'.format(arg1, arg2)].notnull())]\n",
        "  else:\n",
        "    conditioned_df = bl_df[(bl_df['{}_{}'.format(arg1, arg2)] != bl_df['{}_{}'.format(arg2, arg1)]) \n",
        "                          & (bl_df['{}_{}'.format(arg1, arg2)].notnull())\n",
        "                          & (bl_df['{}_{}'.format(arg2, arg1)].notnull())]\n",
        "\n",
        "  variants = conditioned_df['variant'].values\n",
        "\n",
        "  if output_format == 'l':\n",
        "    candidates = [variant for variant in variants \n",
        "                  if (' ' not in variant) \n",
        "                  and (('INFN' or 'VERB') in morph.parse(variant)[0].tag)]\n",
        "  elif output_format == 'd':\n",
        "    pols = conditioned_df['frames.polarity'].values\n",
        "    candidates = {k:v for (k,v) in dict(zip(variants, pols)).items() \n",
        "                  if (' ' not in k)\n",
        "                  and (('INFN' or 'VERB') in morph.parse(k)[0].tag)}\n",
        "  if author==True:\n",
        "    fname = \"{}_author_{}_{}_candidates_dict.json\".format(condition, arg1, arg2)\n",
        "  else:\n",
        "    fname = \"{}_{}_{}_candidates_{}.json\".format(condition, arg1, arg2, output_format)\n",
        "  with open(fname, \"w\") as write_file:\n",
        "      json.dump(candidates, write_file, ensure_ascii=False)\n",
        "  return 0\n",
        "\n",
        "# conditioner('a0', 'a1', 'opp', 'l')\n",
        "conditioner('a0', 'a1', 'opp', 'l', author=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YvVxsYOpjD5s",
        "colab": {}
      },
      "source": [
        "# opposite polarities\n",
        "# случаи, когда различается отношение автора к аргументу и отношение другого аргумента к нему\n",
        "\n",
        "# достаём словарь вида {'глагол':'список полярностей'}\n",
        "# если попадаются дубликаты, записывается полярность последнего\n",
        "\n",
        "condition = 'opp'\n",
        "\n",
        "arg1 = 'a1'\n",
        "arg2 = 'a0' \n",
        "\n",
        "conditioned_df = bl_df[\n",
        "      (bl_df['author_{}'.format(arg2)] != bl_df['{}_{}'.format(arg1, arg2)])\n",
        "      &\n",
        "      (bl_df['author_{}'.format(arg2)].notnull())\n",
        "      &\n",
        "      (bl_df['{}_{}'.format(arg1, arg2)].notnull())\n",
        "]\n",
        "\n",
        "variants = conditioned_df['variant'].values\n",
        "pols = conditioned_df['frames.polarity'].values\n",
        "\n",
        "candidates = {k:v for (k,v) in dict(zip(variants, pols)).items() \n",
        "              if (' ' not in k)\n",
        "              and (('INFN' or 'VERB') in morph.parse(k)[0].tag)\n",
        "              }\n",
        "\n",
        "fname = \"{}_author_{}_{}_candidates_dict.json\".format(condition, arg1, arg2)\n",
        "\n",
        "with open(fname, \"w\") as write_file:\n",
        "    json.dump(candidates, write_file, ensure_ascii=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FRAA6186Xmk-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# for (k,v) in candidates.items():\n",
        "#   print (k,v)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_d5Nle7gCSGV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# dictionary of dative candidates\n",
        "\n",
        "bl_df['to_whom'] = bl_df['roles.a1'].apply(lambda x: (', кому' or ', чему') in str(x))\n",
        "\n",
        "conditioned_df = bl_df[bl_df['to_whom'] == True]\n",
        "\n",
        "variants = conditioned_df['variant'].values\n",
        "pols = conditioned_df['frames.polarity'].values\n",
        "\n",
        "candidates = {k:v for (k,v) in dict(zip(variants, pols)).items() \n",
        "              if (' ' not in k)\n",
        "              and (('INFN' or 'VERB') in morph.parse(k)[0].tag)\n",
        "              }\n",
        "\n",
        "with open(\"dative_candidates_dict.json\", \"w\") as write_file:\n",
        "    json.dump(candidates, write_file, ensure_ascii=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_eYEOqTKD2Aw",
        "colab_type": "text"
      },
      "source": [
        "# Пикл"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNcHlPZnaS8i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # в пикл записываем подопытный датафрейм\n",
        "# bl_df.to_pickle('bl_df.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FnWRs-EdOndJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bl_df = pd.read_pickle(\"/content/bl_df.pkl\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTWZpjPOWKN8",
        "colab_type": "text"
      },
      "source": [
        "#Будем группировать и посмотрим, что выйдет"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFM_MHb_U7ke",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bl_df.to_csv('collection.csv', index=False, encoding='utf-8', )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LyOPOI-gV8aF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open ('collection.csv', 'r', encoding='utf-8') as f:\n",
        "    with open ('contrasting.csv', 'w', encoding='utf-8') as fw:\n",
        "        for i, line in enumerate(f.readlines()):\n",
        "            if i == 0:\n",
        "                fw.write(line)\n",
        "            if (('pos' or '+') in line) & (('neg' or '-') in line):\n",
        "                fw.write(line)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z186zRbIn2vl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bb29072e-8b3c-43f6-805b-4eb913cb6f29"
      },
      "source": [
        "len(bl_df)"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5840"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7A4UnyEicqt-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cont_df = pd.read_csv('contrasting.csv', encoding='utf-8', )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R89Skgsqc66y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "c5e8bf06-a707-44f9-9a9a-125a1a9f1df4"
      },
      "source": [
        "cont_df.columns"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['roles.a0', 'roles.a1', 'roles.a2', 'roles.a3', 'variant', 'key',\n",
              "       'a0_a1', 'a1_a2', 'a2_a1', 'author_a1', 'author_a2', 'a1_a0', 'a0_a2',\n",
              "       'author_a0', 'a2_a0', 'a0_a3', 'a1_a3', 'author_a3', 'a2_a2', 'a2_a3',\n",
              "       'effect_a1', 'effect_a0', 'effect_a2', 'effect_a3', 'state_a0',\n",
              "       'state_a1', 'state_a2', 'len', 'tag'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VSeofEYOWOBd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "outputId": "4ad8e31d-54db-4093-80bb-f57cb349ba90"
      },
      "source": [
        "no_author_df = cont_df[\n",
        "          cont_df['author_a0'].isnull() &\n",
        "          cont_df['author_a1'].isnull() &\n",
        "          cont_df['author_a2'].isnull() &\n",
        "          cont_df['author_a3'].isnull()\n",
        "].drop(['author_a{}'.format(x) for x in range(0, 4)], axis=1)\n",
        "print (len(no_author_df))\n",
        "\n",
        "groups = [no_author_df['roles.a3'].notna(), \n",
        "          no_author_df['roles.a2'].notna(), \n",
        "          no_author_df['roles.a1'].notna(),\n",
        "          no_author_df['roles.a0'].notna(),\n",
        "        #   no_author_df['a2_a1'], \n",
        "        #   no_author_df['a1_a2'],\n",
        "        #   no_author_df['a2_a0'], no_author_df['a0_a2'],\n",
        "\n",
        "\n",
        "        #   no_author_df['roles.a1'].notna(), \n",
        "        #   no_author_df['roles.a0'].notna(), \n",
        "\n",
        "        #   no_author_df['a0_a1'], no_author_df['a1_a0'], \n",
        "\n",
        "          ]\n",
        "no_author_df.groupby(by=groups).agg('nunique')\n",
        "\n",
        "            #  ('' not in no_author_df['variant'])\n",
        "\n",
        "            #   and (('INFN' or 'VERB') in morph.parse(k)[0].tag)"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "893\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>roles.a3</th>\n",
              "      <th>variant</th>\n",
              "      <th>key</th>\n",
              "      <th>a0_a1</th>\n",
              "      <th>a1_a2</th>\n",
              "      <th>a2_a1</th>\n",
              "      <th>a1_a0</th>\n",
              "      <th>a0_a2</th>\n",
              "      <th>a2_a0</th>\n",
              "      <th>a0_a3</th>\n",
              "      <th>a1_a3</th>\n",
              "      <th>a2_a2</th>\n",
              "      <th>a2_a3</th>\n",
              "      <th>effect_a1</th>\n",
              "      <th>effect_a0</th>\n",
              "      <th>effect_a2</th>\n",
              "      <th>effect_a3</th>\n",
              "      <th>state_a0</th>\n",
              "      <th>state_a1</th>\n",
              "      <th>state_a2</th>\n",
              "      <th>len</th>\n",
              "      <th>tag</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>roles.a3</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">False</th>\n",
              "      <th>False</th>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>206</td>\n",
              "      <td>8</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <td>31</td>\n",
              "      <td>31</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "      <td>682</td>\n",
              "      <td>31</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                     roles.a0  roles.a1  ...  len  tag\n",
              "roles.a3 roles.a2 roles.a1 roles.a0                      ...          \n",
              "False    False    True     True             7         7  ...    5    5\n",
              "         True     True     True            31        31  ...    5    8\n",
              "True     True     True     True             1         1  ...    1    2\n",
              "\n",
              "[3 rows x 25 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DJi857r7ek7P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "outputId": "5b9fc12b-d583-4844-a45a-502fce2686b0"
      },
      "source": [
        "groups = [\n",
        "          no_author_df['roles.a3'].notna(), \n",
        "          no_author_df['roles.a2'].notna(), \n",
        "          no_author_df['roles.a1'].notna(),\n",
        "          no_author_df['roles.a0'].notna(),\n",
        "        #   no_author_df['a2_a1'], \n",
        "        #   no_author_df['a1_a2'],\n",
        "        #   no_author_df['a2_a0'], no_author_df['a0_a2'],\n",
        "        #   no_author_df['roles.a1'].notna(), \n",
        "        #   no_author_df['roles.a0'].notna(), \n",
        "          no_author_df['a0_a1'], no_author_df['a1_a0'], \n",
        "          ]\n",
        "\n",
        "no_author_df[\n",
        "             no_author_df['len'] == 1 &\n",
        "             ((no_author_df['tag']=='INFN')\n",
        "             |(no_author_df['tag']=='VERB'))\n",
        "             ].groupby(by=groups).agg('nunique')"
      ],
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>roles.a3</th>\n",
              "      <th>variant</th>\n",
              "      <th>key</th>\n",
              "      <th>a0_a1</th>\n",
              "      <th>a1_a2</th>\n",
              "      <th>a2_a1</th>\n",
              "      <th>a1_a0</th>\n",
              "      <th>a0_a2</th>\n",
              "      <th>a2_a0</th>\n",
              "      <th>a0_a3</th>\n",
              "      <th>a1_a3</th>\n",
              "      <th>a2_a2</th>\n",
              "      <th>a2_a3</th>\n",
              "      <th>effect_a1</th>\n",
              "      <th>effect_a0</th>\n",
              "      <th>effect_a2</th>\n",
              "      <th>effect_a3</th>\n",
              "      <th>state_a0</th>\n",
              "      <th>state_a1</th>\n",
              "      <th>state_a2</th>\n",
              "      <th>len</th>\n",
              "      <th>tag</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>roles.a3</th>\n",
              "      <th>roles.a2</th>\n",
              "      <th>roles.a1</th>\n",
              "      <th>roles.a0</th>\n",
              "      <th>a0_a1</th>\n",
              "      <th>a1_a0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"3\" valign=\"top\">False</th>\n",
              "      <th>False</th>\n",
              "      <th>True</th>\n",
              "      <th>True</th>\n",
              "      <th>neg</th>\n",
              "      <th>neg</th>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>62</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">True</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">True</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">True</th>\n",
              "      <th>neg</th>\n",
              "      <th>neg</th>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>0</td>\n",
              "      <td>179</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>pos</th>\n",
              "      <th>pos</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>19</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 roles.a0  roles.a1  ...  len  tag\n",
              "roles.a3 roles.a2 roles.a1 roles.a0 a0_a1 a1_a0                      ...          \n",
              "False    False    True     True     neg   neg           4         4  ...    1    1\n",
              "         True     True     True     neg   neg          15        15  ...    1    1\n",
              "                                    pos   pos           3         3  ...    1    1\n",
              "\n",
              "[3 rows x 25 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KH0bmpgrOBYf",
        "colab_type": "text"
      },
      "source": [
        "# Delete afterwards"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tq6i_LN5Z2Y-",
        "colab_type": "code",
        "outputId": "9fe5e11e-55c6-4658-ea83-6b524769c5ec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "print ('arg', 'pos', 'neg', '\\n', sep='\\t\\t')\n",
        "for arg in 'a0 a1 a2 a3'.split():\n",
        "  pos = len(bl_df[bl_df['effect_{}'.format(arg)]=='+'])\n",
        "  neg = len(bl_df[bl_df['effect_{}'.format(arg)]=='-'])\n",
        "  print (arg, pos, neg, sep='\\t\\t')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "arg\t\tpos\t\tneg\t\t\n",
            "\n",
            "a0\t\t935\t\t556\n",
            "a1\t\t2013\t\t3288\n",
            "a2\t\t775\t\t1160\n",
            "a3\t\t11\t\t5\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}