talos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import nltk \n",
    "import string\n",
    "import re\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import keras\n",
    "import pickle\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, Dropout\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "import sklearn.datasets as skds\n",
    "from pathlib import Path\n",
    "import keras.optimizers\n",
    "\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "pd.set_option('display.max_colwidth', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>username</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>created_at</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Jaumeasens</td>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>2019-01-22 11:09:09</td>\n",
       "      <td>205</td>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>subirats9</td>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>2018-12-12 10:20:00</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>subirats9</td>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>2018-11-24 23:06:50</td>\n",
       "      <td>20</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>neusmunte</td>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>2018-10-23 20:13:25</td>\n",
       "      <td>161</td>\n",
       "      <td>462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>cuevillasjacs</td>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>2019-03-17 18:45:50</td>\n",
       "      <td>58</td>\n",
       "      <td>238</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id       username  party  \\\n",
       "0   0     Jaumeasens  bcomu   \n",
       "1   1      subirats9  bcomu   \n",
       "2   2      subirats9  bcomu   \n",
       "3   3      neusmunte  jxcat   \n",
       "4   4  cuevillasjacs  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "           created_at  retweet_count  favorite_count  \n",
       "0 2019-01-22 11:09:09            205             378  \n",
       "1 2018-12-12 10:20:00              2               2  \n",
       "2 2018-11-24 23:06:50             20              58  \n",
       "3 2018-10-23 20:13:25            161             462  \n",
       "4 2019-03-17 18:45:50             58             238  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_df = pd.read_excel(\"train-20190402-08_55_19.xlsx\")\n",
    "tweet_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset size: (7219, 7)\n",
      "Columns are: Index(['Id', 'username', 'party', 'text', 'created_at', 'retweet_count',\n",
      "       'favorite_count'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print('Dataset size:',tweet_df.shape)\n",
    "print('Columns are:',tweet_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'psc', 'pp', 'jxcat', 'cup', 'erc', 'bcomu', 'vallsbcn', 'bcncapital'}\n"
     ]
    }
   ],
   "source": [
    "df  = pd.DataFrame(tweet_df[['party', 'text']])\n",
    "print(set(tweet_df.party.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>psc</td>\n",
       "      <td>En un món connectat les ciutats són centres de comerç, creixement, innovació, tecnologia i finan...</td>\n",
       "      <td>En un món connectat les ciutats són centres de comerç creixement innovació tecnologia i finances</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>erc</td>\n",
       "      <td>Moltes gràcies @AntoniBassas per tenir-nos sempre presents i no callar per denunciar aquesta inj...</td>\n",
       "      <td>Moltes gràcies AntoniBassas per tenirnos sempre presents i no callar per denunciar aquesta injus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>psc</td>\n",
       "      <td>Escoltant l'amic i mestre Antón Costas a la presentació de \"El final del desconcierto\" al @CdEco...</td>\n",
       "      <td>Escoltant lamic i mestre Antón Costas a la presentació de El final del desconcierto al CdEconomi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>psc</td>\n",
       "      <td>#Recomano nou vídeo testimonial del Llegat Pasqual Maragall #LlegatPM @catalunyaeuropa En aquest...</td>\n",
       "      <td>Recomano nou vídeo testimonial del Llegat Pasqual Maragall LlegatPM catalunyaeuropa En aquesta o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pp</td>\n",
       "      <td>Entrega anual de premios Rosa de Plata de @BarnaCentre , con el candidato @josebouvila y nuestra...</td>\n",
       "      <td>Entrega anual de premios Rosa de Plata de BarnaCentre  con el candidato josebouvila y nuestra co...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "5    psc   \n",
       "6    erc   \n",
       "7    psc   \n",
       "8    psc   \n",
       "9     pp   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "5  En un món connectat les ciutats són centres de comerç, creixement, innovació, tecnologia i finan...   \n",
       "6  Moltes gràcies @AntoniBassas per tenir-nos sempre presents i no callar per denunciar aquesta inj...   \n",
       "7  Escoltant l'amic i mestre Antón Costas a la presentació de \"El final del desconcierto\" al @CdEco...   \n",
       "8  #Recomano nou vídeo testimonial del Llegat Pasqual Maragall #LlegatPM @catalunyaeuropa En aquest...   \n",
       "9  Entrega anual de premios Rosa de Plata de @BarnaCentre , con el candidato @josebouvila y nuestra...   \n",
       "\n",
       "                                                                                            text_punct  \n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...  \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...  \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...  \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...  \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...  \n",
       "5    En un món connectat les ciutats són centres de comerç creixement innovació tecnologia i finances   \n",
       "6  Moltes gràcies AntoniBassas per tenirnos sempre presents i no callar per denunciar aquesta injus...  \n",
       "7  Escoltant lamic i mestre Antón Costas a la presentació de El final del desconcierto al CdEconomi...  \n",
       "8  Recomano nou vídeo testimonial del Llegat Pasqual Maragall LlegatPM catalunyaeuropa En aquesta o...  \n",
       "9  Entrega anual de premios Rosa de Plata de BarnaCentre  con el candidato josebouvila y nuestra co...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_punct(text):\n",
    "    text  = \"\".join([char for char in text if char not in string.punctuation])\n",
    "    text = re.sub('[0-9]+', '', text)\n",
    "    return text\n",
    "\n",
    "df['text_punct'] = df['text'].apply(lambda x: remove_punct(x))\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "\n",
       "                                                                                        text_tokenized  \n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...  \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...  \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...  \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...  \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenization(text):\n",
    "    text = re.split('\\W+', text)\n",
    "    return text\n",
    "\n",
    "df['text_tokenized'] = df['text_punct'].apply(lambda x: tokenization(x.lower()))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'del', 'se', 'las', 'por', 'un', 'para', 'con', 'no', 'una', 'su', 'al', 'lo', 'como', 'más', 'pero', 'sus', 'le', 'ya', 'o', 'este', 'sí', 'porque', 'esta', 'entre', 'cuando', 'muy', 'sin', 'sobre', 'también', 'me', 'hasta', 'hay', 'donde', 'quien', 'desde', 'todo', 'nos', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'ese', 'eso', 'ante', 'ellos', 'e', 'esto', 'mí', 'antes', 'algunos', 'qué', 'unos', 'yo', 'otro', 'otras', 'otra', 'él', 'tanto', 'esa', 'estos', 'mucho', 'quienes', 'nada', 'muchos', 'cual', 'poco', 'ella', 'estar', 'estas', 'algunas', 'algo', 'nosotros', 'mi', 'mis', 'tú', 'te', 'ti', 'tu', 'tus', 'ellas', 'nosotras', 'vosostros', 'vosostras', 'os', 'mío', 'mía', 'míos', 'mías', 'tuyo', 'tuya', 'tuyos', 'tuyas', 'suyo', 'suya', 'suyos', 'suyas', 'nuestro', 'nuestra', 'nuestros', 'nuestras', 'vuestro', 'vuestra', 'vuestros', 'vuestras', 'esos', 'esas', 'estoy', 'estás', 'está', 'estamos', 'estáis', 'están', 'esté', 'estés', 'estemos', 'estéis', 'estén', 'estaré', 'estarás', 'estará', 'estaremos', 'estaréis', 'estarán', 'estaría', 'estarías', 'estaríamos', 'estaríais', 'estarían', 'estaba', 'estabas', 'estábamos', 'estabais', 'estaban', 'estuve', 'estuviste', 'estuvo', 'estuvimos', 'estuvisteis', 'estuvieron', 'estuviera', 'estuvieras', 'estuviéramos', 'estuvierais', 'estuvieran', 'estuviese', 'estuvieses', 'estuviésemos', 'estuvieseis', 'estuviesen', 'estando', 'estado', 'estada', 'estados', 'estadas', 'estad', 'he', 'has', 'ha', 'hemos', 'habéis', 'han', 'haya', 'hayas', 'hayamos', 'hayáis', 'hayan', 'habré', 'habrás', 'habrá', 'habremos', 'habréis', 'habrán', 'habría', 'habrías', 'habríamos', 'habríais', 'habrían', 'había', 'habías', 'habíamos', 'habíais', 'habían', 'hube', 'hubiste', 'hubo', 'hubimos', 'hubisteis', 'hubieron', 'hubiera', 'hubieras', 'hubiéramos', 'hubierais', 'hubieran', 'hubiese', 'hubieses', 'hubiésemos', 'hubieseis', 'hubiesen', 'habiendo', 'habido', 'habida', 'habidos', 'habidas', 'soy', 'eres', 'es', 'somos', 'sois', 'son', 'sea', 'seas', 'seamos', 'seáis', 'sean', 'seré', 'serás', 'será', 'seremos', 'seréis', 'serán', 'sería', 'serías', 'seríamos', 'seríais', 'serían', 'era', 'eras', 'éramos', 'erais', 'eran', 'fui', 'fuiste', 'fue', 'fuimos', 'fuisteis', 'fueron', 'fuera', 'fueras', 'fuéramos', 'fuerais', 'fueran', 'fuese', 'fueses', 'fuésemos', 'fueseis', 'fuesen', 'sintiendo', 'sentido', 'sentida', 'sentidos', 'sentidas', 'siente', 'sentid', 'tengo', 'tienes', 'tiene', 'tenemos', 'tenéis', 'tienen', 'tenga', 'tengas', 'tengamos', 'tengáis', 'tengan', 'tendré', 'tendrás', 'tendrá', 'tendremos', 'tendréis', 'tendrán', 'tendría', 'tendrías', 'tendríamos', 'tendríais', 'tendrían', 'tenía', 'tenías', 'teníamos', 'teníais', 'tenían', 'tuve', 'tuviste', 'tuvo', 'tuvimos', 'tuvisteis', 'tuvieron', 'tuviera', 'tuvieras', 'tuviéramos', 'tuvierais', 'tuvieran', 'tuviese', 'tuvieses', 'tuviésemos', 'tuvieseis', 'tuviesen', 'teniendo', 'tenido', 'tenida', 'tenidos', 'tenidas', 'tened', 'de', 'es', 'i', 'a', 'o', 'un', 'una', 'unes', 'uns', 'un', 'tot', 'també', 'altre', 'algun', 'alguna', 'alguns', 'algunes', 'ser', 'és', 'soc', 'ets', 'som', 'estic', 'està', 'estem', 'esteu', 'estan', 'com', 'en', 'per', 'perquè', 'per que', 'estat', 'estava', 'ans', 'abans', 'éssent', 'ambdós', 'però', 'per', 'poder', 'potser', 'puc', 'podem', 'podeu', 'poden', 'vaig', 'va', 'van', 'fer \\tfaig', 'fa', 'fem', 'feu', 'fan', 'cada', 'fi', 'inclòs', 'primer', 'des de', 'conseguir', 'consegueixo', 'consigueix', 'consigueixes', 'conseguim', 'consigueixen', 'anar', 'haver', 'tenir', 'tinc', 'te', 'tenim', 'teniu', 'tene', 'el', 'la', 'les', 'els', 'seu', 'aquí', 'meu', 'teu', 'ells', 'elles', 'ens', 'nosaltres', 'vosaltres', 'si', 'dins', 'sols', 'solament', 'saber', 'saps', 'sap', 'sabem', 'sabeu', 'saben', 'últim', 'llarg', 'bastant \\tfas', 'molts', 'aquells', 'aquelles', 'seus', 'llavors', 'sota', 'dalt', 'ús', 'molt', 'era', 'eres', 'erem', 'eren', 'mode', 'bé', 'quant', 'quan', 'on', 'mentre', 'qui', 'amb', 'entre', 'sense', 'jo', 'aquell', 'als', 'un', 'dels', 's', 'l', 'd', 't']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "      <th>text_nonstop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "      <td>[suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "      <td>[cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>psc</td>\n",
       "      <td>En un món connectat les ciutats són centres de comerç, creixement, innovació, tecnologia i finan...</td>\n",
       "      <td>En un món connectat les ciutats són centres de comerç creixement innovació tecnologia i finances</td>\n",
       "      <td>[en, un, món, connectat, les, ciutats, són, centres, de, comerç, creixement, innovació, tecnolog...</td>\n",
       "      <td>[món, connectat, ciutats, són, centres, comerç, creixement, innovació, tecnologia, finances, ]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>erc</td>\n",
       "      <td>Moltes gràcies @AntoniBassas per tenir-nos sempre presents i no callar per denunciar aquesta inj...</td>\n",
       "      <td>Moltes gràcies AntoniBassas per tenirnos sempre presents i no callar per denunciar aquesta injus...</td>\n",
       "      <td>[moltes, gràcies, antonibassas, per, tenirnos, sempre, presents, i, no, callar, per, denunciar, ...</td>\n",
       "      <td>[moltes, gràcies, antonibassas, tenirnos, sempre, presents, callar, denunciar, aquesta, injustíc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>psc</td>\n",
       "      <td>Escoltant l'amic i mestre Antón Costas a la presentació de \"El final del desconcierto\" al @CdEco...</td>\n",
       "      <td>Escoltant lamic i mestre Antón Costas a la presentació de El final del desconcierto al CdEconomi...</td>\n",
       "      <td>[escoltant, lamic, i, mestre, antón, costas, a, la, presentació, de, el, final, del, desconciert...</td>\n",
       "      <td>[escoltant, lamic, mestre, antón, costas, presentació, final, desconcierto, cdeconomia, brillant...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>psc</td>\n",
       "      <td>#Recomano nou vídeo testimonial del Llegat Pasqual Maragall #LlegatPM @catalunyaeuropa En aquest...</td>\n",
       "      <td>Recomano nou vídeo testimonial del Llegat Pasqual Maragall LlegatPM catalunyaeuropa En aquesta o...</td>\n",
       "      <td>[recomano, nou, vídeo, testimonial, del, llegat, pasqual, maragall, llegatpm, catalunyaeuropa, e...</td>\n",
       "      <td>[recomano, nou, vídeo, testimonial, llegat, pasqual, maragall, llegatpm, catalunyaeuropa, aquest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pp</td>\n",
       "      <td>Entrega anual de premios Rosa de Plata de @BarnaCentre , con el candidato @josebouvila y nuestra...</td>\n",
       "      <td>Entrega anual de premios Rosa de Plata de BarnaCentre  con el candidato josebouvila y nuestra co...</td>\n",
       "      <td>[entrega, anual, de, premios, rosa, de, plata, de, barnacentre, con, el, candidato, josebouvila,...</td>\n",
       "      <td>[entrega, anual, premios, rosa, plata, barnacentre, candidato, josebouvila, consellera, ciutatve...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "5    psc   \n",
       "6    erc   \n",
       "7    psc   \n",
       "8    psc   \n",
       "9     pp   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "5  En un món connectat les ciutats són centres de comerç, creixement, innovació, tecnologia i finan...   \n",
       "6  Moltes gràcies @AntoniBassas per tenir-nos sempre presents i no callar per denunciar aquesta inj...   \n",
       "7  Escoltant l'amic i mestre Antón Costas a la presentació de \"El final del desconcierto\" al @CdEco...   \n",
       "8  #Recomano nou vídeo testimonial del Llegat Pasqual Maragall #LlegatPM @catalunyaeuropa En aquest...   \n",
       "9  Entrega anual de premios Rosa de Plata de @BarnaCentre , con el candidato @josebouvila y nuestra...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "5    En un món connectat les ciutats són centres de comerç creixement innovació tecnologia i finances    \n",
       "6  Moltes gràcies AntoniBassas per tenirnos sempre presents i no callar per denunciar aquesta injus...   \n",
       "7  Escoltant lamic i mestre Antón Costas a la presentació de El final del desconcierto al CdEconomi...   \n",
       "8  Recomano nou vídeo testimonial del Llegat Pasqual Maragall LlegatPM catalunyaeuropa En aquesta o...   \n",
       "9  Entrega anual de premios Rosa de Plata de BarnaCentre  con el candidato josebouvila y nuestra co...   \n",
       "\n",
       "                                                                                        text_tokenized  \\\n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...   \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...   \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...   \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...   \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...   \n",
       "5  [en, un, món, connectat, les, ciutats, són, centres, de, comerç, creixement, innovació, tecnolog...   \n",
       "6  [moltes, gràcies, antonibassas, per, tenirnos, sempre, presents, i, no, callar, per, denunciar, ...   \n",
       "7  [escoltant, lamic, i, mestre, antón, costas, a, la, presentació, de, el, final, del, desconciert...   \n",
       "8  [recomano, nou, vídeo, testimonial, del, llegat, pasqual, maragall, llegatpm, catalunyaeuropa, e...   \n",
       "9  [entrega, anual, de, premios, rosa, de, plata, de, barnacentre, con, el, candidato, josebouvila,...   \n",
       "\n",
       "                                                                                          text_nonstop  \n",
       "0  [suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...  \n",
       "1  [artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...  \n",
       "2  [sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...  \n",
       "3  [cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...  \n",
       "4  [sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...  \n",
       "5       [món, connectat, ciutats, són, centres, comerç, creixement, innovació, tecnologia, finances, ]  \n",
       "6  [moltes, gràcies, antonibassas, tenirnos, sempre, presents, callar, denunciar, aquesta, injustíc...  \n",
       "7  [escoltant, lamic, mestre, antón, costas, presentació, final, desconcierto, cdeconomia, brillant...  \n",
       "8  [recomano, nou, vídeo, testimonial, llegat, pasqual, maragall, llegatpm, catalunyaeuropa, aquest...  \n",
       "9  [entrega, anual, premios, rosa, plata, barnacentre, candidato, josebouvila, consellera, ciutatve...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopword = nltk.corpus.stopwords.words('spanish')\n",
    "stopword.extend(nltk.corpus.stopwords.words('catalan'))\n",
    "stopword.extend([\"als\",\"un\",\"dels\",\"s\",\"l\",\"d\",\"t\"])\n",
    "print(stopword)\n",
    "def remove_stopwords(text):\n",
    "    text = [word for word in text if word not in stopword]\n",
    "    return text\n",
    "    \n",
    "df['text_nonstop'] = df['text_tokenized'].apply(lambda x: remove_stopwords(x))\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "      <th>text_nonstop</th>\n",
       "      <th>text_stemmed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "      <td>[suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "      <td>[cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "\n",
       "                                                                                        text_tokenized  \\\n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...   \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...   \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...   \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...   \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...   \n",
       "\n",
       "                                                                                          text_nonstop  \\\n",
       "0  [suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...   \n",
       "1  [artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...   \n",
       "2  [sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...   \n",
       "3  [cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...   \n",
       "\n",
       "                                                                                          text_stemmed  \n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...  \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...  \n",
       "2  [sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...  \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...  \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ps = nltk.PorterStemmer()\n",
    "\n",
    "def stemming(text):\n",
    "    text = [ps.stem(word) for word in text]\n",
    "    return text\n",
    "\n",
    "df['text_stemmed'] = df['text_nonstop'].apply(lambda x: stemming(x))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "      <th>text_nonstop</th>\n",
       "      <th>text_stemmed</th>\n",
       "      <th>text_lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "      <td>[suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "      <td>[cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "\n",
       "                                                                                        text_tokenized  \\\n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...   \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...   \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...   \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...   \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...   \n",
       "\n",
       "                                                                                          text_nonstop  \\\n",
       "0  [suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...   \n",
       "1  [artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...   \n",
       "2  [sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...   \n",
       "3  [cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...   \n",
       "\n",
       "                                                                                          text_stemmed  \\\n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...   \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...   \n",
       "2  [sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...   \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...   \n",
       "\n",
       "                                                                                       text_lemmatized  \n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...  \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...  \n",
       "2  [sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...  \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...  \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn = nltk.WordNetLemmatizer()\n",
    "def lemmatizer(text):\n",
    "    text = [wn.lemmatize(word) for word in text]\n",
    "    return text\n",
    "\n",
    "df['text_lemmatized'] = df['text_stemmed'].apply(lambda x: lemmatizer(x))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "      <th>text_nonstop</th>\n",
       "      <th>text_stemmed</th>\n",
       "      <th>text_lemmatized</th>\n",
       "      <th>final_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "      <td>[suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "      <td>suprem deixi comparèix davant parlament preso polític altra ofensa ciutadania catalunya ho expli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "      <td>artículo leer ainagallego tema important cambio tecnológico trabajo amenaza u oportunidad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...</td>\n",
       "      <td>sentiment desprotecció davant incertes genus canvi època polítiqu neoliber provoca aparició aque...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "      <td>[cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "      <td>cap traça micròfon grabador re veur claveguer ve mé em representen exiliat preso polític</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "      <td>sergimiquel acomiadat avui secretari gener jncatalunya gran discur llargament ovacionat pel comp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "\n",
       "                                                                                        text_tokenized  \\\n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...   \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...   \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...   \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...   \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...   \n",
       "\n",
       "                                                                                          text_nonstop  \\\n",
       "0  [suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...   \n",
       "1  [artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...   \n",
       "2  [sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...   \n",
       "3  [cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...   \n",
       "\n",
       "                                                                                          text_stemmed  \\\n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...   \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...   \n",
       "2  [sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...   \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...   \n",
       "\n",
       "                                                                                       text_lemmatized  \\\n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...   \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...   \n",
       "2  [sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...   \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...   \n",
       "\n",
       "                                                                                            final_text  \n",
       "0  suprem deixi comparèix davant parlament preso polític altra ofensa ciutadania catalunya ho expli...  \n",
       "1           artículo leer ainagallego tema important cambio tecnológico trabajo amenaza u oportunidad   \n",
       "2  sentiment desprotecció davant incertes genus canvi època polítiqu neoliber provoca aparició aque...  \n",
       "3            cap traça micròfon grabador re veur claveguer ve mé em representen exiliat preso polític   \n",
       "4  sergimiquel acomiadat avui secretari gener jncatalunya gran discur llargament ovacionat pel comp...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def clean_text(text):\n",
    "    text = ' '.join(word for word in text)\n",
    "    return text\n",
    "df['final_text'] = df['text_lemmatized'].apply(lambda x: clean_text(x))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ind', 'uni', 'neut'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>text_punct</th>\n",
       "      <th>text_tokenized</th>\n",
       "      <th>text_nonstop</th>\n",
       "      <th>text_stemmed</th>\n",
       "      <th>text_lemmatized</th>\n",
       "      <th>final_text</th>\n",
       "      <th>tend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...</td>\n",
       "      <td>[que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...</td>\n",
       "      <td>[suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "      <td>[suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...</td>\n",
       "      <td>suprem deixi comparèix davant parlament preso polític altra ofensa ciutadania catalunya ho expli...</td>\n",
       "      <td>neut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...</td>\n",
       "      <td>Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...</td>\n",
       "      <td>[un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "      <td>[artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...</td>\n",
       "      <td>artículo leer ainagallego tema important cambio tecnológico trabajo amenaza u oportunidad</td>\n",
       "      <td>neut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcomu</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...</td>\n",
       "      <td>[el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...</td>\n",
       "      <td>[sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...</td>\n",
       "      <td>sentiment desprotecció davant incertes genus canvi època polítiqu neoliber provoca aparició aque...</td>\n",
       "      <td>neut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...</td>\n",
       "      <td>I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...</td>\n",
       "      <td>[i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...</td>\n",
       "      <td>[cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "      <td>[cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...</td>\n",
       "      <td>cap traça micròfon grabador re veur claveguer ve mé em representen exiliat preso polític</td>\n",
       "      <td>ind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jxcat</td>\n",
       "      <td>El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...</td>\n",
       "      <td>El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...</td>\n",
       "      <td>[el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "      <td>[sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...</td>\n",
       "      <td>sergimiquel acomiadat avui secretari gener jncatalunya gran discur llargament ovacionat pel comp...</td>\n",
       "      <td>ind</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   party  \\\n",
       "0  bcomu   \n",
       "1  bcomu   \n",
       "2  bcomu   \n",
       "3  jxcat   \n",
       "4  jxcat   \n",
       "\n",
       "                                                                                                  text  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de @ainagallego sobre un tema importante “El cambio tecnológico en el trabajo...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores, ni res a veure amb les clavegueres, ve...   \n",
       "4  El @sergimiquel s’ha acomiadat avui com a Secretari General de la @JNCatalunya amb un gran discu...   \n",
       "\n",
       "                                                                                            text_punct  \\\n",
       "0  Que el Suprem no deixi comparèixer davant el Parlament als presos polítics és una altra ofensa a...   \n",
       "1  Un artículo a leer de ainagallego sobre un tema importante “El cambio tecnológico en el trabajo ...   \n",
       "2  El sentiment de desprotecció davant les incerteses que genera el canvi d’època i les polítiques ...   \n",
       "3  I cap d’ells te traça amb els micròfons i les grabadores ni res a veure amb les clavegueres ves ...   \n",
       "4  El sergimiquel s’ha acomiadat avui com a Secretari General de la JNCatalunya amb un gran discurs...   \n",
       "\n",
       "                                                                                        text_tokenized  \\\n",
       "0  [que, el, suprem, no, deixi, comparèixer, davant, el, parlament, als, presos, polítics, és, una,...   \n",
       "1  [un, artículo, a, leer, de, ainagallego, sobre, un, tema, importante, el, cambio, tecnológico, e...   \n",
       "2  [el, sentiment, de, desprotecció, davant, les, incerteses, que, genera, el, canvi, d, època, i, ...   \n",
       "3  [i, cap, d, ells, te, traça, amb, els, micròfons, i, les, grabadores, ni, res, a, veure, amb, le...   \n",
       "4  [el, sergimiquel, s, ha, acomiadat, avui, com, a, secretari, general, de, la, jncatalunya, amb, ...   \n",
       "\n",
       "                                                                                          text_nonstop  \\\n",
       "0  [suprem, deixi, comparèixer, davant, parlament, presos, polítics, altra, ofensa, ciutadania, cat...   \n",
       "1  [artículo, leer, ainagallego, tema, importante, cambio, tecnológico, trabajo, amenaza, u, oportu...   \n",
       "2  [sentiment, desprotecció, davant, incerteses, genera, canvi, època, polítiques, neoliberals, pro...   \n",
       "3  [cap, traça, micròfons, grabadores, res, veure, clavegueres, ves, més, em, representen, exiliats...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, general, jncatalunya, gran, discurs, llargament, ovaci...   \n",
       "\n",
       "                                                                                          text_stemmed  \\\n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...   \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...   \n",
       "2  [sentiment, desprotecció, davant, incertes, genera, canvi, època, polítiqu, neoliber, provoca, a...   \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...   \n",
       "\n",
       "                                                                                       text_lemmatized  \\\n",
       "0  [suprem, deixi, comparèix, davant, parlament, preso, polític, altra, ofensa, ciutadania, catalun...   \n",
       "1  [artículo, leer, ainagallego, tema, important, cambio, tecnológico, trabajo, amenaza, u, oportun...   \n",
       "2  [sentiment, desprotecció, davant, incertes, genus, canvi, època, polítiqu, neoliber, provoca, ap...   \n",
       "3  [cap, traça, micròfon, grabador, re, veur, claveguer, ve, mé, em, representen, exiliat, preso, p...   \n",
       "4  [sergimiquel, acomiadat, avui, secretari, gener, jncatalunya, gran, discur, llargament, ovaciona...   \n",
       "\n",
       "                                                                                            final_text  \\\n",
       "0  suprem deixi comparèix davant parlament preso polític altra ofensa ciutadania catalunya ho expli...   \n",
       "1           artículo leer ainagallego tema important cambio tecnológico trabajo amenaza u oportunidad    \n",
       "2  sentiment desprotecció davant incertes genus canvi època polítiqu neoliber provoca aparició aque...   \n",
       "3            cap traça micròfon grabador re veur claveguer ve mé em representen exiliat preso polític    \n",
       "4  sergimiquel acomiadat avui secretari gener jncatalunya gran discur llargament ovacionat pel comp...   \n",
       "\n",
       "   tend  \n",
       "0  neut  \n",
       "1  neut  \n",
       "2  neut  \n",
       "3   ind  \n",
       "4   ind  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def gen_tend(party):\n",
    "    if party == \"pp\" or party == \"vallsbcn\":\n",
    "        tend = \"uni\"\n",
    "    elif party == \"bcomu\"or party == \"psc\":\n",
    "        tend = \"neut\"\n",
    "    else:\n",
    "        tend = \"ind\"\n",
    "    return tend\n",
    "df['tend'] = df['party'].apply(lambda x: gen_tend(x))\n",
    "print(set(df[\"tend\"].values))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7219,) (7219,)\n",
      "(0,) (0,)\n"
     ]
    }
   ],
   "source": [
    "val_size = int(len(df) * 0)\n",
    "x_train = df[\"final_text\"][val_size:]\n",
    "y_train = df[\"party\"][val_size:]\n",
    "x_val = df[\"final_text\"][:val_size]\n",
    "y_val = df[\"party\"][:val_size]\n",
    "print(x_train.shape,y_train.shape)\n",
    "print(x_val.shape,y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26196\n",
      "Train shapes:\n",
      "(7219, 26196) (7219, 8)\n",
      "val shapes:\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(min_df=0, lowercase=True,ngram_range=(1,1))\n",
    "vectorizer.fit(x_train)\n",
    "vectorizer.vocabulary_\n",
    "x_train = vectorizer.transform(x_train).toarray()\n",
    "vocab_size = np.shape(x_train)[1]\n",
    "#x_val = vectorizer.transform(x_val).toarray()\n",
    "print(vocab_size)\n",
    "encoder = LabelBinarizer()\n",
    "encoder.fit(y_train)\n",
    "y_train = encoder.transform(y_train)\n",
    "#y_val = encoder.transform(y_val)\n",
    "print(\"Train shapes:\")\n",
    "print(x_train.shape,y_train.shape)\n",
    "print(\"val shapes:\")\n",
    "#print(x_val.shape,y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first we have to make sure to input data and params into the function\n",
    "def political_model(x_train, y_train, x_val, y_val, params):\n",
    "\n",
    "    # next we can build the model exactly like we would normally do it\n",
    "    model = Sequential()\n",
    "    model.add(Dense(params[\"first_neuron\"], input_dim=x_train.shape[1],\n",
    "                    activation=params['activation'],\n",
    "                    kernel_initializer='normal'))\n",
    "    \n",
    "    model.add(Dropout(params['dropout']))\n",
    "    \n",
    "    model.add(Dense(params[\"second_neuron\"],\n",
    "                    activation=params['activation'],\n",
    "                    kernel_initializer='normal'))\n",
    "    \n",
    "    model.add(Dropout(params['dropout']))\n",
    "\n",
    "    # then we finish again with completely standard Keras way\n",
    "    model.add(Dense(8, activation=params['last_activation'],\n",
    "                    kernel_initializer='normal'))\n",
    "    \n",
    "    model.compile(loss=params['losses'],\n",
    "                  # here we add a regulizer normalization function from Talos\n",
    "                  optimizer=params['optimizer'],\n",
    "                  metrics=['acc'])\n",
    "    \n",
    "    history = model.fit(x_train, y_train,\n",
    "                        batch_size=params['batch_size'],\n",
    "                        validation_data=[x_val, y_val],\n",
    "                        epochs=params['epochs'],\n",
    "                        verbose=0)\n",
    "    \n",
    "    # finally we have to make sure that history object and model are returned\n",
    "    return history, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = {'first_neuron':[32,64,128,256,512,1024,2048],\n",
    "     'second_neuron':[32,64,128,256,512,1024,2048],\n",
    "     'batch_size': [10],\n",
    "     'epochs': [5],\n",
    "     'dropout': [0.1],\n",
    "     'optimizer': [\"Adam\"],\n",
    "     'losses': [\"categorical_crossentropy\"],\n",
    "     'activation':[\"relu\"],\n",
    "     'last_activation': [\"softmax\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 14086303828106771485\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 9224895857\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 13744703829398177097\n",
      "physical_device_desc: \"device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1\"\n",
      "]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49/49 [27:18<00:00, 84.16s/it]\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "import talos as ta\n",
    "x = x_train\n",
    "y = y_train\n",
    "t = ta.Scan(x,y,\n",
    "            model=political_model,\n",
    "            params=p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "experiment_name            062619184800\n",
       "random_method          uniform_mersenne\n",
       "reduction_method                   None\n",
       "reduction_interval                   50\n",
       "reduction_window                     20\n",
       "reduction_threshold                 0.2\n",
       "reduction_metric                val_acc\n",
       "complete_time            06/26/19/19:15\n",
       "x_shape                   (7219, 26196)\n",
       "y_shape                       (7219, 8)\n",
       "dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accessing the results data frame\n",
    "t.data.head()\n",
    "\n",
    "# accessing epoch entropy values for each round\n",
    "\n",
    "# access the summary details\n",
    "t.details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>round_epochs</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>loss</th>\n",
       "      <th>acc</th>\n",
       "      <th>activation</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>dropout</th>\n",
       "      <th>epochs</th>\n",
       "      <th>first_neuron</th>\n",
       "      <th>last_activation</th>\n",
       "      <th>losses</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>second_neuron</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>1.846937</td>\n",
       "      <td>0.493536</td>\n",
       "      <td>0.260474</td>\n",
       "      <td>0.932120</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>2.177223</td>\n",
       "      <td>0.480148</td>\n",
       "      <td>0.143143</td>\n",
       "      <td>0.968534</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1.740310</td>\n",
       "      <td>0.544321</td>\n",
       "      <td>0.062986</td>\n",
       "      <td>0.987136</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1.781457</td>\n",
       "      <td>0.544321</td>\n",
       "      <td>0.025337</td>\n",
       "      <td>0.997427</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.772100</td>\n",
       "      <td>0.554940</td>\n",
       "      <td>0.020016</td>\n",
       "      <td>0.997823</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1.651493</td>\n",
       "      <td>0.568329</td>\n",
       "      <td>0.013234</td>\n",
       "      <td>0.998219</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5</td>\n",
       "      <td>1.807089</td>\n",
       "      <td>0.572022</td>\n",
       "      <td>0.008066</td>\n",
       "      <td>0.998813</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5</td>\n",
       "      <td>2.097432</td>\n",
       "      <td>0.489381</td>\n",
       "      <td>0.154600</td>\n",
       "      <td>0.968336</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5</td>\n",
       "      <td>2.081238</td>\n",
       "      <td>0.500923</td>\n",
       "      <td>0.053090</td>\n",
       "      <td>0.993667</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5</td>\n",
       "      <td>1.673881</td>\n",
       "      <td>0.555863</td>\n",
       "      <td>0.015847</td>\n",
       "      <td>0.998417</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5</td>\n",
       "      <td>1.535579</td>\n",
       "      <td>0.578486</td>\n",
       "      <td>0.008719</td>\n",
       "      <td>0.999010</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>5</td>\n",
       "      <td>1.552300</td>\n",
       "      <td>0.578024</td>\n",
       "      <td>0.005036</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5</td>\n",
       "      <td>1.637317</td>\n",
       "      <td>0.579409</td>\n",
       "      <td>0.005286</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>5</td>\n",
       "      <td>1.639889</td>\n",
       "      <td>0.566020</td>\n",
       "      <td>0.004903</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>5</td>\n",
       "      <td>1.594222</td>\n",
       "      <td>0.573869</td>\n",
       "      <td>0.036321</td>\n",
       "      <td>0.995250</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>5</td>\n",
       "      <td>1.483344</td>\n",
       "      <td>0.587258</td>\n",
       "      <td>0.013155</td>\n",
       "      <td>0.999208</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>5</td>\n",
       "      <td>1.479312</td>\n",
       "      <td>0.590951</td>\n",
       "      <td>0.005624</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>5</td>\n",
       "      <td>1.438159</td>\n",
       "      <td>0.594645</td>\n",
       "      <td>0.003546</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>5</td>\n",
       "      <td>1.471917</td>\n",
       "      <td>0.589566</td>\n",
       "      <td>0.001849</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5</td>\n",
       "      <td>1.497997</td>\n",
       "      <td>0.588643</td>\n",
       "      <td>0.002898</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5</td>\n",
       "      <td>1.571686</td>\n",
       "      <td>0.583102</td>\n",
       "      <td>0.002073</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>128</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>5</td>\n",
       "      <td>1.571944</td>\n",
       "      <td>0.581717</td>\n",
       "      <td>0.015289</td>\n",
       "      <td>0.998417</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>5</td>\n",
       "      <td>1.495492</td>\n",
       "      <td>0.581717</td>\n",
       "      <td>0.006854</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>5</td>\n",
       "      <td>1.482670</td>\n",
       "      <td>0.590489</td>\n",
       "      <td>0.004298</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>5</td>\n",
       "      <td>1.445024</td>\n",
       "      <td>0.592798</td>\n",
       "      <td>0.002333</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5</td>\n",
       "      <td>1.401849</td>\n",
       "      <td>0.597415</td>\n",
       "      <td>0.002668</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>5</td>\n",
       "      <td>1.395952</td>\n",
       "      <td>0.590489</td>\n",
       "      <td>0.001508</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5</td>\n",
       "      <td>1.478404</td>\n",
       "      <td>0.581717</td>\n",
       "      <td>0.002689</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>5</td>\n",
       "      <td>1.570874</td>\n",
       "      <td>0.577562</td>\n",
       "      <td>0.009235</td>\n",
       "      <td>0.999208</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>5</td>\n",
       "      <td>1.463161</td>\n",
       "      <td>0.594183</td>\n",
       "      <td>0.004990</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>5</td>\n",
       "      <td>1.415323</td>\n",
       "      <td>0.593721</td>\n",
       "      <td>0.002190</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>5</td>\n",
       "      <td>1.350500</td>\n",
       "      <td>0.596491</td>\n",
       "      <td>0.002550</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>5</td>\n",
       "      <td>1.376983</td>\n",
       "      <td>0.595106</td>\n",
       "      <td>0.001889</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5</td>\n",
       "      <td>1.448888</td>\n",
       "      <td>0.586334</td>\n",
       "      <td>0.001263</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>5</td>\n",
       "      <td>1.435491</td>\n",
       "      <td>0.586796</td>\n",
       "      <td>0.002017</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>512</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>5</td>\n",
       "      <td>1.479869</td>\n",
       "      <td>0.599261</td>\n",
       "      <td>0.004390</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>5</td>\n",
       "      <td>1.407582</td>\n",
       "      <td>0.600646</td>\n",
       "      <td>0.002842</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>5</td>\n",
       "      <td>1.390071</td>\n",
       "      <td>0.602955</td>\n",
       "      <td>0.001642</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>5</td>\n",
       "      <td>1.389307</td>\n",
       "      <td>0.599261</td>\n",
       "      <td>0.002210</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>5</td>\n",
       "      <td>1.367731</td>\n",
       "      <td>0.599261</td>\n",
       "      <td>0.001724</td>\n",
       "      <td>0.999604</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>5</td>\n",
       "      <td>1.406926</td>\n",
       "      <td>0.596491</td>\n",
       "      <td>0.000834</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>5</td>\n",
       "      <td>1.394756</td>\n",
       "      <td>0.597415</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>1024</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>5</td>\n",
       "      <td>1.858314</td>\n",
       "      <td>0.572022</td>\n",
       "      <td>0.006975</td>\n",
       "      <td>0.999010</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>2048</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>5</td>\n",
       "      <td>1.387317</td>\n",
       "      <td>0.599723</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.999208</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>2048</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>5</td>\n",
       "      <td>1.439372</td>\n",
       "      <td>0.587258</td>\n",
       "      <td>0.003226</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>2048</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>5</td>\n",
       "      <td>1.375000</td>\n",
       "      <td>0.594645</td>\n",
       "      <td>0.001012</td>\n",
       "      <td>0.999802</td>\n",
       "      <td>relu</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5</td>\n",
       "      <td>2048</td>\n",
       "      <td>softmax</td>\n",
       "      <td>categorical_crossentropy</td>\n",
       "      <td>Adam</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    round_epochs  val_loss   val_acc      loss       acc activation  \\\n",
       "0              5  1.846937  0.493536  0.260474  0.932120       relu   \n",
       "1              5  2.177223  0.480148  0.143143  0.968534       relu   \n",
       "2              5  1.740310  0.544321  0.062986  0.987136       relu   \n",
       "3              5  1.781457  0.544321  0.025337  0.997427       relu   \n",
       "4              5  1.772100  0.554940  0.020016  0.997823       relu   \n",
       "5              5  1.651493  0.568329  0.013234  0.998219       relu   \n",
       "6              5  1.807089  0.572022  0.008066  0.998813       relu   \n",
       "7              5  2.097432  0.489381  0.154600  0.968336       relu   \n",
       "8              5  2.081238  0.500923  0.053090  0.993667       relu   \n",
       "9              5  1.673881  0.555863  0.015847  0.998417       relu   \n",
       "10             5  1.535579  0.578486  0.008719  0.999010       relu   \n",
       "11             5  1.552300  0.578024  0.005036  0.999802       relu   \n",
       "12             5  1.637317  0.579409  0.005286  0.999604       relu   \n",
       "13             5  1.639889  0.566020  0.004903  0.999604       relu   \n",
       "14             5  1.594222  0.573869  0.036321  0.995250       relu   \n",
       "15             5  1.483344  0.587258  0.013155  0.999208       relu   \n",
       "16             5  1.479312  0.590951  0.005624  0.999604       relu   \n",
       "17             5  1.438159  0.594645  0.003546  0.999802       relu   \n",
       "18             5  1.471917  0.589566  0.001849  1.000000       relu   \n",
       "19             5  1.497997  0.588643  0.002898  0.999802       relu   \n",
       "20             5  1.571686  0.583102  0.002073  0.999604       relu   \n",
       "21             5  1.571944  0.581717  0.015289  0.998417       relu   \n",
       "22             5  1.495492  0.581717  0.006854  0.999406       relu   \n",
       "23             5  1.482670  0.590489  0.004298  0.999802       relu   \n",
       "24             5  1.445024  0.592798  0.002333  0.999802       relu   \n",
       "25             5  1.401849  0.597415  0.002668  0.999406       relu   \n",
       "26             5  1.395952  0.590489  0.001508  0.999802       relu   \n",
       "27             5  1.478404  0.581717  0.002689  0.999604       relu   \n",
       "28             5  1.570874  0.577562  0.009235  0.999208       relu   \n",
       "29             5  1.463161  0.594183  0.004990  0.999406       relu   \n",
       "30             5  1.415323  0.593721  0.002190  0.999604       relu   \n",
       "31             5  1.350500  0.596491  0.002550  0.999604       relu   \n",
       "32             5  1.376983  0.595106  0.001889  0.999604       relu   \n",
       "33             5  1.448888  0.586334  0.001263  0.999802       relu   \n",
       "34             5  1.435491  0.586796  0.002017  0.999406       relu   \n",
       "35             5  1.479869  0.599261  0.004390  0.999604       relu   \n",
       "36             5  1.407582  0.600646  0.002842  0.999604       relu   \n",
       "37             5  1.390071  0.602955  0.001642  0.999604       relu   \n",
       "38             5  1.389307  0.599261  0.002210  0.999604       relu   \n",
       "39             5  1.367731  0.599261  0.001724  0.999604       relu   \n",
       "40             5  1.406926  0.596491  0.000834  0.999802       relu   \n",
       "41             5  1.394756  0.597415  0.002150  0.999802       relu   \n",
       "42             5  1.858314  0.572022  0.006975  0.999010       relu   \n",
       "43             5  1.387317  0.599723  0.003257  0.999208       relu   \n",
       "44             5  1.439372  0.587258  0.003226  0.999406       relu   \n",
       "45             5  1.375000  0.594645  0.001012  0.999802       relu   \n",
       "\n",
       "    batch_size  dropout  epochs  first_neuron last_activation  \\\n",
       "0           10      0.1       5            32         softmax   \n",
       "1           10      0.1       5            32         softmax   \n",
       "2           10      0.1       5            32         softmax   \n",
       "3           10      0.1       5            32         softmax   \n",
       "4           10      0.1       5            32         softmax   \n",
       "5           10      0.1       5            32         softmax   \n",
       "6           10      0.1       5            32         softmax   \n",
       "7           10      0.1       5            64         softmax   \n",
       "8           10      0.1       5            64         softmax   \n",
       "9           10      0.1       5            64         softmax   \n",
       "10          10      0.1       5            64         softmax   \n",
       "11          10      0.1       5            64         softmax   \n",
       "12          10      0.1       5            64         softmax   \n",
       "13          10      0.1       5            64         softmax   \n",
       "14          10      0.1       5           128         softmax   \n",
       "15          10      0.1       5           128         softmax   \n",
       "16          10      0.1       5           128         softmax   \n",
       "17          10      0.1       5           128         softmax   \n",
       "18          10      0.1       5           128         softmax   \n",
       "19          10      0.1       5           128         softmax   \n",
       "20          10      0.1       5           128         softmax   \n",
       "21          10      0.1       5           256         softmax   \n",
       "22          10      0.1       5           256         softmax   \n",
       "23          10      0.1       5           256         softmax   \n",
       "24          10      0.1       5           256         softmax   \n",
       "25          10      0.1       5           256         softmax   \n",
       "26          10      0.1       5           256         softmax   \n",
       "27          10      0.1       5           256         softmax   \n",
       "28          10      0.1       5           512         softmax   \n",
       "29          10      0.1       5           512         softmax   \n",
       "30          10      0.1       5           512         softmax   \n",
       "31          10      0.1       5           512         softmax   \n",
       "32          10      0.1       5           512         softmax   \n",
       "33          10      0.1       5           512         softmax   \n",
       "34          10      0.1       5           512         softmax   \n",
       "35          10      0.1       5          1024         softmax   \n",
       "36          10      0.1       5          1024         softmax   \n",
       "37          10      0.1       5          1024         softmax   \n",
       "38          10      0.1       5          1024         softmax   \n",
       "39          10      0.1       5          1024         softmax   \n",
       "40          10      0.1       5          1024         softmax   \n",
       "41          10      0.1       5          1024         softmax   \n",
       "42          10      0.1       5          2048         softmax   \n",
       "43          10      0.1       5          2048         softmax   \n",
       "44          10      0.1       5          2048         softmax   \n",
       "45          10      0.1       5          2048         softmax   \n",
       "\n",
       "                      losses optimizer  second_neuron  \n",
       "0   categorical_crossentropy      Adam             32  \n",
       "1   categorical_crossentropy      Adam             64  \n",
       "2   categorical_crossentropy      Adam            128  \n",
       "3   categorical_crossentropy      Adam            256  \n",
       "4   categorical_crossentropy      Adam            512  \n",
       "5   categorical_crossentropy      Adam           1024  \n",
       "6   categorical_crossentropy      Adam           2048  \n",
       "7   categorical_crossentropy      Adam             32  \n",
       "8   categorical_crossentropy      Adam             64  \n",
       "9   categorical_crossentropy      Adam            128  \n",
       "10  categorical_crossentropy      Adam            256  \n",
       "11  categorical_crossentropy      Adam            512  \n",
       "12  categorical_crossentropy      Adam           1024  \n",
       "13  categorical_crossentropy      Adam           2048  \n",
       "14  categorical_crossentropy      Adam             32  \n",
       "15  categorical_crossentropy      Adam             64  \n",
       "16  categorical_crossentropy      Adam            128  \n",
       "17  categorical_crossentropy      Adam            256  \n",
       "18  categorical_crossentropy      Adam            512  \n",
       "19  categorical_crossentropy      Adam           1024  \n",
       "20  categorical_crossentropy      Adam           2048  \n",
       "21  categorical_crossentropy      Adam             32  \n",
       "22  categorical_crossentropy      Adam             64  \n",
       "23  categorical_crossentropy      Adam            128  \n",
       "24  categorical_crossentropy      Adam            256  \n",
       "25  categorical_crossentropy      Adam            512  \n",
       "26  categorical_crossentropy      Adam           1024  \n",
       "27  categorical_crossentropy      Adam           2048  \n",
       "28  categorical_crossentropy      Adam             32  \n",
       "29  categorical_crossentropy      Adam             64  \n",
       "30  categorical_crossentropy      Adam            128  \n",
       "31  categorical_crossentropy      Adam            256  \n",
       "32  categorical_crossentropy      Adam            512  \n",
       "33  categorical_crossentropy      Adam           1024  \n",
       "34  categorical_crossentropy      Adam           2048  \n",
       "35  categorical_crossentropy      Adam             32  \n",
       "36  categorical_crossentropy      Adam             64  \n",
       "37  categorical_crossentropy      Adam            128  \n",
       "38  categorical_crossentropy      Adam            256  \n",
       "39  categorical_crossentropy      Adam            512  \n",
       "40  categorical_crossentropy      Adam           1024  \n",
       "41  categorical_crossentropy      Adam           2048  \n",
       "42  categorical_crossentropy      Adam             32  \n",
       "43  categorical_crossentropy      Adam             64  \n",
       "44  categorical_crossentropy      Adam            128  \n",
       "45  categorical_crossentropy      Adam            256  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use Scan object as input\n",
    "r = ta.Reporting(t)\n",
    "\n",
    "# access the dataframe with the results\n",
    "r.data.head(-3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}