Draft 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('/Users/noahpovis/Desktop/Buildweek22/Build Week 2 /DX.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, val, test = np.split(df.sample(frac=1), [int(.6*len(df)), int(.8*len(df))])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(16278, 18)"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(5426, 18)"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(5427, 18)"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(i):\n",
    "    i = i.copy()\n",
    "    i.drop_duplicates(subset=['Organization Name','Funding Type'], inplace = True)\n",
    "    i.drop_duplicates(keep='first', inplace= True)\n",
    "    i['Raised Series A'] = np.where(i['Organization Name'].duplicated(), 1, 0)\n",
    "    i.drop_duplicates(subset='Organization Name', keep='last', inplace= True)\n",
    "    i['Money Raised Currency (in USD)'] = pd.to_numeric(i['Money Raised Currency (in USD)'], errors='coerce')\n",
    "    i['Total Funding Amount Currency (in USD)'] = pd.to_numeric(i['Total Funding Amount Currency (in USD)'], errors='coerce')\n",
    "    i['Announced Date'] = pd.to_datetime(i['Announced Date'], errors='coerce')\n",
    "    i['Total Funding Amount Currency (in USD)'] = pd.to_numeric(i['Total Funding Amount Currency (in USD)'], errors='coerce')\n",
    "    i  = i[i['Funding Type'] != 'Series A'] \n",
    "    return i \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = clean(train)\n",
    "val = clean(val)\n",
    "test = clean(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    12302\n1      179\nName: Raised Series A, dtype: int64"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "train['Raised Series A'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "((12481, 19), (4554, 19), (4568, 19))"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "train.shape , val.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "3578"
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "train['Investor Names'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_columns(text):\n",
    "    text = text.copy()\n",
    "    # new data frame with split value columns \n",
    "    new = text[\"Organization Industries\"].str.split(\",\", n = 7, expand = True) \n",
    "    # making separate first name column from new data frame \n",
    "    text[\"Primary Industry\"]= new[0] \n",
    "    # making separate last name column from new data frame \n",
    "    text[\"Sub_Ind\"]= new[1] \n",
    "    text[\"Sub_Ind2\"]= new[2]\n",
    "    text[\"Sub_Ind3\"]= new[3]\n",
    "    text[\"Sub_Ind4\"]= new[4]\n",
    "    text[\"Sub_Ind5\"]= new[5]\n",
    "    text[\"Sub_Ind6\"]= new[6]\n",
    "    text[\"Sub_Ind7\"]= new[7]\n",
    "    # Dropping old Name columns \n",
    "    text.drop(columns =[\"Organization Industries\"], inplace = True)\n",
    "\n",
    "    #same thing but for investor column \n",
    "    new1 = text[\"Investor Names\"].str.split(\",\", n = 11, expand = True) \n",
    "    # making separate first name column from new data frame \n",
    "    text[\"Investor1\"]= new1[0] \n",
    "    # making separate last name column from new data frame \n",
    "    text[\"Investor1\"]= new1[1] \n",
    "    text[\"Investor2\"]= new1[2]\n",
    "    text[\"Investor3\"]= new1[3]\n",
    "    text[\"Investor4\"]= new1[4]\n",
    "    text[\"Investor5\"]= new1[5]\n",
    "    text[\"Investor6\"]= new1[6]\n",
    "    text[\"Investor7\"]= new1[7]\n",
    "    text[\"Investor8\"]= new1[8]\n",
    "    text[\"Investor9\"]= new1[9]\n",
    "    text[\"Investor10\"]= new1[10]\n",
    "    text[\"Investor11\"]= new1[11]\n",
    "    # Dropping old Name columns \n",
    "    text.drop(columns =[\"Investor Names\"], inplace = True)\n",
    "    text.drop(columns =[\"Lead Investors\"], inplace = True)\n",
    "    text.drop(columns =[\"Number of Investors\"], inplace = True)\n",
    "    text = text[text['Money Raised'].notna()]\n",
    "    text = text[text['Organization Location'].notna()]\n",
    "    text = text[text['Primary Industry'].notna()]\n",
    "    text = text[text['Money Raised Currency (in USD)'].notna()]\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "    return text \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = split_columns(train)\n",
    "val = split_columns(val)\n",
    "test = split_columns(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "               Transaction Name  \\\n19602        Seed Round - Finzy   \n24817     Seed Round - Passbase   \n21855  Seed Round - Flex-Appeal   \n8045          Seed Round - Pype   \n13954      Seed Round - Kinside   \n\n                                    Transaction Name URL Organization Name  \\\n19602  https://www.crunchbase.com/funding_round/finzy...             Finzy   \n24817  https://www.crunchbase.com/funding_round/passb...          Passbase   \n21855  https://www.crunchbase.com/funding_round/flex-...       Flex-Appeal   \n8045   https://www.crunchbase.com/funding_round/pype-...              Pype   \n13954  https://www.crunchbase.com/funding_round/kinsi...           Kinside   \n\n                                   Organization Name URL Funding Type  \\\n19602      https://www.crunchbase.com/organization/finzy         Seed   \n24817   https://www.crunchbase.com/organization/passbase         Seed   \n21855  https://www.crunchbase.com/organization/flex-a...         Seed   \n8045        https://www.crunchbase.com/organization/pype         Seed   \n13954    https://www.crunchbase.com/organization/kinside         Seed   \n\n      Money Raised Money Raised Currency  Money Raised Currency (in USD)  \\\n19602      1300000                   USD                       1300000.0   \n24817      3600000                   USD                       3600000.0   \n21855       600000                   EUR                        734076.0   \n8045       3000000                   USD                       3000000.0   \n13954      1300000                   USD                       1300000.0   \n\n      Announced Date Total Funding Amount  ...  Investor2  \\\n19602     2018-04-03              2300000  ...        NaN   \n24817     2019-09-25              4200000  ...   Seedcamp   \n21855     2018-01-18              1585000  ...        NaN   \n8045      2019-03-04              5950000  ...       None   \n13954     2018-08-20              4300000  ...   Haystack   \n\n                   Investor3      Investor4            Investor5  \\\n19602                    NaN            NaN                  NaN   \n24817   Upheaval Investments           None                 None   \n21855                    NaN            NaN                  NaN   \n8045                    None           None                 None   \n13954       January Ventures   K50 Ventures   Precursor Ventures   \n\n           Investor6 Investor7 Investor8 Investor9 Investor10 Investor11  \n19602            NaN       NaN       NaN       NaN        NaN        NaN  \n24817           None      None      None      None       None       None  \n21855            NaN       NaN       NaN       NaN        NaN        NaN  \n8045            None      None      None      None       None       None  \n13954   Y Combinator      None      None      None       None       None  \n\n[5 rows x 34 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Transaction Name</th>\n      <th>Transaction Name URL</th>\n      <th>Organization Name</th>\n      <th>Organization Name URL</th>\n      <th>Funding Type</th>\n      <th>Money Raised</th>\n      <th>Money Raised Currency</th>\n      <th>Money Raised Currency (in USD)</th>\n      <th>Announced Date</th>\n      <th>Total Funding Amount</th>\n      <th>...</th>\n      <th>Investor2</th>\n      <th>Investor3</th>\n      <th>Investor4</th>\n      <th>Investor5</th>\n      <th>Investor6</th>\n      <th>Investor7</th>\n      <th>Investor8</th>\n      <th>Investor9</th>\n      <th>Investor10</th>\n      <th>Investor11</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>19602</th>\n      <td>Seed Round - Finzy</td>\n      <td>https://www.crunchbase.com/funding_round/finzy...</td>\n      <td>Finzy</td>\n      <td>https://www.crunchbase.com/organization/finzy</td>\n      <td>Seed</td>\n      <td>1300000</td>\n      <td>USD</td>\n      <td>1300000.0</td>\n      <td>2018-04-03</td>\n      <td>2300000</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>24817</th>\n      <td>Seed Round - Passbase</td>\n      <td>https://www.crunchbase.com/funding_round/passb...</td>\n      <td>Passbase</td>\n      <td>https://www.crunchbase.com/organization/passbase</td>\n      <td>Seed</td>\n      <td>3600000</td>\n      <td>USD</td>\n      <td>3600000.0</td>\n      <td>2019-09-25</td>\n      <td>4200000</td>\n      <td>...</td>\n      <td>Seedcamp</td>\n      <td>Upheaval Investments</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>21855</th>\n      <td>Seed Round - Flex-Appeal</td>\n      <td>https://www.crunchbase.com/funding_round/flex-...</td>\n      <td>Flex-Appeal</td>\n      <td>https://www.crunchbase.com/organization/flex-a...</td>\n      <td>Seed</td>\n      <td>600000</td>\n      <td>EUR</td>\n      <td>734076.0</td>\n      <td>2018-01-18</td>\n      <td>1585000</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>8045</th>\n      <td>Seed Round - Pype</td>\n      <td>https://www.crunchbase.com/funding_round/pype-...</td>\n      <td>Pype</td>\n      <td>https://www.crunchbase.com/organization/pype</td>\n      <td>Seed</td>\n      <td>3000000</td>\n      <td>USD</td>\n      <td>3000000.0</td>\n      <td>2019-03-04</td>\n      <td>5950000</td>\n      <td>...</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>13954</th>\n      <td>Seed Round - Kinside</td>\n      <td>https://www.crunchbase.com/funding_round/kinsi...</td>\n      <td>Kinside</td>\n      <td>https://www.crunchbase.com/organization/kinside</td>\n      <td>Seed</td>\n      <td>1300000</td>\n      <td>USD</td>\n      <td>1300000.0</td>\n      <td>2018-08-20</td>\n      <td>4300000</td>\n      <td>...</td>\n      <td>Haystack</td>\n      <td>January Ventures</td>\n      <td>K50 Ventures</td>\n      <td>Precursor Ventures</td>\n      <td>Y Combinator</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n      <td>None</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 34 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Index(['Transaction Name', 'Transaction Name URL', 'Organization Name',\n       'Organization Name URL', 'Funding Type', 'Money Raised',\n       'Money Raised Currency', 'Money Raised Currency (in USD)',\n       'Announced Date', 'Total Funding Amount',\n       'Total Funding Amount Currency',\n       'Total Funding Amount Currency (in USD)', 'Organization Description',\n       'Organization Location', 'Raised Series A', 'Primary Industry',\n       'Sub_Ind', 'Sub_Ind2', 'Sub_Ind3', 'Sub_Ind4', 'Sub_Ind5', 'Sub_Ind6',\n       'Sub_Ind7', 'Investor1', 'Investor2', 'Investor3', 'Investor4',\n       'Investor5', 'Investor6', 'Investor7', 'Investor8', 'Investor9',\n       'Investor10', 'Investor11'],\n      dtype='object')"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Transaction Name                             0\nTransaction Name URL                         0\nOrganization Name                            0\nOrganization Name URL                        0\nFunding Type                                 0\nMoney Raised                                 0\nMoney Raised Currency                        0\nMoney Raised Currency (in USD)               0\nAnnounced Date                               0\nTotal Funding Amount                         0\nTotal Funding Amount Currency                0\nTotal Funding Amount Currency (in USD)       0\nOrganization Description                     0\nOrganization Location                        0\nRaised Series A                              0\nPrimary Industry                             0\nSub_Ind                                   1077\nSub_Ind2                                  2551\nSub_Ind3                                  4689\nSub_Ind4                                  6033\nSub_Ind5                                  6887\nSub_Ind6                                  7416\nSub_Ind7                                  7661\nInvestor1                                 5266\nInvestor2                                 6212\nInvestor3                                 6802\nInvestor4                                 7191\nInvestor5                                 7438\nInvestor6                                 7622\nInvestor7                                 7751\nInvestor8                                 7825\nInvestor9                                 7872\nInvestor10                                7899\nInvestor11                                7915\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    7828\n1     143\nName: Raised Series A, dtype: int64"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "train['Raised Series A'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "750000.0"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "train['Money Raised Currency (in USD)'].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                                                                                           Organization Description  \\\n3240   Xperiti is an AI-powered enterprise software designed to give investment management firms a strategic advantage in research.   \n3241  New Age Meats focuses on rapid development of healthy culitvated meat -- grown from animal cells instead of animal slaughter.   \n3251                                                                    Gangz is the first digital platform for talent recruitment.   \n3254         ProctorEdu solves personality verification problem during online exams by using audio and video recording and proctor.   \n3255                                             Parlor is an all-in-one feedback and discovery software for digital product teams.   \n\n     Organization Name  \n3240           Xperiti  \n3241     New Age Meats  \n3251             Gangz  \n3254        ProctorEdu  \n3255            Parlor  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Organization Description</th>\n      <th>Organization Name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3240</th>\n      <td>Xperiti is an AI-powered enterprise software designed to give investment management firms a strategic advantage in research.</td>\n      <td>Xperiti</td>\n    </tr>\n    <tr>\n      <th>3241</th>\n      <td>New Age Meats focuses on rapid development of healthy culitvated meat -- grown from animal cells instead of animal slaughter.</td>\n      <td>New Age Meats</td>\n    </tr>\n    <tr>\n      <th>3251</th>\n      <td>Gangz is the first digital platform for talent recruitment.</td>\n      <td>Gangz</td>\n    </tr>\n    <tr>\n      <th>3254</th>\n      <td>ProctorEdu solves personality verification problem during online exams by using audio and video recording and proctor.</td>\n      <td>ProctorEdu</td>\n    </tr>\n    <tr>\n      <th>3255</th>\n      <td>Parlor is an all-in-one feedback and discovery software for digital product teams.</td>\n      <td>Parlor</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "pd.set_option('max_colwidth', 150)\n",
    "text = train[['Organization Description', 'Organization Name']]\n",
    "text = text.sort_index()\n",
    "text= pd.DataFrame(text)\n",
    "\n",
    "text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply a first round of text cleaning techniques\n",
    "import re\n",
    "import string\n",
    "\n",
    "def clean_text_round1(text):\n",
    "    '''Make text lowercase, remove text in square brackets, remove punctuation and remove words containing numbers.'''\n",
    "    text = text.lower()\n",
    "    text = re.sub('\\[.*?\\]', '', text)\n",
    "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)\n",
    "    return text\n",
    "\n",
    "round1 = lambda x: clean_text_round1(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                                                                                                     Organization Description\n3240               xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research\n3241               new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter\n3251                                                                               gangz is the first digital platform for talent recruitment\n3254                    proctoredu solves personality verification problem during online exams by using audio and video recording and proctor\n3255                                                          parlor is an allinone feedback and discovery software for digital product teams\n...                                                                                                                                       ...\n27118  virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions\n27119                                                           experience the stock market as a game pick stocks earn coins  get cool prizes\n27123                                       flatfile is a developer of a database software designed to import map and normalize customer data\n27125                               invygo is an appbased car subscription company that gives customers an access to dealership verified cars\n27129           bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection\n\n[7971 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Organization Description</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3240</th>\n      <td>xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research</td>\n    </tr>\n    <tr>\n      <th>3241</th>\n      <td>new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter</td>\n    </tr>\n    <tr>\n      <th>3251</th>\n      <td>gangz is the first digital platform for talent recruitment</td>\n    </tr>\n    <tr>\n      <th>3254</th>\n      <td>proctoredu solves personality verification problem during online exams by using audio and video recording and proctor</td>\n    </tr>\n    <tr>\n      <th>3255</th>\n      <td>parlor is an allinone feedback and discovery software for digital product teams</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>27118</th>\n      <td>virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions</td>\n    </tr>\n    <tr>\n      <th>27119</th>\n      <td>experience the stock market as a game pick stocks earn coins  get cool prizes</td>\n    </tr>\n    <tr>\n      <th>27123</th>\n      <td>flatfile is a developer of a database software designed to import map and normalize customer data</td>\n    </tr>\n    <tr>\n      <th>27125</th>\n      <td>invygo is an appbased car subscription company that gives customers an access to dealership verified cars</td>\n    </tr>\n    <tr>\n      <th>27129</th>\n      <td>bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection</td>\n    </tr>\n  </tbody>\n</table>\n<p>7971 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "\n",
    "# Let's take a look at the updated text\n",
    "data_clean = pd.DataFrame(text['Organization Description'].apply(round1))\n",
    "data_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply a second round of cleaning\n",
    "def clean_text_round2(text):\n",
    "    '''Get rid of some additional punctuation and non-sensical text that was missed the first time around.'''\n",
    "    text = re.sub('[‘’“”…]', '', text)\n",
    "    text = re.sub('\\n', '', text)\n",
    "    return text\n",
    "\n",
    "round2 = lambda x: clean_text_round2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                                                                                                     Organization Description\n3240               xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research\n3241               new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter\n3251                                                                               gangz is the first digital platform for talent recruitment\n3254                    proctoredu solves personality verification problem during online exams by using audio and video recording and proctor\n3255                                                          parlor is an allinone feedback and discovery software for digital product teams\n...                                                                                                                                       ...\n27118  virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions\n27119                                                           experience the stock market as a game pick stocks earn coins  get cool prizes\n27123                                       flatfile is a developer of a database software designed to import map and normalize customer data\n27125                               invygo is an appbased car subscription company that gives customers an access to dealership verified cars\n27129           bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection\n\n[7971 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Organization Description</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3240</th>\n      <td>xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research</td>\n    </tr>\n    <tr>\n      <th>3241</th>\n      <td>new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter</td>\n    </tr>\n    <tr>\n      <th>3251</th>\n      <td>gangz is the first digital platform for talent recruitment</td>\n    </tr>\n    <tr>\n      <th>3254</th>\n      <td>proctoredu solves personality verification problem during online exams by using audio and video recording and proctor</td>\n    </tr>\n    <tr>\n      <th>3255</th>\n      <td>parlor is an allinone feedback and discovery software for digital product teams</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>27118</th>\n      <td>virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions</td>\n    </tr>\n    <tr>\n      <th>27119</th>\n      <td>experience the stock market as a game pick stocks earn coins  get cool prizes</td>\n    </tr>\n    <tr>\n      <th>27123</th>\n      <td>flatfile is a developer of a database software designed to import map and normalize customer data</td>\n    </tr>\n    <tr>\n      <th>27125</th>\n      <td>invygo is an appbased car subscription company that gives customers an access to dealership verified cars</td>\n    </tr>\n    <tr>\n      <th>27129</th>\n      <td>bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection</td>\n    </tr>\n  </tbody>\n</table>\n<p>7971 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "# Let's take a look at the updated text\n",
    "data_clean = pd.DataFrame(data_clean['Organization Description'].apply(round2))\n",
    "data_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                                                                                                                 Organization Description\nOrganization Name                                                                                                                                        \nXperiti                        xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research\nNew Age Meats                  new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter\nGangz                                                                                          gangz is the first digital platform for talent recruitment\nProctorEdu                          proctoredu solves personality verification problem during online exams by using audio and video recording and proctor\nParlor                                                                    parlor is an allinone feedback and discovery software for digital product teams\n...                                                                                                                                                   ...\nVirtAI Tech        virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions\nFantasy Invest                                                              experience the stock market as a game pick stocks earn coins  get cool prizes\nFlatfile                                                flatfile is a developer of a database software designed to import map and normalize customer data\nInvygo                                          invygo is an appbased car subscription company that gives customers an access to dealership verified cars\nBioFlyte                    bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection\n\n[7971 rows x 1 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Organization Description</th>\n    </tr>\n    <tr>\n      <th>Organization Name</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Xperiti</th>\n      <td>xperiti is an aipowered enterprise software designed to give investment management firms a strategic advantage in research</td>\n    </tr>\n    <tr>\n      <th>New Age Meats</th>\n      <td>new age meats focuses on rapid development of healthy culitvated meat  grown from animal cells instead of animal slaughter</td>\n    </tr>\n    <tr>\n      <th>Gangz</th>\n      <td>gangz is the first digital platform for talent recruitment</td>\n    </tr>\n    <tr>\n      <th>ProctorEdu</th>\n      <td>proctoredu solves personality verification problem during online exams by using audio and video recording and proctor</td>\n    </tr>\n    <tr>\n      <th>Parlor</th>\n      <td>parlor is an allinone feedback and discovery software for digital product teams</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>VirtAI Tech</th>\n      <td>virtai tech focuses on providing global data center class ai accelerator virtualization and resource management software and solutions</td>\n    </tr>\n    <tr>\n      <th>Fantasy Invest</th>\n      <td>experience the stock market as a game pick stocks earn coins  get cool prizes</td>\n    </tr>\n    <tr>\n      <th>Flatfile</th>\n      <td>flatfile is a developer of a database software designed to import map and normalize customer data</td>\n    </tr>\n    <tr>\n      <th>Invygo</th>\n      <td>invygo is an appbased car subscription company that gives customers an access to dealership verified cars</td>\n    </tr>\n    <tr>\n      <th>BioFlyte</th>\n      <td>bioflyte is a biodefense company created to commercialize a revolutionary new class of fieldable biological threat collection</td>\n    </tr>\n  </tbody>\n</table>\n<p>7971 rows × 1 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "data_clean['Organization Name'] = train['Organization Name']\n",
    "data_clean.set_index('Organization Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "4"
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "# use this link to start tomorrow: https://stackoverflow.com/questions/37807308/how-can-i-reduce-the-file-size-of-my-ipython-notebook\n",
    "2+2 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyError",
     "evalue": "\"['tokenized_Description'] not found in axis\"",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-fa130931d85a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Arrange data into X features matrix and y target vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Raised Series A'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'tokenized_Description'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Announced Date'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'tokenized_Description'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Announced Date'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   3995\u001b[0m             \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3996\u001b[0m             \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3997\u001b[0;31m             \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3998\u001b[0m         )\n\u001b[1;32m   3999\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   3934\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3935\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3936\u001b[0;31m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_drop_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3937\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3938\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_drop_axis\u001b[0;34m(self, labels, axis, level, errors)\u001b[0m\n\u001b[1;32m   3968\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3969\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3970\u001b[0;31m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3971\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3972\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, errors)\u001b[0m\n\u001b[1;32m   5016\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5017\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"ignore\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5018\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{labels[mask]} not found in axis\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5019\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5020\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"['tokenized_Description'] not found in axis\""
     ]
    }
   ],
   "source": [
    "# Arrange data into X features matrix and y target vector\n",
    "target = 'Raised Series A'\n",
    "X_train = train.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_train = train[target]\n",
    "X_val = val.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_val = val[target]\n",
    "X_test = test.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_test = test[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-de02fd11f310>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "X_train.shape , y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    2899\n1      13\nName: Raised Series A, dtype: int64"
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "source": [
    "val['Raised Series A'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-f6576de5cd71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mpipe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "import category_encoders as ce\n",
    "from sklearn.impute import SimpleImputer\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "pipeline = make_pipeline(\n",
    "    ce.OrdinalEncoder(),\n",
    "    SimpleImputer(strategy='median'),\n",
    "    XGBClassifier(n_estimators=100,\n",
    "                  random_state=42,\n",
    "                  n_jobs=-1          \n",
    ")\n",
    ")\n",
    "\n",
    "###scores = cross_val_score(pipeline, X_train, y_train, cv=10, \n",
    "                         ##scoring = 'average_precision')\n",
    "\n",
    "###scores_val = cross_val_score(pipeline, X_val, y_val, cv=10, \n",
    "                         ##scoring = 'average_precision')'''\n",
    "\n",
    "\n",
    "pipe = pipeline.fit(X_train, y_train);\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'pipe' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-ebee1ac97706>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mxgb_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"XGBoost Classifier val Recall Accuracy:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecall_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"XGBoost Classifier val Accuracy Score:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"XGBoost Classifier val Precision Accuracy:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprecision_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pipe' is not defined"
     ]
    }
   ],
   "source": [
    "xgb_val = pipe.predict(X_val)\n",
    "print(\"XGBoost Classifier val Recall Accuracy:\", recall_score(y_val, xgb_val))\n",
    "print(\"XGBoost Classifier val Accuracy Score:\", accuracy_score(y_val, xgb_val))\n",
    "print(\"XGBoost Classifier val Precision Accuracy:\", precision_score(y_val, xgb_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'xgb_val' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-bf736b189cf2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# when i ran predict(on X_val, it predicted that 10 of the 18 'Yes' values incorrectly. This could explain why the precision and recall accuracy is so low.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0munique_elements\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounts_elements\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxgb_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_counts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_elements\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounts_elements\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'xgb_val' is not defined"
     ]
    }
   ],
   "source": [
    "# when i ran predict(on X_val, it predicted that 10 of the 18 'Yes' values incorrectly. This could explain why the precision and recall accuracy is so low. \n",
    "unique_elements, counts_elements = np.unique(xgb_val, return_counts=True)\n",
    "print(np.asarray((unique_elements, counts_elements)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyError",
     "evalue": "\"['tokenized_Description'] not found in axis\"",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-bb8f760d9b78>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#make another\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Raised Series A'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'tokenized_Description'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Announced Date'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'tokenized_Description'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Announced Date'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   3995\u001b[0m             \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3996\u001b[0m             \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3997\u001b[0;31m             \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3998\u001b[0m         )\n\u001b[1;32m   3999\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   3934\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3935\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3936\u001b[0;31m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_drop_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3937\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3938\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_drop_axis\u001b[0;34m(self, labels, axis, level, errors)\u001b[0m\n\u001b[1;32m   3968\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3969\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3970\u001b[0;31m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3971\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3972\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/unit2/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, errors)\u001b[0m\n\u001b[1;32m   5016\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5017\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"ignore\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5018\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{labels[mask]} not found in axis\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5019\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5020\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"['tokenized_Description'] not found in axis\""
     ]
    }
   ],
   "source": [
    "#make another \n",
    "target = 'Raised Series A'\n",
    "X_train = train.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_train = train[target]\n",
    "X_val = val.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_val = val[target]\n",
    "X_test = test.drop(columns=[target,'tokenized_Description','Announced Date'])\n",
    "y_test = test[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "error",
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-33-15af87e117b1>, line 2)",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-33-15af87e117b1>\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m    pip install -U imbalanced-learn\u001b[0m\n\u001b[0m              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "#install imbalance package\n",
    "pip install -U imbalanced-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-3989c2d97a16>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mimblearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mover_sampling\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSMOTE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0msm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSMOTE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train_res\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_res\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_sample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import SMOTE \n",
    "sm = SMOTE(random_state = 2) \n",
    "X_train_res, y_train_res = sm.fit_sample(X_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def corpora(d):\n",
    "    #'Dictionary' Representation of all the words in our corpus\n",
    "    #id2word = corpora.Dictionary(d[\"tokenized_Description\"])\n",
    "    #Removing extreme values from the dataset\n",
    "    #id2word.filter_extremes(no_below=5, no_above=0.95)\n",
    "    #bag of words(bow) representation of our corpus\n",
    "    #corpus = [id2word.doc2bow(text) for text in d[\"tokenized_Description\"]]\n",
    "    #return d "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train = corpora(train)\n",
    "#val = corpora(val)\n",
    "#test = corpora(test)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'y_train' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-2e5d667bca68>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mlb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLabelBinarizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mnumber\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mrecall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_val_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'recall'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'y_train' is not defined"
     ]
    }
   ],
   "source": [
    "# precision, recall and F1\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "lb = LabelBinarizer()\n",
    "y_train = np.array([number[0] for number in lb.fit_transform(y_train)])\n",
    "\n",
    "recall = cross_val_score(pipeline, X_train, y_train, cv=5, scoring='recall')\n",
    "print('Recall', np.mean(recall), recall)\n",
    "precision = cross_val_score(pipeline, X_train, y_train, cv=5, scoring='precision')\n",
    "print('Precision', np.mean(precision), precision)\n",
    "f1 = cross_val_score(pipeline, X_train, y_train, cv=5, scoring='f1')\n",
    "print('F1', np.mean(f1), f1)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37764bitunit2condaf2d4faf30a5848cbbdf1cd45adcc297a",
   "display_name": "Python 3.7.7 64-bit ('unit2': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}