incidents.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:45.627597Z",
     "start_time": "2019-04-15T23:20:45.616657Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import re\n",
    "# Tutorial about Python regular expressions: https://pymotw.com/2/re/\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors\n",
    "import pickle\n",
    "\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "\n",
    "from bs4 import BeautifulSoup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:47.438751Z",
     "start_time": "2019-04-15T23:20:47.416811Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('./incident.csv', encoding='latin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:48.026181Z",
     "start_time": "2019-04-15T23:20:48.018201Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_product</th>\n",
       "      <th>assignment_group</th>\n",
       "      <th>short_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-FIT-TTR-L2</td>\n",
       "      <td>IMPORTED RECEIPTS IN ARIBA FROM SAP!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SonicWall(DRR) (422697)</td>\n",
       "      <td>ITOPS-DISD-L1</td>\n",
       "      <td>hi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-CORP-SUPPORT</td>\n",
       "      <td>Jabra Headset not functioning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-CORP-SUPPORT</td>\n",
       "      <td>I have a plantronics headset and it is not wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SonicWall(DRR) (422697)</td>\n",
       "      <td>ITOPS-DISD-L1</td>\n",
       "      <td>Couldnot connect to VPN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 u_product    assignment_group  \\\n",
       "0     Ariba (DRR) (703440)    ITOPS-FIT-TTR-L2   \n",
       "1  SonicWall(DRR) (422697)       ITOPS-DISD-L1   \n",
       "2     Ariba (DRR) (703440)  ITOPS-CORP-SUPPORT   \n",
       "3     Ariba (DRR) (703440)  ITOPS-CORP-SUPPORT   \n",
       "4  SonicWall(DRR) (422697)       ITOPS-DISD-L1   \n",
       "\n",
       "                                   short_description  \n",
       "0               IMPORTED RECEIPTS IN ARIBA FROM SAP!  \n",
       "1                                                 hi  \n",
       "2                      Jabra Headset not functioning  \n",
       "3  I have a plantronics headset and it is not wor...  \n",
       "4                            Couldnot connect to VPN  "
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:49.110282Z",
     "start_time": "2019-04-15T23:20:49.103298Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Corporate Oracle WebCenter (COWC) (379227)                 2835\n",
       "Corporate Extended Oracle WebCenter (CEOWC) (417127)       1813\n",
       "BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)             995\n",
       "Ariba (DRR) (703440)                                        343\n",
       "HRDM - Human Resources Document Management (32300)          306\n",
       "SonicWall(DRR) (422697)                                     235\n",
       "EDMS - ELECTRONIC DOCUMENT MANAGEMENT SYSTEM (35994)         48\n",
       "ES - EMPLOYMENT SERVICES (35995)                             37\n",
       "MARS (Marketing and Advertising Review System) (250250)      17\n",
       "Gale(DRR) (422694)                                            7\n",
       "Performance Link (DRR) (883344)                               2\n",
       "Softheon (DRR) (883348)                                       2\n",
       "Force 10(DRR) (422695)                                        2\n",
       "GOE - GLOBAL OPERATIONS ENGINEERING (35997)                   2\n",
       "Name: u_product, dtype: int64"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['u_product'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:49.679757Z",
     "start_time": "2019-04-15T23:20:49.672776Z"
    }
   },
   "outputs": [],
   "source": [
    "df['level'] = df['assignment_group'].apply(lambda x: x.split('-')[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:50.950357Z",
     "start_time": "2019-04-15T23:20:50.942378Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_product</th>\n",
       "      <th>assignment_group</th>\n",
       "      <th>short_description</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-FIT-TTR-L2</td>\n",
       "      <td>IMPORTED RECEIPTS IN ARIBA FROM SAP!</td>\n",
       "      <td>L2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SonicWall(DRR) (422697)</td>\n",
       "      <td>ITOPS-DISD-L1</td>\n",
       "      <td>hi</td>\n",
       "      <td>L1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-CORP-SUPPORT</td>\n",
       "      <td>Jabra Headset not functioning</td>\n",
       "      <td>SUPPORT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ariba (DRR) (703440)</td>\n",
       "      <td>ITOPS-CORP-SUPPORT</td>\n",
       "      <td>I have a plantronics headset and it is not wor...</td>\n",
       "      <td>SUPPORT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SonicWall(DRR) (422697)</td>\n",
       "      <td>ITOPS-DISD-L1</td>\n",
       "      <td>Couldnot connect to VPN</td>\n",
       "      <td>L1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 u_product    assignment_group  \\\n",
       "0     Ariba (DRR) (703440)    ITOPS-FIT-TTR-L2   \n",
       "1  SonicWall(DRR) (422697)       ITOPS-DISD-L1   \n",
       "2     Ariba (DRR) (703440)  ITOPS-CORP-SUPPORT   \n",
       "3     Ariba (DRR) (703440)  ITOPS-CORP-SUPPORT   \n",
       "4  SonicWall(DRR) (422697)       ITOPS-DISD-L1   \n",
       "\n",
       "                                   short_description    level  \n",
       "0               IMPORTED RECEIPTS IN ARIBA FROM SAP!       L2  \n",
       "1                                                 hi       L1  \n",
       "2                      Jabra Headset not functioning  SUPPORT  \n",
       "3  I have a plantronics headset and it is not wor...  SUPPORT  \n",
       "4                            Couldnot connect to VPN       L1  "
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:52.253869Z",
     "start_time": "2019-04-15T23:20:52.238911Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/a/47091490/4084039\n",
    "import re\n",
    "\n",
    "def decontracted(phrase):\n",
    "    # specific\n",
    "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
    "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "\n",
    "    # general\n",
    "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
    "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
    "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "    return phrase\n",
    "\n",
    "# https://gist.github.com/sebleier/554280\n",
    "# we are removing the words from the stop words list: 'no', 'nor', 'not'\n",
    "# <br /><br /> ==> after the above steps, we are getting \"br br\"\n",
    "# we are including them into stop words list\n",
    "# instead of <br /> if we have <br/> these tags would have revmoved in the 1st step\n",
    "\n",
    "stopwords= set(['br', 'the', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\\\n",
    "            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \\\n",
    "            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\\\n",
    "            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', \\\n",
    "            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \\\n",
    "            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \\\n",
    "            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\\\n",
    "            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\\\n",
    "            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\\\n",
    "            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \\\n",
    "            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', \\\n",
    "            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\\\n",
    "            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\\\n",
    "            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", \\\n",
    "            'won', \"won't\", 'wouldn', \"wouldn't\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:55.212955Z",
     "start_time": "2019-04-15T23:20:53.125539Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6644/6644 [00:02<00:00, 3199.64it/s]\n"
     ]
    }
   ],
   "source": [
    "# Combining all the above stundents \n",
    "from tqdm import tqdm\n",
    "short_desc = []\n",
    "# tqdm is for printing the status bar\n",
    "for sentance in tqdm(df['short_description'].values):\n",
    "    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n",
    "    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n",
    "    sentance = decontracted(sentance)\n",
    "    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n",
    "    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n",
    "    # https://gist.github.com/sebleier/554280\n",
    "    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n",
    "    short_desc.append(sentance.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:55.834293Z",
     "start_time": "2019-04-15T23:20:55.829305Z"
    }
   },
   "outputs": [],
   "source": [
    "df['cleaned'] = short_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:57.041064Z",
     "start_time": "2019-04-15T23:20:57.035079Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(df['cleaned'], df['u_product'],\n",
    "                                                    shuffle=False, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:20:58.225895Z",
     "start_time": "2019-04-15T23:20:58.062333Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "some sample features(unique words in the corpus) ['able', 'able access', 'able approve', 'able connect', 'able connecting', 'able create', 'able log', 'able login', 'able push', 'able raise']\n",
      "==================================================\n",
      "the type of count vectorizer  <class 'scipy.sparse.csr.csr_matrix'>\n",
      "the shape of out text TFIDF vectorizer  (4650, 3602)\n",
      "the number of unique words including both unigrams and bigrams  3602\n"
     ]
    }
   ],
   "source": [
    "tf_idf_vect = TfidfVectorizer(ngram_range=(1,2))\n",
    "tf_idf_vect.fit(X_train)\n",
    "print(\"some sample features(unique words in the corpus)\",tf_idf_vect.get_feature_names()[0:10])\n",
    "print('='*50)\n",
    "\n",
    "X_train_tf_idf = tf_idf_vect.transform(X_train)\n",
    "print(\"the type of count vectorizer \",type(X_train_tf_idf))\n",
    "print(\"the shape of out text TFIDF vectorizer \",X_train_tf_idf.get_shape())\n",
    "print(\"the number of unique words including both unigrams and bigrams \", X_train_tf_idf.get_shape()[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:06.773029Z",
     "start_time": "2019-04-15T23:21:06.758109Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('tf_idf_vect_incidents_product_6k.pickle', 'wb') as f:\n",
    "    pickle.dump(tf_idf_vect, f, pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:09.212503Z",
     "start_time": "2019-04-15T23:21:09.175603Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test_tfidf = tf_idf_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:09.931582Z",
     "start_time": "2019-04-15T23:21:09.927591Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1994x3602 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 22606 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:11.835487Z",
     "start_time": "2019-04-15T23:21:11.831532Z"
    }
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:16.498014Z",
     "start_time": "2019-04-15T23:21:13.052232Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6780341023069207"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = XGBClassifier()\n",
    "clf.fit(X_train_tf_idf, Y_train)\n",
    "clf.score(X_test_tfidf, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:23:18.344109Z",
     "start_time": "2019-04-15T23:23:18.336080Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('clf_incidents_product_6k.pickle', 'wb') as f:\n",
    "    pickle.dump(clf, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:21:54.942170Z",
     "start_time": "2019-04-15T23:21:54.935189Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                    Ariba (DRR) (703440)\n",
       "1                                 SonicWall(DRR) (422697)\n",
       "2                                    Ariba (DRR) (703440)\n",
       "3                                    Ariba (DRR) (703440)\n",
       "4                                 SonicWall(DRR) (422697)\n",
       "5                                 SonicWall(DRR) (422697)\n",
       "6       Corporate Extended Oracle WebCenter (CEOWC) (4...\n",
       "7                                    Ariba (DRR) (703440)\n",
       "8                                 SonicWall(DRR) (422697)\n",
       "9                                    Ariba (DRR) (703440)\n",
       "10                                SonicWall(DRR) (422697)\n",
       "11             Corporate Oracle WebCenter (COWC) (379227)\n",
       "12                                SonicWall(DRR) (422697)\n",
       "13                       ES - EMPLOYMENT SERVICES (35995)\n",
       "14                                SonicWall(DRR) (422697)\n",
       "15                                SonicWall(DRR) (422697)\n",
       "16                                SonicWall(DRR) (422697)\n",
       "17                                SonicWall(DRR) (422697)\n",
       "18                                SonicWall(DRR) (422697)\n",
       "19                                   Ariba (DRR) (703440)\n",
       "20                                SonicWall(DRR) (422697)\n",
       "21      HRDM - Human Resources Document Management (32...\n",
       "22                                SonicWall(DRR) (422697)\n",
       "23                                SonicWall(DRR) (422697)\n",
       "24             Corporate Oracle WebCenter (COWC) (379227)\n",
       "25      Corporate Extended Oracle WebCenter (CEOWC) (4...\n",
       "26      Corporate Extended Oracle WebCenter (CEOWC) (4...\n",
       "27                                SonicWall(DRR) (422697)\n",
       "28                                   Ariba (DRR) (703440)\n",
       "29             Corporate Oracle WebCenter (COWC) (379227)\n",
       "                              ...                        \n",
       "4620      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4621      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4622      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4623      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4624           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4625      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4626           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4627           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4628           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4629      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4630      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4631           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4632      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4633      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4634           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4635      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4636           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4637    HRDM - Human Resources Document Management (32...\n",
       "4638    Corporate Extended Oracle WebCenter (CEOWC) (4...\n",
       "4639    HRDM - Human Resources Document Management (32...\n",
       "4640      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4641           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4642           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4643           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4644      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4645      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4646           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4647      BQMS - BRAZIL QUALITY MANAGEMENT SYSTEM (35998)\n",
       "4648           Corporate Oracle WebCenter (COWC) (379227)\n",
       "4649    HRDM - Human Resources Document Management (32...\n",
       "Name: u_product, Length: 4650, dtype: object"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T22:25:54.026287Z",
     "start_time": "2019-04-15T22:25:53.322082Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('./largest.csv', encoding='latin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T22:26:18.710108Z",
     "start_time": "2019-04-15T22:26:18.664123Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MICROSOFT SQL SERVER                                      5702\n",
       "ORACLE DATABASE                                           5577\n",
       "DELTA (36122)                                             5348\n",
       "OMEGA (34653)                                             4533\n",
       "NETWORK DATA DEVICE                                       4253\n",
       "POWEREDGE                                                 4087\n",
       "Oracle Enterprise Manager (32850)                         3268\n",
       "SFDC - Dell Main (31930)                                  3027\n",
       "DragonFx DAO (32090)                                      2596\n",
       "Oracle Transportation Management (OTM) (416055)           2152\n",
       "OFS (34520)                                               2141\n",
       "Foglight (Dell IT) (720743)                               2128\n",
       "WINDOWS 2008 SERVER                                       2090\n",
       "DSA - Dell Sales Application (903974)                     2090\n",
       "GCM Pipeline Services (537008)                            2021\n",
       "GII - APJ (36268)                                         1978\n",
       "Syncplicity                                               1737\n",
       "DragonFX APJ (32214)                                      1560\n",
       "DOMINO GLOBAL (35882)                                     1548\n",
       "LEGATO (EMC Networker) (32797)                            1545\n",
       "MES Dynamics AX (249340)                                  1523\n",
       "DIGITAL FULFILLMENT (242975)                              1497\n",
       "SFDC - SALESFORCE1 (412467)                               1488\n",
       "GOAL (Global Opportunity Approval Live) (32252)           1476\n",
       "LA DOMS                                                   1277\n",
       "Infosphere Advanced Edition (778827)                      1252\n",
       "ORACLE 11I APPLICATIONS - CSI ACCOUNT                     1246\n",
       "VIRTUAL SERVER/MACHINE                                    1182\n",
       "GII - EMEA (36111)                                        1136\n",
       "ServiceNow (391696)                                       1101\n",
       "                                                          ... \n",
       "Web Proxy PAC Script (502968)                                1\n",
       "BO Reports GLOVIA (246804)                                   1\n",
       "EDGE- DOCUMENTUM                                             1\n",
       "Cisco Secure ACS (TACACS) v5 (695295)                        1\n",
       "IDS_SUB_INT_SABA_FEED                                        1\n",
       "PRECISION NOTEBOOK                                           1\n",
       "Cisco ACS (legacy Dell IT TACACS) (33905)                    1\n",
       "OpenText SECURELINK (36157)                                  1\n",
       "IDS_SUB_INT_S360_SFDC                                        1\n",
       "ADMINSCAN (32275)                                            1\n",
       "iMAP CUSTOMER HUB                                            1\n",
       "ORACLE 11I APPLICATIONS - ADVANCED PRICING                   1\n",
       "BigIP Dashboard (702306)                                     1\n",
       "ITDS Storage Operations (352446)                             1\n",
       "Coveo (700341)                                               1\n",
       "EXPERIAN TRANSACT ODS (722253)                               1\n",
       "APJ - KEIJIBAN (34442)                                       1\n",
       "OLS - MIDDLEWARE                                             1\n",
       "EMS (EMPLOYEE MANAGEMENT SUITE) (36169)                      1\n",
       "FRDI (36227)                                                 1\n",
       "CTI DATA LOADER (34722)                                      1\n",
       "MASTER.DB.ON.AUSFWCPSDRDB02.PRODUCTION.ONLINE.DELL.COM       1\n",
       "PasswordPortal                                               1\n",
       "zzzSAP - ECC System                                          1\n",
       "SSO.EMC.COM LOGIN                                            1\n",
       "MICROSOFT OUTLOOK DISTRIBUTION LIST                          1\n",
       "ACCESS - RECERTIFY - CONTROL-M                               1\n",
       "Retail Demand Planner - RDP (358775)                         1\n",
       "CITRIX WEB CLIENT                                            1\n",
       "DASH (Dell Audit Solution) (445691)                          1\n",
       "Name: ClosureProductName, Length: 1168, dtype: int64"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['ClosureProductName'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:25:49.775956Z",
     "start_time": "2019-04-15T23:25:49.769004Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(df['cleaned'], df['level'], shuffle=False, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:25:51.914234Z",
     "start_time": "2019-04-15T23:25:51.744688Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "some sample features(unique words in the corpus) ['able', 'able access', 'able approve', 'able connect', 'able connecting', 'able create', 'able log', 'able login', 'able push', 'able raise']\n",
      "==================================================\n",
      "the type of count vectorizer  <class 'scipy.sparse.csr.csr_matrix'>\n",
      "the shape of out text TFIDF vectorizer  (5315, 3783)\n",
      "the number of unique words including both unigrams and bigrams  3783\n"
     ]
    }
   ],
   "source": [
    "tf_idf_vect = TfidfVectorizer(ngram_range=(1,2))\n",
    "tf_idf_vect.fit(X_train)\n",
    "print(\"some sample features(unique words in the corpus)\",tf_idf_vect.get_feature_names()[0:10])\n",
    "print('='*50)\n",
    "\n",
    "X_train_tf_idf = tf_idf_vect.transform(X_train)\n",
    "print(\"the type of count vectorizer \",type(X_train_tf_idf))\n",
    "print(\"the shape of out text TFIDF vectorizer \",X_train_tf_idf.get_shape())\n",
    "print(\"the number of unique words including both unigrams and bigrams \", X_train_tf_idf.get_shape()[1])\n",
    "\n",
    "X_test_tfidf = tf_idf_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:25:53.012296Z",
     "start_time": "2019-04-15T23:25:52.995341Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('tf_idf_vect_level_product_6k.pickle', 'wb') as f:\n",
    "    pickle.dump(tf_idf_vect, f, pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:26:09.567010Z",
     "start_time": "2019-04-15T23:25:53.897958Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7863054928517682"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = XGBClassifier()\n",
    "clf.fit(X_train_tf_idf, Y_train)\n",
    "clf.score(X_test_tfidf, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T23:26:09.582967Z",
     "start_time": "2019-04-15T23:26:09.569036Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('clf_level_product_6k.pickle', 'wb') as f:\n",
    "    pickle.dump(clf, f, pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}