exam29_iris07_Gradient_Boosting.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"exam29_iris07_Gradient_Boosting.ipynb","provenance":[],"authorship_tag":"ABX9TyPil/EBC5B51OScQSjeuBzW"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"CxFoFlltz32W"},"outputs":[],"source":["#gradient boosting regression tree : 여러개의 decision tree를 묶어 강력한 model을 만드는 앙상블 기법\n","#random forest와 달리 이전 tree의 오차를 보완하는 방식으로 tree생성\n","#1~5 정도 깊이의 tree를 사용하므로 메모리를 적게 사용하고 예측도 빠름\n","#얕은 트리들을 계속 연결, parameter설정에서 random forest보다 민감, 잘 조정하면 높은 정확도 제공\n","#learnig_rate를 높이면 보정을 강하게 하기 때문에 복잡한 모델을 생성\n","#n_estimator값을 키우면 ensemble에 트리가 더 많이 추가되어 모델의 복잡도가 커지고 train세트를 더 정확하게 figure\n","# https://woolulu.tistory.com/30"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"KhmgMnCX5Ckd","executionInfo":{"status":"ok","timestamp":1650608701450,"user_tz":-540,"elapsed":2771,"user":{"displayName":"최하진","userId":"13771240898172212384"}}},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","from sklearn.datasets import load_iris\n","from sklearn.preprocessing import StandardScaler\n","from sklearn import svm\n","from sklearn.model_selection import *\n","from sklearn.metrics import classification_report, confusion_matrix\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.ensemble import GradientBoostingClassifier\n","from sklearn import tree"]},{"cell_type":"code","source":["plt.rcParams['figure.figsize'] = [7, 7]\n","sns.set(style='darkgrid')\n","plt.rcParams['scatter.edgecolors'] = 'black'\n","pd.set_option('display.max_columns', None)\n","pd.set_option('display.max_row', None)\n","pd.set_option('display.unicode.east_asian_width', True)"],"metadata":{"id":"Rj3RQQkt5TVn","executionInfo":{"status":"ok","timestamp":1650608702814,"user_tz":-540,"elapsed":262,"user":{"displayName":"최하진","userId":"13771240898172212384"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["iris_dataset = load_iris()\n","iris = pd.DataFrame(iris_dataset.data,\n","        columns=iris_dataset.feature_names)\n","labels = iris_dataset.target_names\n","iris.info()\n","print(iris.head())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6MqREdnB5PSa","executionInfo":{"status":"ok","timestamp":1650608704583,"user_tz":-540,"elapsed":373,"user":{"displayName":"최하진","userId":"13771240898172212384"}},"outputId":"391b6b3e-f182-437b-eee3-697fd63869b0"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 150 entries, 0 to 149\n","Data columns (total 4 columns):\n"," #   Column             Non-Null Count  Dtype  \n","---  ------             --------------  -----  \n"," 0   sepal length (cm)  150 non-null    float64\n"," 1   sepal width (cm)   150 non-null    float64\n"," 2   petal length (cm)  150 non-null    float64\n"," 3   petal width (cm)   150 non-null    float64\n","dtypes: float64(4)\n","memory usage: 4.8 KB\n","   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)\n","0                5.1               3.5                1.4               0.2\n","1                4.9               3.0                1.4               0.2\n","2                4.7               3.2                1.3               0.2\n","3                4.6               3.1                1.5               0.2\n","4                5.0               3.6                1.4               0.2\n"]}]},{"cell_type":"code","source":["label = iris_dataset.target\n","print(label)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1DH-lmSJ9EUo","executionInfo":{"status":"ok","timestamp":1650608705829,"user_tz":-540,"elapsed":5,"user":{"displayName":"최하진","userId":"13771240898172212384"}},"outputId":"c71c107b-bcd7-4537-f68c-b7dc3b60e969"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"," 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"," 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2\n"," 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n"," 2 2]\n"]}]},{"cell_type":"code","source":["scaler = StandardScaler()\n","iris = scaler.fit_transform(iris)\n","Features = pd.DataFrame(iris, columns=['SL', 'SW', 'PL', 'PW'])\n","print(Features.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"L780Qbwi9PXH","executionInfo":{"status":"ok","timestamp":1650608707103,"user_tz":-540,"elapsed":4,"user":{"displayName":"최하진","userId":"13771240898172212384"}},"outputId":"90c92531-a8c7-4273-e00d-c79d41b7ff59"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["(150, 4)\n"]}]},{"cell_type":"code","source":["X_train, X_test, Y_train, Y_test = train_test_split(\n","            Features, label, test_size=0.2)"],"metadata":{"id":"fhkjyYPDzwsf","executionInfo":{"status":"ok","timestamp":1650608710263,"user_tz":-540,"elapsed":265,"user":{"displayName":"최하진","userId":"13771240898172212384"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["n_estimators = [50, 100, 150, 200, 250, 300, 350, 400]\n","learning_rate = [0.1, 0.2, .3, .4, .5, .6, .7, .8, .9, 1.0]\n","max_depth = [2, 3, 4, 5, 6, 7, 8]\n","param = {'n_estimators':n_estimators, 'learning_rate':learning_rate,\n","         'max_depth':max_depth}\n","cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, \n","                            random_state=868)\n","iris_GBC = GridSearchCV(estimator=GradientBoostingClassifier(),\n","            param_grid=param, scoring='accuracy',\n","            n_jobs=-1, cv=cv)\n","iris_GBC.fit(X_train, Y_train)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YPAJmXMTxor_","executionInfo":{"status":"ok","timestamp":1650592889696,"user_tz":-540,"elapsed":763384,"user":{"displayName":"KyoungYoon Jeoung","userId":"08675786042851087109"}},"outputId":"fafb55e1-ee4c-4127-961c-3cefba3e6acc"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["GridSearchCV(cv=StratifiedShuffleSplit(n_splits=5, random_state=868, test_size=0.2,\n","            train_size=None),\n","             estimator=GradientBoostingClassifier(), n_jobs=-1,\n","             param_grid={'learning_rate': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7,\n","                                           0.8, 0.9, 1.0],\n","                         'max_depth': [2, 3, 4, 5, 6, 7, 8],\n","                         'n_estimators': [50, 100, 150, 200, 250, 300, 350,\n","                                          400]},\n","             scoring='accuracy')"]},"metadata":{},"execution_count":11}]},{"cell_type":"code","source":["print(iris_GBC.best_score_)\n","print(iris_GBC.best_params_)\n","print(iris_GBC.best_estimator_)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4s-pIuxsAQv-","executionInfo":{"status":"ok","timestamp":1650592923889,"user_tz":-540,"elapsed":350,"user":{"displayName":"KyoungYoon Jeoung","userId":"08675786042851087109"}},"outputId":"6e0264db-a873-4b76-ce83-198567845435"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["0.95\n","{'learning_rate': 0.5, 'max_depth': 4, 'n_estimators': 50}\n","GradientBoostingClassifier(learning_rate=0.5, max_depth=4, n_estimators=50)\n"]}]},{"cell_type":"code","source":["for i in range(1, 100):\n","    X_train, X_test, Y_train, Y_test = train_test_split(\n","            Features, label, test_size=0.2, random_state=i)\n","    iris_GBC_best = iris_GBC.best_estimator_\n","    iris_GBC_best.fit(X_train, Y_train)\n","\n","    train_score = iris_GBC_best.score(X_train, Y_train)\n","    test_score = iris_GBC_best.score(X_test, Y_test)\n","    if test_score >= train_score:\n","        print('test:{} train:{} random_state:{}'.format(\n","            test_score, train_score, i))"],"metadata":{"id":"AQnLxMV_1FHT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X_train, X_test, Y_train, Y_test = train_test_split(\n","            Features, label, test_size=0.2, random_state=42)\n","iris_GBC_best = iris_GBC.best_estimator_\n","iris_GBC_best.fit(X_train, Y_train)\n","\n","train_score = iris_GBC_best.score(X_train, Y_train)\n","test_score = iris_GBC_best.score(X_test, Y_test)\n","\n","print('test:{} train:{}'.format(\n","        test_score, train_score))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oR1dRQN75ly_","executionInfo":{"status":"ok","timestamp":1650593194915,"user_tz":-540,"elapsed":355,"user":{"displayName":"KyoungYoon Jeoung","userId":"08675786042851087109"}},"outputId":"e94ae870-df9e-4d41-9046-71b692cbb9f8"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["test:1.0 train:1.0\n"]}]},{"cell_type":"code","source":["X_train, X_test, Y_train, Y_test = train_test_split(\n","            Features, label, test_size=0.2, random_state=42)\n","iris_GBC_best = GradientBoostingClassifier(learning_rate=0.4, max_depth=3, n_estimators=30)\n","iris_GBC_best.fit(X_train, Y_train)\n","\n","train_score = iris_GBC_best.score(X_train, Y_train)\n","test_score = iris_GBC_best.score(X_test, Y_test)\n","\n","print('test:{} train:{}'.format(\n","        test_score, train_score))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"V1wN7sIc7opt","executionInfo":{"status":"ok","timestamp":1650608760984,"user_tz":-540,"elapsed":259,"user":{"displayName":"최하진","userId":"13771240898172212384"}},"outputId":"e1f53b0e-7ce2-4fa2-97b7-5356dd5e072c"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["test:1.0 train:1.0\n"]}]},{"cell_type":"code","source":["pd.DataFrame(confusion_matrix(Y_test, iris_GBC_best.predict(X_test)),\n","        columns=['P_setosa', 'P_versicolor', 'P_virginica'],\n","        index=['A_setosa', 'A_versicolor', 'A_virginica'])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"4ymtL_Gg5z0K","executionInfo":{"status":"ok","timestamp":1650593205362,"user_tz":-540,"elapsed":7,"user":{"displayName":"KyoungYoon Jeoung","userId":"08675786042851087109"}},"outputId":"fe4a0c92-968f-437f-a9f9-20d936b82045"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["              P_setosa  P_versicolor  P_virginica\n","A_setosa            10             0            0\n","A_versicolor         0             9            0\n","A_virginica          0             0           11"],"text/html":["\n","  <div id=\"df-3d505de2-0f05-4090-a2b2-9299b258dfff\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>P_setosa</th>\n","      <th>P_versicolor</th>\n","      <th>P_virginica</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>A_setosa</th>\n","      <td>10</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>A_versicolor</th>\n","      <td>0</td>\n","      <td>9</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>A_virginica</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>11</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d505de2-0f05-4090-a2b2-9299b258dfff')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-3d505de2-0f05-4090-a2b2-9299b258dfff button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-3d505de2-0f05-4090-a2b2-9299b258dfff');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["print(classification_report(Y_test, iris_GBC_best.predict(X_test)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8fvks2Wf6Di7","executionInfo":{"status":"ok","timestamp":1650593209912,"user_tz":-540,"elapsed":347,"user":{"displayName":"KyoungYoon Jeoung","userId":"08675786042851087109"}},"outputId":"508af470-f58f-46f9-fe5d-846c5ca3f530"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","           0       1.00      1.00      1.00        10\n","           1       1.00      1.00      1.00         9\n","           2       1.00      1.00      1.00        11\n","\n","    accuracy                           1.00        30\n","   macro avg       1.00      1.00      1.00        30\n","weighted avg       1.00      1.00      1.00        30\n","\n"]}]}]}