Feature_Importance.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Feature Importance.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CI-SSLab/bladder_cancer_tucr_xai/blob/main/Feature_Importance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xp62mmTyuczb",
        "outputId": "7447d79c-4482-460f-8ec3-1a6a8bb3604a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.0.2\n"
          ]
        }
      ],
      "source": [
        "# Check scikit-learn version\n",
        "import sklearn\n",
        "print(sklearn.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from numpy import genfromtxt\n",
        "\n",
        "X = np.array(pd.read_csv(\"X_0001_2.csv\", header=None))\n",
        "\n",
        "y = np.array(pd.read_csv(\"Y.csv\", header=None)).T\n",
        "\n",
        "labels = np.array(pd.read_csv(\"Labels_0001_2.csv\", header=None))\n",
        "\n",
        "print(X.shape)"
      ],
      "metadata": {
        "id": "pFL-hCFz4CIX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a4fb4b2-e1fc-4f14-df87-c6e9bee9620f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41, 70)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Lasso \n",
        "from sklearn import linear_model\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.linear_model import Lasso\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import cross_validate\n",
        "from sklearn.model_selection import RepeatedKFold\n",
        "import seaborn as sns\n",
        "\n",
        "model = make_pipeline(StandardScaler(), Lasso(alpha=.015))\n",
        "\n",
        "model.fit(X, y[0,:])\n",
        "\n",
        "coefs = pd.DataFrame(\n",
        "   model[1].coef_,\n",
        "   columns=['Coefficients'], index=labels\n",
        ")\n",
        "\n",
        "coefs.plot(kind='barh', figsize=(9, 7))\n",
        "plt.title('Lasso model, strong regularization')\n",
        "plt.axvline(x=0, color='.5')\n",
        "plt.subplots_adjust(left=.3)\n",
        "\n",
        "feature_importance = abs(model[1].coef_)\n",
        "\n",
        "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
        "sorted_idx = np.argsort(feature_importance)\n",
        "\n",
        "ranked = sorted_idx[::-1][:70]\n",
        "\n",
        "fi = feature_importance[ranked]\n",
        "\n",
        "print(labels[ranked])"
      ],
      "metadata": {
        "id": "f6yqAp8K6S6M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# logistic regression for feature importance (classification)\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from matplotlib import pyplot\n",
        "\n",
        "# Define the model\n",
        "model = LogisticRegression()\n",
        "\n",
        "# Fit the model\n",
        "model.fit(X, y[0,:])\n",
        "\n",
        "# Get importance\n",
        "importance = model.coef_[0]\n",
        "\n",
        "# Summarize feature importance\n",
        "feature_importance = abs(model.coef_[0])\n",
        "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
        "sorted_idx = np.argsort(feature_importance)\n",
        "pos = np.arange(sorted_idx.shape[0]) + .5\n",
        "\n",
        "ranked = sorted_idx[::-1][:70]\n",
        "\n",
        "fi = feature_importance[ranked]\n",
        "\n",
        "pyplot.bar([x for x in range(len(fi))], fi)\n",
        "pyplot.show()\n",
        "\n",
        "print(labels[ranked])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "_fir36z5vPNE",
        "outputId": "ac9d5b4a-dd4e-49fa-d4a2-b5f3841c813c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAO1klEQVR4nO3dYaxkdX3G8e9TVqqgZUFuNlsWe7eBQExTgd4gBmMstM0CBnhBCMTYrdlm32iL1USWNinpO0gakSYNyQbUbWJQilYINlq6Ypo26epdQAVWyhZBlizstQVpNGml/vpiDjre3mX3zpm7M/Pf7ye5mXPOnJl5spl97v/+55wzqSokSW35pUkHkCSNn+UuSQ2y3CWpQZa7JDXIcpekBq2bdACA008/vebn5ycdQ5Jmyt69e39QVXMr3TcV5T4/P8/i4uKkY0jSTEny7OHuc1pGkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNeiI5Z7kU0kOJXlsaNtpSR5M8lR3e2q3PUn+Ksn+JN9OcsFahpckrexoRu6fAbYs27YD2F1VZwO7u3WAy4Czu5/twB3jiSlJWo0jlntV/RPwn8s2XwXs6pZ3AVcPbf+bGvhXYH2SjeMKK0k6OqOeobqhqg52yy8AG7rlM4DnhvY70G07yDJJtjMY3fO2t71txBgwv+PLv7D+zC1XjPxcktSK3h+o1uCrnFb9dU5VtbOqFqpqYW5uxUsjSJJGNGq5v/jadEt3e6jb/jxw5tB+m7ptkqRjaNRyvx/Y2i1vBe4b2v773VEzFwE/HJq+kSQdI0ecc09yN/Be4PQkB4CbgVuAe5JsA54Fru12/3vgcmA/8GPgg2uQWZJ0BEcs96q6/jB3XbrCvgV8qG8oSVI/nqEqSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJalCvck/yJ0keT/JYkruTvDHJ5iR7kuxP8vkkJ44rrCTp6Ixc7knOAP4YWKiq3wBOAK4DbgVuq6qzgJeAbeMIKkk6en2nZdYBb0qyDjgJOAhcAtzb3b8LuLrna0iSVmnkcq+q54G/BL7PoNR/COwFXq6qV7vdDgBnrPT4JNuTLCZZXFpaGjWGJGkFfaZlTgWuAjYDvwqcDGw52sdX1c6qWqiqhbm5uVFjSJJW0Gda5neA71XVUlX9BPgicDGwvpumAdgEPN8zoyRplfqU+/eBi5KclCTApcATwEPANd0+W4H7+kWUJK1Wnzn3PQw+OH0Y+E73XDuBG4GPJtkPvBW4aww5JUmrsO7IuxxeVd0M3Lxs89PAhX2eV5LUj2eoSlKDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhrUq9yTrE9yb5LvJtmX5F1JTkvyYJKnuttTxxVWknR0+o7cbwe+UlXnAu8A9gE7gN1VdTawu1uXJB1DI5d7klOA9wB3AVTV/1TVy8BVwK5ut13A1X1DSpJWp8/IfTOwBHw6ySNJ7kxyMrChqg52+7wAbOgbUpK0On3KfR1wAXBHVZ0P/IhlUzBVVUCt9OAk25MsJllcWlrqEUOStFyfcj8AHKiqPd36vQzK/sUkGwG620MrPbiqdlbVQlUtzM3N9YghSVpu5HKvqheA55Kc0226FHgCuB/Y2m3bCtzXK6EkadXW9Xz8HwGfTXIi8DTwQQa/MO5Jsg14Fri252tIklapV7lX1aPAwgp3XdrneSVJ/fQduU+l+R1f/tnyM7dcMcEkkjQZXn5AkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoOavPzAcsOXIwAvSSCpfY7cJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQcfFGarLecaqpNY5cpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAYdl2eormT4rFXPWJU06yz3w/ASBZJmWe9pmSQnJHkkyQPd+uYke5LsT/L5JCf2jylJWo1xzLnfAOwbWr8VuK2qzgJeAraN4TUkSavQq9yTbAKuAO7s1gNcAtzb7bILuLrPa0iSVq/vyP2TwMeBn3brbwVerqpXu/UDwBkrPTDJ9iSLSRaXlpZ6xpAkDRu53JO8DzhUVXtHeXxV7ayqhapamJubGzWGJGkFfY6WuRi4MsnlwBuBXwFuB9YnWdeN3jcBz/ePKUlajZFH7lV1U1Vtqqp54Drga1X1fuAh4Jput63Afb1TSpJWZS3OUL0R+GiS/Qzm4O9ag9eQJL2OsZzEVFVfB77eLT8NXDiO55UkjcZry0hSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ3yC7KP0kpfmD28zS/QljRNHLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBnko5BitdLikJE2C5b7GPBZe0iRY7seYZS/pWHDOXZIaZLlLUoMsd0lqkOUuSQ3yA9UJ8/BJSWvBkbskNchyl6QGWe6S1CDLXZIa5AeqU8izWCX1ZbnPgOVl7xE2ko5k5GmZJGcmeSjJE0keT3JDt/20JA8meaq7PXV8cSVJR6PPyP1V4GNV9XCStwB7kzwI/AGwu6puSbID2AHc2D+qXo9TOZKGjTxyr6qDVfVwt/xfwD7gDOAqYFe32y7g6r4hJUmrM5ajZZLMA+cDe4ANVXWwu+sFYMNhHrM9yWKSxaWlpXHEkCR1epd7kjcDXwA+UlWvDN9XVQXUSo+rqp1VtVBVC3Nzc31jSJKG9DpaJskbGBT7Z6vqi93mF5NsrKqDSTYCh/qG1OotP6JmOeflpbb1OVomwF3Avqr6xNBd9wNbu+WtwH2jx5MkjaLPyP1i4APAd5I82m37U+AW4J4k24BngWv7RdRa8QgbqV0jl3tV/TOQw9x96ajPK0nqzzNU9TOe+Sq1w3LX63LqRppNXhVSkhpkuUtSgyx3SWqQc+5aFefgpdngyF2SGmS5S1KDnJZRLysdG+/UjTR5jtwlqUGO3LXmHMlLx54jd0lqkCN3HXNHM0/v9eilfix3zSyne6TDs9zVDEf70s855y5JDXLkruPKkeb2Hd2rFY7cJalBjtylZV5vdO9oX7PCcpfGwF8ImjaWuzQhHsqptWS5S1PC0b3GyXKXptQoZ/L6C0GvsdylxjjdI/BQSElqkiN3qXGO5I9Plrt0nPHbs44PTstIUoMcuUv6fzwqZ/ZZ7pJG4i+A6ea0jCQ1yJG7pDUxrq9T9C+E0VjukmbekS7cdjxak3JPsgW4HTgBuLOqblmL15Gko3E8Hv459nJPcgLw18DvAgeAbya5v6qeGPdrSdK4HM30zyxNEa3FyP1CYH9VPQ2Q5HPAVYDlLqlp0/QXQqpqvE+YXANsqao/7NY/ALyzqj68bL/twPZu9RzgyZ4vfTrwg57PcSzNUt5ZygqzlXeWsoJ519IoWX+tquZWumNiH6hW1U5g57ieL8liVS2M6/nW2izlnaWsMFt5ZykrmHctjTvrWhzn/jxw5tD6pm6bJOkYWYty/yZwdpLNSU4ErgPuX4PXkSQdxtinZarq1SQfBr7K4FDIT1XV4+N+nRWMbYrnGJmlvLOUFWYr7yxlBfOupbFmHfsHqpKkyfPaMpLUIMtdkhrURLkn2ZLkyST7k+yYdJ5hST6V5FCSx4a2nZbkwSRPdbenTjLjsCRnJnkoyRNJHk9yQ7d96jIneWOSbyT5Vpf1L7rtm5Ps6d4Pn+8+2J8aSU5I8kiSB7r1qc2b5Jkk30nyaJLFbtvUvRcAkqxPcm+S7ybZl+RdU5z1nO7f9LWfV5J8ZJx5Z77chy53cBnwduD6JG+fbKpf8Blgy7JtO4DdVXU2sLtbnxavAh+rqrcDFwEf6v49pzHzfwOXVNU7gPOALUkuAm4Fbquqs4CXgG0TzLiSG4B9Q+vTnve3q+q8oWOwp/G9AIPrWX2lqs4F3sHg33gqs1bVk92/6XnAbwE/Bv6Oceatqpn+Ad4FfHVo/SbgpknnWpZxHnhsaP1JYGO3vBF4ctIZXyf7fQyuEzTVmYGTgIeBdzI4y2/dSu+PSf8wOO9jN3AJ8ACQKc/7DHD6sm1T914ATgG+R3eQyDRnXSH77wH/Mu68Mz9yB84AnhtaP9Btm2Ybqupgt/wCsGGSYQ4nyTxwPrCHKc3cTXE8ChwCHgT+HXi5ql7tdpm298MngY8DP+3W38p05y3gH5Ls7S4ZAtP5XtgMLAGf7qa87kxyMtOZdbnrgLu75bHlbaHcZ1oNfkVP3fGoSd4MfAH4SFW9MnzfNGWuqv+twZ+2mxhctO7cCUc6rCTvAw5V1d5JZ1mFd1fVBQymPT+U5D3Dd07Re2EdcAFwR1WdD/yIZVMaU5T1Z7rPV64E/nb5fX3ztlDus3i5gxeTbATobg9NOM8vSPIGBsX+2ar6Yrd5qjNX1cvAQwymNdYnee0EvWl6P1wMXJnkGeBzDKZmbmd681JVz3e3hxjMCV/IdL4XDgAHqmpPt34vg7KfxqzDLgMerqoXu/Wx5W2h3Gfxcgf3A1u75a0M5rWnQpIAdwH7quoTQ3dNXeYkc0nWd8tvYvDZwD4GJX9Nt9tUZAWoqpuqalNVzTN4n36tqt7PlOZNcnKSt7y2zGBu+DGm8L1QVS8AzyU5p9t0KYPLjE9d1mWu5+dTMjDOvJP+MGFMH0hcDvwbg/nWP5t0nmXZ7gYOAj9hMLrYxmCedTfwFPCPwGmTzjmU990M/hT8NvBo93P5NGYGfhN4pMv6GPDn3fZfB74B7Gfw5+4vTzrrCtnfCzwwzXm7XN/qfh5/7f/WNL4XulznAYvd++FLwKnTmrXLezLwH8ApQ9vGltfLD0hSg1qYlpEkLWO5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAb9H640q8s/uX0HAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['uc.8+']\n",
            " ['uc.283+A']\n",
            " ['uc.160+']\n",
            " ['uc.145+A']\n",
            " ['uc.339+']\n",
            " ['uc.369+']\n",
            " ['uc.325+']\n",
            " ['uc.466+A']\n",
            " ['uc.3+']\n",
            " ['uc.263+A']\n",
            " ['uc.346+']\n",
            " ['uc.229+A']\n",
            " ['uc.282+A']\n",
            " ['uc.213+A']\n",
            " ['uc.142+A']\n",
            " ['uc.450+']\n",
            " ['uc.230+']\n",
            " ['uc.287+A']\n",
            " ['uc.363+A']\n",
            " ['uc.288+A']\n",
            " ['uc.20+']\n",
            " ['uc.217+A']\n",
            " ['uc.238+']\n",
            " ['uc.473+A']\n",
            " ['uc.195+']\n",
            " ['uc.362+A']\n",
            " ['uc.462+A']\n",
            " ['uc.469+A']\n",
            " ['uc.31+']\n",
            " ['uc.378+A']\n",
            " ['uc.248+A']\n",
            " ['uc.477+A']\n",
            " ['uc.33+']\n",
            " ['uc.153+A']\n",
            " ['uc.342+']\n",
            " ['uc.325+A']\n",
            " ['uc.448+A']\n",
            " ['uc.392+A']\n",
            " ['uc.354+A']\n",
            " ['uc.374+A']\n",
            " ['uc.163+']\n",
            " ['uc.48+A']\n",
            " ['uc.177+A']\n",
            " ['uc.34+A']\n",
            " ['uc.298+']\n",
            " ['uc.106+A']\n",
            " ['uc.200+']\n",
            " ['uc.96+']\n",
            " ['uc.190+A']\n",
            " ['uc.372+']\n",
            " ['uc.206+A']\n",
            " ['uc.43+']\n",
            " ['uc.468+']\n",
            " ['uc.138+']\n",
            " ['uc.44+A']\n",
            " ['uc.412+A']\n",
            " ['uc.275+A']\n",
            " ['uc.456+A']\n",
            " ['uc.404+']\n",
            " ['uc.189+']\n",
            " ['uc.268+']\n",
            " ['uc.28+']\n",
            " ['uc.4+']\n",
            " ['uc.457+']\n",
            " ['uc.388+A']\n",
            " ['uc.21+A']\n",
            " ['uc.234+']\n",
            " ['uc.445+A']\n",
            " ['uc.89+']\n",
            " ['uc.16+']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Random Forest for feature importance on a classification problem\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from matplotlib import pyplot\n",
        "\n",
        "# Define the model\n",
        "model = RandomForestClassifier()\n",
        "\n",
        "# Fit the model\n",
        "model.fit(X, y[0,:].T)\n",
        "\n",
        "# Get importance\n",
        "importance = model.feature_importances_\n",
        "\n",
        "# Summarize feature importance\n",
        "feature_importance = abs(model.feature_importances_)\n",
        "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
        "sorted_idx = np.argsort(feature_importance)\n",
        "\n",
        "ranked = sorted_idx[::-1][:70]\n",
        "\n",
        "fi = feature_importance[ranked]\n",
        "\n",
        "pyplot.bar([x for x in range(len(fi))], fi)\n",
        "pyplot.show()\n",
        "\n",
        "print(labels[ranked])"
      ],
      "metadata": {
        "id": "ZF4FVBNqvkPD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "2bbb4ec2-e792-405c-f169-e6872fbad287"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOv0lEQVR4nO3df6zddX3H8edrVIbiZkFumo7S3S42ELJMYDeIwRgH21LBCH8QgzGuMV36D24wTbRsycj+g2RRWbKQNIB2iUEdukHQ6FjFLFuyagsohcrosEhJoXUDXTTZZL73x/nCDje3tPd8z73nnE+fj+TmfH+dc15pvn2d7/18v+d7U1VIktryS5MOIEkaP8tdkhpkuUtSgyx3SWqQ5S5JDVoz6QAA55xzTs3Pz086hiTNlH379v2oquaWWjcV5T4/P8/evXsnHUOSZkqSZ463zmEZSWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1KATlnuSu5McTbJ/aNnZSR5M8lT3eFa3PEn+KsnBJN9LcslKhpckLe1kjtw/B2xZtGwHsLuqNgO7u3mA9wKbu5/twB3jiSlJWo4TlntV/RPwn4sWXwPs6qZ3AdcOLf+bGvhXYG2S9eMKK0k6OaN+Q3VdVR3ppp8H1nXT5wLPDm13uFt2hEWSbGdwdM/GjRtHjAHzO776mvlDt1498mtJUit6n1CtwZ9yWvafc6qqnVW1UFULc3NL3hpBkjSiUcv9hVeGW7rHo93y54Dzhrbb0C2TJK2iUcv9fmBrN70VuG9o+R90V81cBvx4aPhGkrRKTjjmnuQe4D3AOUkOA7cAtwJfSrINeAb4QLf514CrgIPAz4CPrEBmSdIJnLDcq+qDx1l15RLbFnBD31CSpH78hqokNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQb3KPcmfJHk8yf4k9yQ5I8mmJHuSHEzyxSSnjyusJOnkjFzuSc4F/hhYqKrfBE4DrgduAz5dVW8DXgS2jSOoJOnk9R2WWQO8Mcka4E3AEeAK4N5u/S7g2p7vIUlappHLvaqeA/4S+CGDUv8xsA94qape7jY7DJy71POTbE+yN8neY8eOjRpDkrSEPsMyZwHXAJuAXwPOBLac7POramdVLVTVwtzc3KgxJElL6DMs87vAD6rqWFX9HPgKcDmwthumAdgAPNczoyRpmfqU+w+By5K8KUmAK4EngIeA67pttgL39YsoSVquPmPuexicOH0YeKx7rZ3AJ4GPJTkIvBW4aww5JUnLsObEmxxfVd0C3LJo8dPApX1eV5LUj99QlaQGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDWoV7knWZvk3iTfT3IgyTuTnJ3kwSRPdY9njSusJOnk9D1yvx34elVdALwdOADsAHZX1WZgdzcvSVpFI5d7krcA7wbuAqiq/6mql4BrgF3dZruAa/uGlCQtT58j903AMeCzSR5JcmeSM4F1VXWk2+Z5YF3fkJKk5elT7muAS4A7qupi4KcsGoKpqgJqqScn2Z5kb5K9x44d6xFDkrRYn3I/DByuqj3d/L0Myv6FJOsBusejSz25qnZW1UJVLczNzfWIIUlabORyr6rngWeTnN8tuhJ4Argf2Not2wrc1yuhJGnZ1vR8/h8Bn09yOvA08BEGHxhfSrINeAb4QM/3kCQtU69yr6pHgYUlVl3Z53UlSf34DVVJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBfW8cNpXmd3z11elDt149wSSSNBkeuUtSgyx3SWqQ5S5JDbLcJalBTZ5QXWz4BCt4klVS+06Jcl/MspfUulOy3Jfi5ZOSWuKYuyQ1yCP343DoRtIs88hdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ3qXe5JTkvySJIHuvlNSfYkOZjki0lO7x9TkrQc4zhyvxE4MDR/G/Dpqnob8CKwbQzvIUlahl7lnmQDcDVwZzcf4Arg3m6TXcC1fd5DkrR8fY/cPwN8AvhFN/9W4KWqermbPwycu9QTk2xPsjfJ3mPHjvWMIUkaNnK5J3kfcLSq9o3y/KraWVULVbUwNzc3agxJ0hL6/CWmy4H3J7kKOAP4VeB2YG2SNd3R+wbguf4xJUnLMfKRe1XdXFUbqmoeuB74ZlV9CHgIuK7bbCtwX++UkqRlWYnr3D8JfCzJQQZj8HetwHtIkl7HWP5AdlV9C/hWN/00cOk4XleSNJqxlPupYH7HV18zf+jWqyeURJJOzNsPSFKDPHLvYfho3iN5SdPEI3dJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoP8Yx1j5J/ikzQtPHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoNGLvck5yV5KMkTSR5PcmO3/OwkDyZ5qns8a3xxJUkno8+R+8vAx6vqQuAy4IYkFwI7gN1VtRnY3c1LklbRyOVeVUeq6uFu+r+AA8C5wDXArm6zXcC1fUNKkpZnLPdzTzIPXAzsAdZV1ZFu1fPAuuM8ZzuwHWDjxo3jiDGVhu/x7v3dJa2W3uWe5M3Al4GbquonSV5dV1WVpJZ6XlXtBHYCLCwsLLlNiyx7Sauh19UySd7AoNg/X1Vf6Ra/kGR9t349cLRfREnScvW5WibAXcCBqvrU0Kr7ga3d9FbgvtHjSZJG0WdY5nLgw8BjSR7tlv0pcCvwpSTbgGeAD/SLKElarpHLvar+GchxVl856utKkvrzG6qS1CDLXZIaZLlLUoPG8iUmjW74unfw2ndJ42G5TyG/6CSpL4dlJKlBlrskNchyl6QGWe6S1CDLXZIa5NUyM8CrZyQtl+U+g7w2XtKJWO6NWHx07weAdGpzzF2SGuSR+ynCI3np1GK5n8I8USu1y3LXq5Y6uvcDQJpNjrlLUoM8cteyeCQvzQbLXb14olaaTpa7xu71rrk/3jX4XqcvjZflrpngyV5peSx3NWPxB8BifgDoVGK565QyyhCRNIssd+kE+p4z8ANCk2C5S6vA3xC02vwSkyQ1yCN3aUp4+afGyXKXppRDOerDcpdmmB8AOh7LXWqc3/49NVnukryUs0FeLSNJDVqRI/ckW4DbgdOAO6vq1pV4H0mrY5SbvS3mbwSra+zlnuQ04K+B3wMOA99Jcn9VPTHu95I020b5QPAk8slZiSP3S4GDVfU0QJIvANcAlrukVeEtIyBVNd4XTK4DtlTVH3bzHwbeUVUfXbTddmB7N3s+8GTPtz4H+FHP11hNs5R3lrLCbOWdpaxg3pU0StZfr6q5pVZM7GqZqtoJ7BzX6yXZW1UL43q9lTZLeWcpK8xW3lnKCuZdSePOuhJXyzwHnDc0v6FbJklaJStR7t8BNifZlOR04Hrg/hV4H0nScYx9WKaqXk7yUeAbDC6FvLuqHh/3+yxhbEM8q2SW8s5SVpitvLOUFcy7ksaadewnVCVJk+c3VCWpQZa7JDWoiXJPsiXJk0kOJtkx6TzDktyd5GiS/UPLzk7yYJKnusezJplxWJLzkjyU5Ikkjye5sVs+dZmTnJHk20m+22X9i275piR7uv3hi92J/amR5LQkjyR5oJuf2rxJDiV5LMmjSfZ2y6ZuXwBIsjbJvUm+n+RAkndOcdbzu3/TV35+kuSmcead+XIfut3Be4ELgQ8muXCyqV7jc8CWRct2ALurajOwu5ufFi8DH6+qC4HLgBu6f89pzPzfwBVV9XbgImBLksuA24BPV9XbgBeBbRPMuJQbgQND89Oe93eq6qKha7CncV+Awf2svl5VFwBvZ/BvPJVZq+rJ7t/0IuC3gZ8Bf8c481bVTP8A7wS+MTR/M3DzpHMtyjgP7B+afxJY302vB56cdMbXyX4fg/sETXVm4E3Aw8A7GHzLb81S+8ekfxh872M3cAXwAJApz3sIOGfRsqnbF4C3AD+gu0hkmrMukf33gX8Zd96ZP3IHzgWeHZo/3C2bZuuq6kg3/TywbpJhjifJPHAxsIcpzdwNcTwKHAUeBP4deKmqXu42mbb94TPAJ4BfdPNvZbrzFvAPSfZ1twyB6dwXNgHHgM92Q153JjmT6cy62PXAPd302PK2UO4zrQYf0VN3PWqSNwNfBm6qqp8Mr5umzFX1vzX41XYDg5vWXTDhSMeV5H3A0araN+ksy/CuqrqEwbDnDUnePbxyivaFNcAlwB1VdTHwUxYNaUxR1ld151feD/zt4nV987ZQ7rN4u4MXkqwH6B6PTjjPayR5A4Ni/3xVfaVbPNWZq+ol4CEGwxprk7zyBb1p2h8uB96f5BDwBQZDM7czvXmpque6x6MMxoQvZTr3hcPA4ara083fy6DspzHrsPcCD1fVC9382PK2UO6zeLuD+4Gt3fRWBuPaUyFJgLuAA1X1qaFVU5c5yVyStd30GxmcGzjAoOSv6zabiqwAVXVzVW2oqnkG++k3q+pDTGneJGcm+ZVXphmMDe9nCveFqnoeeDbJ+d2iKxncZnzqsi7yQf5/SAbGmXfSJxPGdELiKuDfGIy3/tmk8yzKdg9wBPg5g6OLbQzGWXcDTwH/CJw96ZxDed/F4FfB7wGPdj9XTWNm4LeAR7qs+4E/75b/BvBt4CCDX3d/edJZl8j+HuCBac7b5fpu9/P4K/+3pnFf6HJdBOzt9oe/B86a1qxd3jOB/wDeMrRsbHm9/YAkNaiFYRlJ0iKWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWrQ/wG8J5Hr6soJHAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['uc.325+A']\n",
            " ['uc.466+A']\n",
            " ['uc.346+']\n",
            " ['uc.369+']\n",
            " ['uc.354+A']\n",
            " ['uc.8+']\n",
            " ['uc.283+A']\n",
            " ['uc.153+A']\n",
            " ['uc.217+A']\n",
            " ['uc.145+A']\n",
            " ['uc.362+A']\n",
            " ['uc.160+']\n",
            " ['uc.287+A']\n",
            " ['uc.363+A']\n",
            " ['uc.177+A']\n",
            " ['uc.33+']\n",
            " ['uc.263+A']\n",
            " ['uc.325+']\n",
            " ['uc.462+A']\n",
            " ['uc.229+A']\n",
            " ['uc.404+']\n",
            " ['uc.282+A']\n",
            " ['uc.106+A']\n",
            " ['uc.3+']\n",
            " ['uc.445+A']\n",
            " ['uc.20+']\n",
            " ['uc.473+A']\n",
            " ['uc.16+']\n",
            " ['uc.392+A']\n",
            " ['uc.48+A']\n",
            " ['uc.457+']\n",
            " ['uc.89+']\n",
            " ['uc.268+']\n",
            " ['uc.4+']\n",
            " ['uc.388+A']\n",
            " ['uc.469+A']\n",
            " ['uc.138+']\n",
            " ['uc.477+A']\n",
            " ['uc.288+A']\n",
            " ['uc.468+']\n",
            " ['uc.298+']\n",
            " ['uc.238+']\n",
            " ['uc.163+']\n",
            " ['uc.450+']\n",
            " ['uc.374+A']\n",
            " ['uc.31+']\n",
            " ['uc.372+']\n",
            " ['uc.96+']\n",
            " ['uc.213+A']\n",
            " ['uc.206+A']\n",
            " ['uc.21+A']\n",
            " ['uc.248+A']\n",
            " ['uc.200+']\n",
            " ['uc.34+A']\n",
            " ['uc.189+']\n",
            " ['uc.412+A']\n",
            " ['uc.28+']\n",
            " ['uc.190+A']\n",
            " ['uc.378+A']\n",
            " ['uc.339+']\n",
            " ['uc.44+A']\n",
            " ['uc.234+']\n",
            " ['uc.342+']\n",
            " ['uc.142+A']\n",
            " ['uc.456+A']\n",
            " ['uc.230+']\n",
            " ['uc.195+']\n",
            " ['uc.448+A']\n",
            " ['uc.275+A']\n",
            " ['uc.43+']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# xgboost for feature importance on a classification problem\n",
        "from sklearn.datasets import make_classification\n",
        "from xgboost import XGBClassifier\n",
        "from matplotlib import pyplot\n",
        "\n",
        "# Define the model\n",
        "model = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
        "       colsample_bynode=1, colsample_bytree=1, gamma=0, learning_rate=0.1,\n",
        "       max_delta_step=0, max_depth=3, min_child_weight=1, missing=None,\n",
        "       n_estimators=100, n_jobs=1, nthread=None, random_state=0, reg_alpha=0,\n",
        "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=None,\n",
        "       subsample=1, verbosity=1)\n",
        "\n",
        "# Fit the model\n",
        "model.fit(X, y[0,:].T)\n",
        "\n",
        "# Get importance\n",
        "importance = model.feature_importances_\n",
        "\n",
        "# Summarize feature importance\n",
        "\n",
        "feature_importance = abs(model.feature_importances_)\n",
        "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
        "sorted_idx = np.argsort(feature_importance)\n",
        "\n",
        "ranked = sorted_idx[::-1][:70]\n",
        "\n",
        "fi = feature_importance[ranked]\n",
        "\n",
        "pyplot.bar([x for x in range(len(fi))], fi)\n",
        "pyplot.show()\n",
        "\n",
        "print(labels[ranked])"
      ],
      "metadata": {
        "id": "lRsl4mSwvxqP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "652e4868-ae59-4dc4-e62c-c7ca6ed2be30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOXElEQVR4nO3df6zddX3H8edrVKbiRkFumq6F3S4QCFnGj90gBmMY3ZYCRviDGIhxnenSf3CDaaJlS2b2X00WlSULSQNolxAGq2wQWHSsYpYtWfUWUAuV0WGRkkKvm+iiyZT53h/nCzvc3Erv+Z7bc+7H5yM5Od/v5/s95/vKzbev+72f86OpKiRJbfmFSQeQJI2f5S5JDbLcJalBlrskNchyl6QGrZl0AICzzjqrZmdnJx1DklaV/fv3f7eqZpbaNhXlPjs7y/z8/KRjSNKqkuT5421zWkaSGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ16E3LPcndSY4lOTA0dmaSR5M8292f0Y0nyV8mOZTkG0kuXcnwkqSlnciV++eBLYvGdgB7q+o8YG+3DnA1cF532w7cMZ6YkqTleNNyr6p/Bv5r0fB1wO5ueTdw/dD4X9fAvwFrk6wfV1hJ0okZ9ROq66rqaLf8ErCuW94AvDC035Fu7CiLJNnO4Oqec845Z8QYMLvjkTesH9557cjPJUmt6P2Cag3+K6dl/3dOVbWrquaqam5mZsmvRpAkjWjUcn/5temW7v5YN/4icPbQfhu7MUnSSTRquT8EbO2WtwIPDo3/XveumcuB7w9N30iSTpI3nXNPci9wJXBWkiPAJ4GdwP1JtgHPAx/odv8H4BrgEPAj4MMrkFmS9CbetNyr6qbjbNq8xL4F3Nw3lCSpHz+hKkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWpQr3JP8sdJnkpyIMm9Sd6aZFOSfUkOJbkvyanjCitJOjEjl3uSDcAfAXNV9evAKcCNwKeAz1TVucD3gG3jCCpJOnF9p2XWAG9LsgZ4O3AUuArY023fDVzf8xiSpGUaudyr6kXgL4DvMCj17wP7gVeq6tVutyPAhqUen2R7kvkk8wsLC6PGkCQtoc+0zBnAdcAm4FeA04AtJ/r4qtpVVXNVNTczMzNqDEnSEvpMy/w28O2qWqiqnwAPAFcAa7tpGoCNwIs9M0qSlqlPuX8HuDzJ25ME2Aw8DTwG3NDtsxV4sF9ESdJy9Zlz38fghdPHgW92z7UL+ATw0SSHgHcCd40hpyRpGda8+S7HV1WfBD65aPg54LI+zytJ6sdPqEpSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIa1Kvck6xNsifJt5IcTPLuJGcmeTTJs939GeMKK0k6MX2v3G8HvlhVFwAXAQeBHcDeqjoP2NutS5JOojWjPjDJ6cB7gd8HqKofAz9Och1wZbfbbuArwCf6hFyu2R2PvL58eOe1J/PQkjQV+ly5bwIWgM8leSLJnUlOA9ZV1dFun5eAdX1DSpKWp0+5rwEuBe6oqkuAH7JoCqaqCqilHpxke5L5JPMLCws9YkiSFutT7keAI1W1r1vfw6DsX06yHqC7P7bUg6tqV1XNVdXczMxMjxiSpMVGLveqegl4Icn53dBm4GngIWBrN7YVeLBXQknSso38gmrnD4F7kpwKPAd8mMEvjPuTbAOeBz7Q8xiSpGXqVe5V9SQwt8SmzX2eV5LUj59QlaQGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBvUu9ySnJHkiycPd+qYk+5IcSnJfklP7x5QkLcc4rtxvAQ4OrX8K+ExVnQt8D9g2hmNIkpahV7kn2QhcC9zZrQe4CtjT7bIbuL7PMSRJy9f3yv2zwMeBn3br7wReqapXu/UjwIalHphke5L5JPMLCws9Y0iSho1c7kneBxyrqv2jPL6qdlXVXFXNzczMjBpDkrSENT0eewXw/iTXAG8Ffhm4HVibZE139b4ReLF/TEnScox85V5Vt1XVxqqaBW4EvlxVHwQeA27odtsKPNg7pSRpWVbife6fAD6a5BCDOfi7VuAYkqSfoc+0zOuq6ivAV7rl54DLxvG8kqTR+AlVSWqQ5S5JDbLcJalBY5lzn3azOx55w/rhnddOKIkknRxeuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBI5d7krOTPJbk6SRPJbmlGz8zyaNJnu3uzxhfXEnSiehz5f4q8LGquhC4HLg5yYXADmBvVZ0H7O3WJUkn0cjlXlVHq+rxbvm/gYPABuA6YHe3227g+r4hJUnLM5Y59ySzwCXAPmBdVR3tNr0ErDvOY7YnmU8yv7CwMI4YkqRO73JP8g7gC8CtVfWD4W1VVUAt9biq2lVVc1U1NzMz0zeGJGlIr3JP8hYGxX5PVT3QDb+cZH23fT1wrF9ESdJy9Xm3TIC7gINV9emhTQ8BW7vlrcCDo8eTJI1iTY/HXgF8CPhmkie7sT8BdgL3J9kGPA98oF9ESdJyjVzuVfUvQI6zefOozytJ6s9PqEpSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQX2+FXLVmt3xyBvWD++8dkJJJGlleOUuSQ2y3CWpQZa7JDXo53LOfSnD8/CHd17rvLykVc0rd0lqkOUuSQ2y3CWpQZa7JDXIcpekBvlumRPku2ckrSZeuUtSg7xy72Hxe+MlaVp45S5JDfLKfYycl5c0Lbxyl6QGWe6S1CCnZVaYL7pKmgSv3CWpQV65n2ReyUs6GSz3CfMdNpJWwoqUe5ItwO3AKcCdVbVzJY7Tqp/1H4dY/pJOxNjLPckpwF8BvwMcAb6W5KGqenrcx/p5tdTVvr8AJA1biRdULwMOVdVzVfVj4G+A61bgOJKk40hVjfcJkxuALVX1B936h4B3VdVHFu23HdjerZ4PPNPz0GcB3+35HCfTasq7mrLC6sq7mrKCeVfSKFl/tapmltowsRdUq2oXsGtcz5dkvqrmxvV8K2015V1NWWF15V1NWcG8K2ncWVdiWuZF4Oyh9Y3dmCTpJFmJcv8acF6STUlOBW4EHlqB40iSjmPs0zJV9WqSjwBfYvBWyLur6qlxH2cJY5viOUlWU97VlBVWV97VlBXMu5LGmnXsL6hKkibP75aRpAZZ7pLUoCbKPcmWJM8kOZRkx6TzDEtyd5JjSQ4MjZ2Z5NEkz3b3Z0wy47AkZyd5LMnTSZ5Kcks3PnWZk7w1yVeTfL3L+ufd+KYk+7rz4b7uhf2pkeSUJE8kebhbn9q8SQ4n+WaSJ5PMd2NTdy4AJFmbZE+SbyU5mOTdU5z1/O5n+trtB0luHWfeVV/uQ193cDVwIXBTkgsnm+oNPg9sWTS2A9hbVecBe7v1afEq8LGquhC4HLi5+3lOY+b/Aa6qqouAi4EtSS4HPgV8pqrOBb4HbJtgxqXcAhwcWp/2vL9VVRcPvQd7Gs8FGHyf1Rer6gLgIgY/46nMWlXPdD/Ti4HfBH4E/B3jzFtVq/oGvBv40tD6bcBtk861KOMscGBo/Rlgfbe8Hnhm0hl/RvYHGXxP0FRnBt4OPA68i8Gn/NYsdX5M+sbgcx97gauAh4FMed7DwFmLxqbuXABOB75N9yaRac66RPbfBf513HlX/ZU7sAF4YWj9SDc2zdZV1dFu+SVg3STDHE+SWeASYB9Tmrmb4ngSOAY8CvwH8EpVvdrtMm3nw2eBjwM/7dbfyXTnLeAfk+zvvjIEpvNc2AQsAJ/rprzuTHIa05l1sRuBe7vlseVtodxXtRr8ip6696MmeQfwBeDWqvrB8LZpylxV/1uDP203MvjSugsmHOm4krwPOFZV+yedZRneU1WXMpj2vDnJe4c3TtG5sAa4FLijqi4BfsiiKY0pyvq67vWV9wN/u3hb37wtlPtq/LqDl5OsB+juj004zxskeQuDYr+nqh7ohqc6c1W9AjzGYFpjbZLXPqA3TefDFcD7kxxm8G2pVzGYJ57WvFTVi939MQZzwpcxnefCEeBIVe3r1vcwKPtpzDrsauDxqnq5Wx9b3hbKfTV+3cFDwNZueSuDee2pkCTAXcDBqvr00Kapy5xkJsnabvltDF4bOMig5G/odpuKrABVdVtVbayqWQbn6Zer6oNMad4kpyX5pdeWGcwNH2AKz4Wqegl4Icn53dBm4GmmMOsiN/H/UzIwzryTfjFhTC9IXAP8O4P51j+ddJ5F2e4FjgI/YXB1sY3BPOte4Fngn4AzJ51zKO97GPwp+A3gye52zTRmBn4DeKLLegD4s27814CvAocY/Ln7i5POukT2K4GHpzlvl+vr3e2p1/5tTeO50OW6GJjvzoe/B86Y1qxd3tOA/wROHxobW16/fkCSGtTCtIwkaRHLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXo/wBC3xM7zmKpxwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['uc.346+']\n",
            " ['uc.354+A']\n",
            " ['uc.369+']\n",
            " ['uc.3+']\n",
            " ['uc.8+']\n",
            " ['uc.325+A']\n",
            " ['uc.362+A']\n",
            " ['uc.298+']\n",
            " ['uc.392+A']\n",
            " ['uc.268+']\n",
            " ['uc.195+']\n",
            " ['uc.238+']\n",
            " ['uc.450+']\n",
            " ['uc.466+A']\n",
            " ['uc.404+']\n",
            " ['uc.213+A']\n",
            " ['uc.163+']\n",
            " ['uc.275+A']\n",
            " ['uc.325+']\n",
            " ['uc.469+A']\n",
            " ['uc.445+A']\n",
            " ['uc.190+A']\n",
            " ['uc.189+']\n",
            " ['uc.142+A']\n",
            " ['uc.229+A']\n",
            " ['uc.43+']\n",
            " ['uc.234+']\n",
            " ['uc.378+A']\n",
            " ['uc.473+A']\n",
            " ['uc.230+']\n",
            " ['uc.374+A']\n",
            " ['uc.4+']\n",
            " ['uc.282+A']\n",
            " ['uc.21+A']\n",
            " ['uc.138+']\n",
            " ['uc.206+A']\n",
            " ['uc.342+']\n",
            " ['uc.462+A']\n",
            " ['uc.106+A']\n",
            " ['uc.28+']\n",
            " ['uc.477+A']\n",
            " ['uc.217+A']\n",
            " ['uc.363+A']\n",
            " ['uc.287+A']\n",
            " ['uc.283+A']\n",
            " ['uc.263+A']\n",
            " ['uc.339+']\n",
            " ['uc.96+']\n",
            " ['uc.200+']\n",
            " ['uc.468+']\n",
            " ['uc.31+']\n",
            " ['uc.288+A']\n",
            " ['uc.456+A']\n",
            " ['uc.448+A']\n",
            " ['uc.48+A']\n",
            " ['uc.44+A']\n",
            " ['uc.160+']\n",
            " ['uc.248+A']\n",
            " ['uc.372+']\n",
            " ['uc.412+A']\n",
            " ['uc.388+A']\n",
            " ['uc.145+A']\n",
            " ['uc.457+']\n",
            " ['uc.89+']\n",
            " ['uc.20+']\n",
            " ['uc.16+']\n",
            " ['uc.177+A']\n",
            " ['uc.153+A']\n",
            " ['uc.33+']\n",
            " ['uc.34+A']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Permutation feature importance with knn for classification\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.inspection import permutation_importance\n",
        "from matplotlib import pyplot\n",
        "\n",
        "# Define the model\n",
        "model = KNeighborsClassifier()\n",
        "\n",
        "# Fit the model\n",
        "model.fit(X, y[0,:])\n",
        "\n",
        "# Perform permutation importance\n",
        "results = permutation_importance(model, X, y[0,:], scoring='accuracy')\n",
        "\n",
        "# Summarize feature importance\n",
        "feature_importance = abs(results.importances_mean)\n",
        "feature_importance = 100.0 * (feature_importance / feature_importance.max())\n",
        "sorted_idx = np.argsort(feature_importance)\n",
        "\n",
        "ranked = sorted_idx[::-1][:10]\n",
        "\n",
        "fi = feature_importance[ranked]\n",
        "\n",
        "pyplot.bar([x for x in range(len(fi))], fi)\n",
        "pyplot.show()\n",
        "\n",
        "print(labels[ranked])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "iQMFCRirv7su",
        "outputId": "11260e12-6c07-48c4-e661-c280d8fe42ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMLklEQVR4nO3dX4hmhXnH8e+vTsRoaNQ4LGZXuwuRBAkEZbCmQihuLkwM0YsghjZdRNgbm5g/kJjceKsQklgowqJJt1RsZCMoaUgrG0PJRZfOqtQ/m+Biou52dSc0mjS9MJKnF3MswzLT3XnPzLzxeb8fkHnPec95z/Pq7tezZ+Y9m6pCktTLH017AEnSxjPuktSQcZekhoy7JDVk3CWpoblpDwBw0UUX1c6dO6c9hiS9rRw+fPiXVTW/2nN/EHHfuXMni4uL0x5Dkt5Wkry41nNelpGkhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkOnjXuSbyc5meSZFesuTPJYkueHrxcM65Pkb5IcTfIfSa7czOElSas7kzP3vwOuO2XdHcDBqroMODgsA3wMuGz4Zy9w78aMKUlaj9PGvar+FfivU1bfAOwfHu8Hblyx/u9r2b8B5ye5eKOGlSSdmUk/obqtqk4Mj18Btg2PtwMvr9ju2LDuBKdIspfls3suvfTSCceAnXf808T7nqlf3HX9ph9DkjbS6G+o1vJf5bTuv86pqvZV1UJVLczPr3prBEnShCaN+6tvXW4Zvp4c1h8HLlmx3Y5hnSRpC00a90eBPcPjPcAjK9b/1fBTM1cDr6+4fCNJ2iKnveae5EHgz4GLkhwD7gTuAh5KcivwInDTsPkPgI8DR4H/AW7ZhJklSadx2rhX1afXeGr3KtsWcNvYoSRJ4/gJVUlqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDY2Ke5IvJHk2yTNJHkxyTpJdSQ4lOZrku0nO3qhhJUlnZuK4J9kOfA5YqKoPAmcBNwN3A9+sqvcBvwJu3YhBJUlnbuxlmTngnUnmgHOBE8C1wIHh+f3AjSOPIUlap4njXlXHga8DL7Ec9deBw8BrVfXmsNkxYPtq+yfZm2QxyeLS0tKkY0iSVjHmsswFwA3ALuC9wHnAdWe6f1Xtq6qFqlqYn5+fdAxJ0irGXJb5KPDzqlqqqt8BDwPXAOcPl2kAdgDHR84oSVqnMXF/Cbg6yblJAuwGngMeBz41bLMHeGTciJKk9Rpzzf0Qy984fQJ4enitfcBXgC8mOQq8B7h/A+aUJK3D3Ok3WVtV3QncecrqF4CrxryuJGkcP6EqSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJamhUXFPcn6SA0l+muRIkg8nuTDJY0meH75esFHDSpLOzNgz93uAH1bVB4APAUeAO4CDVXUZcHBYliRtoYnjnuTdwEeA+wGq6o2qeg24Adg/bLYfuHHskJKk9Rlz5r4LWAK+k+TJJPclOQ/YVlUnhm1eAbaNHVKStD5j4j4HXAncW1VXAL/llEswVVVArbZzkr1JFpMsLi0tjRhDknSqMXE/BhyrqkPD8gGWY/9qkosBhq8nV9u5qvZV1UJVLczPz48YQ5J0qonjXlWvAC8nef+wajfwHPAosGdYtwd4ZNSEkqR1mxu5/2eBB5KcDbwA3MLy/zAeSnIr8CJw08hjSJLWaVTcq+opYGGVp3aPeV1J0jh+QlWSGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaGh33JGcleTLJ94flXUkOJTma5LtJzh4/piRpPTbizP124MiK5buBb1bV+4BfAbduwDEkSeswKu5JdgDXA/cNywGuBQ4Mm+wHbhxzDEnS+o09c/8W8GXg98Pye4DXqurNYfkYsH21HZPsTbKYZHFpaWnkGJKklSaOe5JPACer6vAk+1fVvqpaqKqF+fn5SceQJK1ibsS+1wCfTPJx4Bzgj4F7gPOTzA1n7zuA4+PHlCStx8Rn7lX11araUVU7gZuBH1XVXwCPA58aNtsDPDJ6SknSumzGz7l/BfhikqMsX4O/fxOOIUn6f4y5LPN/qurHwI+Hxy8AV23E60qSJuMnVCWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNTRx3JNckuTxJM8leTbJ7cP6C5M8luT54esFGzeuJOlMjDlzfxP4UlVdDlwN3JbkcuAO4GBVXQYcHJYlSVto4rhX1YmqemJ4/BvgCLAduAHYP2y2H7hx7JCSpPXZkGvuSXYCVwCHgG1VdWJ46hVg2xr77E2ymGRxaWlpI8aQJA1Gxz3Ju4DvAZ+vql+vfK6qCqjV9quqfVW1UFUL8/PzY8eQJK0wKu5J3sFy2B+oqoeH1a8muXh4/mLg5LgRJUnrNeanZQLcDxypqm+seOpRYM/weA/wyOTjSZImMTdi32uAzwBPJ3lqWPc14C7goSS3Ai8CN40bUZK0XhPHvap+AmSNp3dP+rqSpPH8hKokNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1tClxT3Jdkp8lOZrkjs04hiRpbRse9yRnAX8LfAy4HPh0kss3+jiSpLVtxpn7VcDRqnqhqt4A/hG4YROOI0law9wmvOZ24OUVy8eAPz11oyR7gb3D4n8n+dkmzLKWi4BfnunGuXsTJ9la63rfjfi+Z8ssve8/WeuJzYj7GamqfcC+aRw7yWJVLUzj2NPk+54tvu/ZthmXZY4Dl6xY3jGskyRtkc2I+78DlyXZleRs4Gbg0U04jiRpDRt+Waaq3kzy18A/A2cB366qZzf6OCNN5XLQHwDf92zxfc+wVNW0Z5AkbTA/oSpJDRl3SWpopuI+i7dFSHJJkseTPJfk2SS3T3umrZTkrCRPJvn+tGfZSknOT3IgyU+THEny4WnPtBWSfGH4df5MkgeTnDPtmaZlZuI+w7dFeBP4UlVdDlwN3DYj7/sttwNHpj3EFNwD/LCqPgB8iBn4d5BkO/A5YKGqPsjyD3TcPN2ppmdm4s6M3hahqk5U1RPD49+w/Jt8+3Sn2hpJdgDXA/dNe5atlOTdwEeA+wGq6o2qem26U22ZOeCdSeaAc4H/nPI8UzNLcV/ttggzEbm3JNkJXAEcmu4kW+ZbwJeB3097kC22C1gCvjNckrovyXnTHmqzVdVx4OvAS8AJ4PWq+pfpTjU9sxT3mZbkXcD3gM9X1a+nPc9mS/IJ4GRVHZ72LFMwB1wJ3FtVVwC/Bdp/jynJBSz/aXwX8F7gvCR/Od2ppmeW4j6zt0VI8g6Ww/5AVT087Xm2yDXAJ5P8guVLcNcm+YfpjrRljgHHquqtP6EdYDn23X0U+HlVLVXV74CHgT+b8kxTM0txn8nbIiQJy9dej1TVN6Y9z1apqq9W1Y6q2snyf+sfVdVMnMVV1SvAy0neP6zaDTw3xZG2ykvA1UnOHX7d72YGvpG8lqndFXKrvU1ui7AZrgE+Azyd5Klh3deq6gdTnEmb77PAA8OJzAvALVOeZ9NV1aEkB4AnWP4psSeZ4VsRePsBSWpoli7LSNLMMO6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrofwEmvalQyeOL3QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['uc.8+']\n",
            " ['uc.427+']\n",
            " ['uc.378+A']\n",
            " ['uc.238+']\n",
            " ['uc.142+A']\n",
            " ['uc.190+A']\n",
            " ['uc.445+A']\n",
            " ['uc.469+A']\n",
            " ['uc.325+']\n",
            " ['uc.369+']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation of a model using all features\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "# Define the dataset\n",
        "X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
        "\n",
        "# Split into train and test sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)\n",
        "\n",
        "# Fit the model\n",
        "model = LogisticRegression(solver='liblinear')\n",
        "model.fit(X_train, y_train)\n",
        "\n",
        "# Evaluate the model\n",
        "yhat = model.predict(X_test)\n",
        "\n",
        "# Evaluate predictions\n",
        "accuracy = accuracy_score(y_test, yhat)\n",
        "print('Accuracy: %.2f' % (accuracy*100))"
      ],
      "metadata": {
        "id": "OCyXLAJAwhif"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Configure to select a subset of features\n",
        "fs = SelectFromModel(RandomForestClassifier(n_estimators=200), max_features=5)"
      ],
      "metadata": {
        "id": "Nv_loiFrwk6p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Learn relationship from training data\n",
        "fs.fit(X_train, y_train)\n",
        "\n",
        "# Transform train input data\n",
        "X_train_fs = fs.transform(X_train)\n",
        "\n",
        "# Transform test input data\n",
        "X_test_fs = fs.transform(X_test)"
      ],
      "metadata": {
        "id": "z4-YzFHtwqlG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation of a model using 5 features chosen with random forest importance\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "# Feature selection\n",
        "def select_features(X_train, y_train, X_test):\n",
        "\t# Configure to select a subset of features\n",
        "\tfs = SelectFromModel(RandomForestClassifier(n_estimators=1000), max_features=5)\n",
        " \n",
        "\t# Learn relationship from training data\n",
        "\tfs.fit(X_train, y_train)\n",
        "\n",
        "\t# Transform train input data\n",
        "\tX_train_fs = fs.transform(X_train)\n",
        "\n",
        "\t# Transform test input data\n",
        "\tX_test_fs = fs.transform(X_test)\n",
        "\n",
        "\treturn X_train_fs, X_test_fs, fs\n",
        "\n",
        "# Define the dataset\n",
        "X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
        "\n",
        "# Split into train and test sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)\n",
        "\n",
        "# Feature selection\n",
        "X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test)\n",
        "\n",
        "# Fit the model\n",
        "model = LogisticRegression(solver='liblinear')\n",
        "model.fit(X_train_fs, y_train)\n",
        "\n",
        "# Evaluate the model\n",
        "yhat = model.predict(X_test_fs)\n",
        "\n",
        "# Evaluate predictions\n",
        "accuracy = accuracy_score(y_test, yhat)\n",
        "\n",
        "print('Accuracy: %.2f' % (accuracy*100))"
      ],
      "metadata": {
        "id": "X7Ior4urwti2"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}