0413happyday.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web Economic CW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import warnings; warnings.filterwarnings(\"ignore\");\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn import metrics\n",
    "from pyfm import pylibfm\n",
    "\n",
    "import pickle\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "pd.options.mode.chained_assignment = None  # default='warn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Read CSV data\n",
    "df_train = pd.read_csv('dataset/train.csv', encoding=\"ISO-8859-1\")\n",
    "df_test = pd.read_csv('dataset/test.csv', encoding=\"ISO-8859-1\")\n",
    "df_validation = pd.read_csv('dataset/validation.csv', encoding=\"ISO-8859-1\")\n",
    "# raw_data = pd.read_csv('train.csv', header=None, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.999246\n",
       "1    0.000754\n",
       "Name: click, dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['click'].value_counts()/ df_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2664159"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "impression = 2664159\n",
    "#impression = 0\n",
    "#for i in range (df_train.shape[0]):\n",
    "#    if df_train['bidprice'][i] >= df_train['payprice'][i]:\n",
    "        \n",
    "#        impression += 1\n",
    "#return impression\n",
    "impression        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00076346794616987952"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_ctr = (df_train['click'].value_counts()/impression)[1]\n",
    "average_ctr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_train_0 = df_train[df_train['click'] == 0]\n",
    "df_train_1 = df_train[df_train['click'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_train_0 = df_train_0.sample(frac=0.0075, random_state=2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_train = pd.concat([df_train_1, df_train_0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature extraction & hashing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['click', 'weekday', 'hour', 'bidid', 'logtype', 'userid', 'useragent',\n",
       "       'IP', 'region', 'city', 'adexchange', 'domain', 'url', 'urlid',\n",
       "       'slotid', 'slotwidth', 'slotheight', 'slotvisibility', 'slotformat',\n",
       "       'slotprice', 'creative', 'bidprice', 'payprice', 'keypage',\n",
       "       'advertiser', 'usertag'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "header_train = [u'click', u'weekday', u'hour', u'bidid', u'logtype', u'userid', u'useragent',\n",
    "       u'IP', u'region', u'city', u'adexchange', u'domain', u'url', u'urlid',\n",
    "       u'slotid', u'slotwidth', u'slotheight', u'slotvisibility', u'slotformat',\n",
    "       u'slotprice', u'creative', u'bidprice', u'payprice', u'keypage',\n",
    "       u'advertiser', u'usertag']\n",
    "X_header = [u'click', u'weekday', u'hour', u'bidid', u'logtype', u'userid', u'useragent',\n",
    "       u'IP', u'region', u'city', u'adexchange', u'domain', u'url', u'urlid',\n",
    "       u'slotid', u'slotwidth', u'slotheight', u'slotvisibility', u'slotformat',\n",
    "       u'slotprice', u'creative', u'bidprice', u'payprice', u'keypage',\n",
    "       u'advertiser', u'usertag']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train.columns = header_train\n",
    "X = np.array(df_train[X_header])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 6, 16, '61868b8d67b4a94f917cb98cb579578f', 1,\n",
       "       'u_Vhk7PrMPOZM6GQ2', 'windows_ie', '1.86.243.*', 333, 334, '2',\n",
       "       'trqRTuMvjTN7X9KbuKz', '9d960a6b69c7a2ca69625dec449e338a', 'null',\n",
       "       '593718700', 728, 90, '0', '0', 192,\n",
       "       '4ad7e35171a3d8de73bb862791575f2e', 238, 216,\n",
       "       'd29e59bf0f7f8243858b8183f14d4412', 3358,\n",
       "       '10083,13776,10059,10057,10110'], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "D = 50000\n",
    "\n",
    "def process_row(line, header):\n",
    "    rst = []\n",
    "    non_use_keys = set(['logtype', 'click', 'bidid', 'bidprice', 'payprice'])\n",
    "    for key,value in zip(header, line):\n",
    "        value = str(value)\n",
    "        key = str(key)\n",
    "        if value == '' or key in non_use_keys:\n",
    "            continue\n",
    "#         elif key == 'timestamp':\n",
    "#             rst += ['time_day_'+value[6:8]]\n",
    "        elif key == 'useragent':\n",
    "            value = value.split('_')\n",
    "            rst += ['useragent_os_'+value[0]]\n",
    "            rst += ['useragent_browser_'+value[1]]\n",
    "        elif key == 'usertag':\n",
    "            for sub_value in value.split(','):\n",
    "                rst += ['usertags_'+str(sub_value)]\n",
    "        else:\n",
    "            rst += [key+'_'+value]\n",
    "    return [abs(hash(v)) % D for v in rst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = [process_row(x, X_header) for x in X]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.2107345943185512"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.std([len(x) for x in X])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Y = np.array(df_train[['bidid', 'click', 'payprice']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('train.xyz.csv','w') as f:\n",
    "    for y, x in zip(Y, X):\n",
    "        line = ','.join(map(str, y)) + ',' + ','.join(map(str, x)) + '\\n'\n",
    "        f.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for validation data\n",
    "\n",
    "X_val = np.array(df_validation[X_header])\n",
    "df_validation.columns = header_train\n",
    "X_val = np.array(df_validation[X_header])\n",
    "Y_val = np.array(df_validation[['bidid', 'click', 'payprice']])\n",
    "X_val = [process_row(x, X_header) for x in X_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('validation.xyz.csv','w') as f:\n",
    "    for y, x in zip(Y_val, X_val):\n",
    "        line = ','.join(map(str, y)) + ',' + ','.join(map(str, x)) + '\\n'\n",
    "        f.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for test data\n",
    "header_test = [u'weekday', u'hour', u'bidid', u'logtype',\n",
    "       u'userid', u'useragent', u'IP', u'region', u'city', u'adexchange',\n",
    "       u'domain', u'url', u'urlid', u'slotid', u'slotwidth', u'slotheight',\n",
    "       u'slotvisibility', u'slotformat', u'slotprice', u'creative',\n",
    "        u'keypage', u'advertiser', u'usertag']\n",
    "X_header = [u'weekday', u'hour', u'bidid', u'logtype',\n",
    "       u'userid', u'useragent', u'IP', u'region', u'city', u'adexchange',\n",
    "       u'domain', u'url', u'urlid', u'slotid', u'slotwidth', u'slotheight',\n",
    "       u'slotvisibility', u'slotformat', u'slotprice', u'creative',\n",
    "     u'keypage', u'advertiser', u'usertag']\n",
    "\n",
    "df_test.columns = header_test\n",
    "X_test = np.array(df_test[X_header])\n",
    "Y_test = np.array(df_test[['bidid']])\n",
    "X_test = [process_row(x, X_header) for x in X_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('test.xyz.csv','w') as f:\n",
    "    for y, x in zip(Y_test, X_test):\n",
    "        line = ','.join(map(str, y)) + ',' + ','.join(map(str, x)) + '\\n'\n",
    "        f.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = []\n",
    "Y = []\n",
    "with open('train.xyz.csv','r') as f:\n",
    "    for row in f.readlines():\n",
    "        r = row.strip().split(',')\n",
    "        Y.append(r[0:3])\n",
    "        X.append(list(map(int,r[3:])))\n",
    "               \n",
    "X_val = []\n",
    "Y_val = []\n",
    "with open('validation.xyz.csv','r') as f:\n",
    "    for row in f.readlines():\n",
    "        r = row.strip().split(',')\n",
    "        Y_val.append(r[0:3])\n",
    "        X_val.append(list(map(int,r[3:])))\n",
    "        \n",
    "X_test = []\n",
    "Y_test = []\n",
    "with open('test.xyz.csv','r') as f:\n",
    "    for row in f.readlines():\n",
    "        r = row.strip().split(',')\n",
    "        Y_test.append(r[0:1])\n",
    "        X_test.append(list(map(int,r[1:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[25555,\n",
       " 5679,\n",
       " 13501,\n",
       " 10061,\n",
       " 8471,\n",
       " 29474,\n",
       " 32274,\n",
       " 33549,\n",
       " 17563,\n",
       " 45779,\n",
       " 47731,\n",
       " 48679,\n",
       " 41427,\n",
       " 7447,\n",
       " 26703,\n",
       " 16063,\n",
       " 311,\n",
       " 25182,\n",
       " 29730,\n",
       " 39871,\n",
       " 16760,\n",
       " 27390,\n",
       " 42721,\n",
       " 32104,\n",
       " 32374,\n",
       " 34129]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = np.array([np.array(x, dtype=np.int32) for x in X])\n",
    "X_val = np.array([np.array(x, dtype=np.int32) for x in X_val])\n",
    "X_test = np.array([np.array(x, dtype=np.int32) for x in X_test])\n",
    "Y = np.array(Y)\n",
    "shuffle = np.random.permutation(len(Y))\n",
    "X = X[shuffle]\n",
    "Y = Y[shuffle]\n",
    "\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "train = [{str(xx):1 for xx in x} for x in X]\n",
    "val = [{str(xx):1 for xx in x} for x in X_val]\n",
    "test = [{str(xx):1 for xx in x} for x in X_test]\n",
    "v = DictVectorizer()\n",
    "X_FM = v.fit_transform(train)\n",
    "X_val_FM = v.transform(val)\n",
    "X_test_FM = v.fit_transform(test)\n",
    "\n",
    "Y_val = np.array(Y_val)\n",
    "target = np.array(Y[:,1], dtype=np.int32)\n",
    "target_val = np.array(Y_val[:,1], dtype=np.int32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Caculate the calibrated CTR\n",
    "def Cali_CTR(p, w):\n",
    "\n",
    "    cali_ctr = p/(p+(1-p)/w)\n",
    "\n",
    "    return cali_ctr   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Caculate the root mean square rooot\n",
    "def rmse(predictions, targets):\n",
    "\n",
    "    differences = predictions - targets                       #the DIFFERENCEs.\n",
    "\n",
    "    differences_squared = differences ** 2                    #the SQUAREs of ^\n",
    "\n",
    "    mean_of_differences_squared = differences_squared.mean()  #the MEAN of ^\n",
    "\n",
    "    rmse_val = np.sqrt(mean_of_differences_squared)           #ROOT of ^\n",
    "\n",
    "    return rmse_val     #get the ^"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "##  Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 1.14 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "# Initialize the Random Forest model\n",
    "rf_model = RandomForestClassifier(n_estimators=1000,\n",
    "                                  max_features = 0.5,\n",
    "                                  min_samples_leaf = 50,\n",
    "                                  oob_score = True,\n",
    "                                  max_depth = 8,\n",
    "                                  n_jobs = -1,\n",
    "                                  random_state = 2017).fit(X_FM, target)\n",
    "#pickle.dump(rf_model,open('random_forest_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.6 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "# predict with model\n",
    "#rf_model = pickle.load(open('random_forest_model.sav', 'rb'))  \n",
    "Cali_rf = Cali_CTR(rf_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.875567159158\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_rf, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_rf)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "click                                            1\n",
       "weekday                                          3\n",
       "hour                                             9\n",
       "bidid             0942e488346a1a30cfe08f80425bc278\n",
       "logtype                                          1\n",
       "userid                           u_Vh5hC3MlP9udlCl\n",
       "useragent                               windows_ie\n",
       "IP                                    110.81.254.*\n",
       "region                                         124\n",
       "city                                           129\n",
       "adexchange                                       2\n",
       "domain                                        null\n",
       "url               1094ee84b13d866cb43c94f9b986e1f5\n",
       "urlid                                         null\n",
       "slotid                                   463662836\n",
       "slotwidth                                      728\n",
       "slotheight                                      90\n",
       "slotvisibility                                   2\n",
       "slotformat                                       0\n",
       "slotprice                                       39\n",
       "creative          b90c12ed2bd7950c6027bf9c6937c48a\n",
       "bidprice                                       300\n",
       "payprice                                       218\n",
       "keypage           361e128affece850342293213691a043\n",
       "advertiser                                    3386\n",
       "usertag                                       null\n",
       "Name: 25069, dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.iloc[20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "aaaaaaa = pd.DataFrame({'test':rf_model.feature_importances_})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test\n",
       "0   0.0\n",
       "1   0.0\n",
       "2   0.0\n",
       "3   0.0\n",
       "4   0.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aaaaaaa.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "aaaaaaa.sort_values(by=['test'],inplace=True,ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35061</th>\n",
       "      <td>0.229885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6738</th>\n",
       "      <td>0.136558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.088494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12225</th>\n",
       "      <td>0.061008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9794</th>\n",
       "      <td>0.059942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35375</th>\n",
       "      <td>0.057616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11187</th>\n",
       "      <td>0.043539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24017</th>\n",
       "      <td>0.040312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5478</th>\n",
       "      <td>0.037641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5940</th>\n",
       "      <td>0.026839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17569</th>\n",
       "      <td>0.022365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4371</th>\n",
       "      <td>0.017611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21610</th>\n",
       "      <td>0.017527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1906</th>\n",
       "      <td>0.016366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21665</th>\n",
       "      <td>0.015585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13818</th>\n",
       "      <td>0.009761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2475</th>\n",
       "      <td>0.006922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12709</th>\n",
       "      <td>0.006322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32015</th>\n",
       "      <td>0.005584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24861</th>\n",
       "      <td>0.005530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32257</th>\n",
       "      <td>0.005515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17336</th>\n",
       "      <td>0.005500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33207</th>\n",
       "      <td>0.004129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28431</th>\n",
       "      <td>0.004028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34636</th>\n",
       "      <td>0.003967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10028</th>\n",
       "      <td>0.003431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25316</th>\n",
       "      <td>0.003378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18180</th>\n",
       "      <td>0.003349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29234</th>\n",
       "      <td>0.003283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16960</th>\n",
       "      <td>0.003015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12086</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12087</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12088</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12089</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12091</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12066</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12092</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12093</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12094</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12095</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12096</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12097</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12083</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12082</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12081</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12080</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12079</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12078</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12077</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12076</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12075</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12074</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12073</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12072</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12071</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12070</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12069</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12068</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12067</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36145</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36146 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           test\n",
       "35061  0.229885\n",
       "6738   0.136558\n",
       "50     0.088494\n",
       "12225  0.061008\n",
       "9794   0.059942\n",
       "35375  0.057616\n",
       "11187  0.043539\n",
       "24017  0.040312\n",
       "5478   0.037641\n",
       "5940   0.026839\n",
       "17569  0.022365\n",
       "4371   0.017611\n",
       "21610  0.017527\n",
       "1906   0.016366\n",
       "21665  0.015585\n",
       "13818  0.009761\n",
       "2475   0.006922\n",
       "12709  0.006322\n",
       "32015  0.005584\n",
       "24861  0.005530\n",
       "32257  0.005515\n",
       "17336  0.005500\n",
       "33207  0.004129\n",
       "28431  0.004028\n",
       "34636  0.003967\n",
       "10028  0.003431\n",
       "25316  0.003378\n",
       "18180  0.003349\n",
       "29234  0.003283\n",
       "16960  0.003015\n",
       "...         ...\n",
       "12086  0.000000\n",
       "12087  0.000000\n",
       "12088  0.000000\n",
       "12089  0.000000\n",
       "12091  0.000000\n",
       "12066  0.000000\n",
       "12092  0.000000\n",
       "12093  0.000000\n",
       "12094  0.000000\n",
       "12095  0.000000\n",
       "12096  0.000000\n",
       "12097  0.000000\n",
       "12083  0.000000\n",
       "12082  0.000000\n",
       "12081  0.000000\n",
       "12080  0.000000\n",
       "12079  0.000000\n",
       "12078  0.000000\n",
       "12077  0.000000\n",
       "12076  0.000000\n",
       "12075  0.000000\n",
       "12074  0.000000\n",
       "12073  0.000000\n",
       "12072  0.000000\n",
       "12071  0.000000\n",
       "12070  0.000000\n",
       "12069  0.000000\n",
       "12068  0.000000\n",
       "12067  0.000000\n",
       "36145  0.000000\n",
       "\n",
       "[36146 rows x 1 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aaaaaaa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#zip(map(lambda x: x, rf_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extra Tree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.04 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "extra_model = ExtraTreesClassifier(bootstrap=False, \n",
    "                                   min_samples_leaf=2,\n",
    "                                   min_samples_split=6,\n",
    "                                   n_estimators=100,\n",
    "                                   n_jobs=-1,\n",
    "                                   random_state=2017).fit(X_FM, target)\n",
    "pickle.dump(extra_model,open('extra_tree_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.09 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "# predict with model\n",
    "extra_model = pickle.load(open('extra_tree_model.sav', 'rb'))\n",
    "\n",
    "Cali_extra = Cali_CTR(extra_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.823916073755\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_extra, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_extra)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.01 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "decision_model = DecisionTreeClassifier(criterion='gini', \n",
    "                                        splitter='best', #'random' \n",
    "                                        max_depth=8, \n",
    "                                        min_samples_split=4, \n",
    "                                        min_samples_leaf=1, \n",
    "                                        random_state=2017, \n",
    "                                        class_weight='balanced').fit(X_FM, target)\n",
    "pickle.dump(decision_model,open('decision_tree_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.0 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "# predict with model\n",
    "decision_model = pickle.load(open('decision_tree_model.sav', 'rb'))\n",
    "Cali_dec = Cali_CTR(decision_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.844066542203\n",
      "RMSE on test dataset = 0.04\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_dec, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_dec)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]--- time use: 0.01 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "start_time = time.time()\n",
    "lr_model = LogisticRegression(C=0.137,#9,  #0.665045541881\n",
    "                              penalty='l2',\n",
    "                              class_weight='balanced',\n",
    "                              fit_intercept=True,\n",
    "                              intercept_scaling=1,\n",
    "                              max_iter=20,\n",
    "                              multi_class='ovr',\n",
    "                              n_jobs = -1,\n",
    "                              random_state=2017,\n",
    "                              solver='liblinear',\n",
    "                              verbose=True).fit(X_FM,target)\n",
    "pickle.dump(lr_model,open('logistic_regression_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.0 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "lr_model = pickle.load(open('logistic_regression_model.sav', 'rb'))\n",
    "Cali_lr = Cali_CTR(lr_model.predict(X_val_FM), 0.0075)\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.774905595472\n",
      "RMSE on test dataset = 0.35\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_lr, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_lr)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elastic Net "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 4.77 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import ElasticNet\n",
    "\n",
    "start_time = time.time()\n",
    "elastic_model = ElasticNet(alpha=0.0005,\n",
    "                        l1_ratio = 0.5, #0.7\n",
    "                        random_state = 2017).fit(X_FM, target)\n",
    "pickle.dump(elastic_model,open('elatic_net_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.0 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "elastic_model = pickle.load(open('elatic_net_model.sav', 'rb'))\n",
    "Cali_elas = Cali_CTR(np.expm1(elastic_model.predict(X_val_FM)), 0.0075)\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.801791308357\n",
      "RMSE on test dataset = 0.05\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_elas, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_elas)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.0 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "sgd_model = SGDClassifier(loss='log',\n",
    "                          penalty=\"elasticnet\",\n",
    "                          #penalty='l2',\n",
    "                          alpha=0.0003,\n",
    "                          l1_ratio=0.009,\n",
    "                          fit_intercept=True,\n",
    "                          n_iter=5,\n",
    "                          shuffle=True,\n",
    "                          verbose=0,\n",
    "                          epsilon=0.1,\n",
    "                          n_jobs = -1,\n",
    "                          random_state=2017,\n",
    "                          learning_rate='optimal', \n",
    "                          eta0=0.0, \n",
    "                          power_t=0.5, \n",
    "                          class_weight=None, \n",
    "                          warm_start=False, \n",
    "                          average=False).fit(X_FM, target)\n",
    "pickle.dump(sgd_model,open('SGDClassifier_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.0 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "sgd_model = pickle.load(open('SGDClassifier_model.sav', 'rb'))\n",
    "Cali_sgd = Cali_CTR(sgd_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.829974851755\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_sgd, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_sgd)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "dtrain = xgb.DMatrix(X_FM, label=target)\n",
    "dval = xgb.DMatrix(X_val_FM, label=target_val)\n",
    "dtest = xgb.DMatrix(X_test_FM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.69 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "#xgbclassifier\n",
    "xgb_model = xgb.XGBClassifier(max_depth=6,\n",
    "                         learning_rate=0.095,\n",
    "                         n_estimators=1000,\n",
    "                         silent=True,\n",
    "                         objective='binary:logistic',\n",
    "                         nthread=-1,\n",
    "                         gamma=0, \n",
    "                         min_child_weight=1,\n",
    "                         max_delta_step=0,\n",
    "                         subsample=1,\n",
    "                         colsample_bytree=1,\n",
    "                         colsample_bylevel=1,\n",
    "                         reg_alpha=0,\n",
    "                         reg_lambda=1,\n",
    "                         scale_pos_weight=1,\n",
    "                         base_score=0.5,\n",
    "                         seed=2017).fit(X_FM, target)\n",
    "pickle.dump(xgb_model,open('XGBClassifier_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.15 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "xgb_model = pickle.load(open('XGBClassifier_model.sav', 'rb'))\n",
    "Cali_xgb = Cali_CTR(xgb_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.87568908606\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_xgb, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_xgb)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "dtrain = xgb.DMatrix(X_FM, label=target)\n",
    "dval = xgb.DMatrix(X_val_FM, label=target_val)\n",
    "dtest = xgb.DMatrix(X_test_FM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-rmse:0.460769\teval-rmse:0.456051\n",
      "[1]\ttrain-rmse:0.426627\teval-rmse:0.416922\n",
      "[2]\ttrain-rmse:0.396816\teval-rmse:0.382042\n",
      "[3]\ttrain-rmse:0.370844\teval-rmse:0.350959\n",
      "[4]\ttrain-rmse:0.348303\teval-rmse:0.323199\n",
      "[5]\ttrain-rmse:0.328774\teval-rmse:0.298434\n",
      "[6]\ttrain-rmse:0.311938\teval-rmse:0.27639\n",
      "[7]\ttrain-rmse:0.29747\teval-rmse:0.256811\n",
      "[8]\ttrain-rmse:0.285041\teval-rmse:0.239305\n",
      "[9]\ttrain-rmse:0.274458\teval-rmse:0.223594\n",
      "[10]\ttrain-rmse:0.265497\teval-rmse:0.209648\n",
      "[11]\ttrain-rmse:0.257818\teval-rmse:0.197398\n",
      "[12]\ttrain-rmse:0.251303\teval-rmse:0.186475\n",
      "[13]\ttrain-rmse:0.245819\teval-rmse:0.176787\n",
      "[14]\ttrain-rmse:0.241111\teval-rmse:0.168123\n",
      "[15]\ttrain-rmse:0.237169\teval-rmse:0.160569\n",
      "[16]\ttrain-rmse:0.233817\teval-rmse:0.153872\n",
      "[17]\ttrain-rmse:0.230905\teval-rmse:0.147859\n",
      "[18]\ttrain-rmse:0.228527\teval-rmse:0.142651\n",
      "[19]\ttrain-rmse:0.226359\teval-rmse:0.137899\n",
      "[20]\ttrain-rmse:0.22451\teval-rmse:0.133782\n",
      "[21]\ttrain-rmse:0.223004\teval-rmse:0.130187\n",
      "[22]\ttrain-rmse:0.22169\teval-rmse:0.126964\n",
      "[23]\ttrain-rmse:0.220576\teval-rmse:0.124071\n",
      "[24]\ttrain-rmse:0.219553\teval-rmse:0.121532\n",
      "[25]\ttrain-rmse:0.218735\teval-rmse:0.119169\n",
      "[26]\ttrain-rmse:0.217969\teval-rmse:0.117046\n",
      "[27]\ttrain-rmse:0.217331\teval-rmse:0.115022\n",
      "[28]\ttrain-rmse:0.216781\teval-rmse:0.113276\n",
      "[29]\ttrain-rmse:0.216296\teval-rmse:0.111772\n",
      "[30]\ttrain-rmse:0.215938\teval-rmse:0.110358\n",
      "[31]\ttrain-rmse:0.215515\teval-rmse:0.109221\n",
      "[32]\ttrain-rmse:0.21505\teval-rmse:0.10822\n",
      "[33]\ttrain-rmse:0.214675\teval-rmse:0.107069\n",
      "[34]\ttrain-rmse:0.214315\teval-rmse:0.106333\n",
      "[35]\ttrain-rmse:0.213966\teval-rmse:0.10545\n",
      "[36]\ttrain-rmse:0.213614\teval-rmse:0.104847\n",
      "[37]\ttrain-rmse:0.21334\teval-rmse:0.104143\n",
      "[38]\ttrain-rmse:0.21305\teval-rmse:0.103619\n",
      "[39]\ttrain-rmse:0.212796\teval-rmse:0.103107\n",
      "[40]\ttrain-rmse:0.212563\teval-rmse:0.102751\n",
      "[41]\ttrain-rmse:0.212306\teval-rmse:0.102365\n",
      "[42]\ttrain-rmse:0.212129\teval-rmse:0.102056\n",
      "[43]\ttrain-rmse:0.211936\teval-rmse:0.101693\n",
      "[44]\ttrain-rmse:0.211718\teval-rmse:0.101283\n",
      "[45]\ttrain-rmse:0.211557\teval-rmse:0.10098\n",
      "[46]\ttrain-rmse:0.211373\teval-rmse:0.100736\n",
      "[47]\ttrain-rmse:0.211173\teval-rmse:0.100565\n",
      "[48]\ttrain-rmse:0.21102\teval-rmse:0.100328\n",
      "[49]\ttrain-rmse:0.210868\teval-rmse:0.100241\n",
      "--- time use: 0.13 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "learning_rate = 0.1 #0.879086213156\n",
    "max_depth = 12\n",
    "min_child_weight = 6\n",
    "num_round =50\n",
    "\n",
    "xgb_params = {\n",
    "    #General Parameters\n",
    "    'booster': 'gbtree',\n",
    "    'silent': 0,    \n",
    "    #Tree Booster Parameters\n",
    "    'eta': learning_rate,    \n",
    "    'max_depth': max_depth, #usually 3-10, big value can provide from overfitting\n",
    "    'min_child_weight': min_child_weight,\n",
    "    'subsample': 1,\n",
    "    'colsample_bytree': 1,\n",
    "    'alpha': 1, # improve speed\n",
    "    'scale_pos_weight': 1, #default = 1, positive value can converge faster\n",
    "    'early_stopping_rounds': 50,   \n",
    "    #learn objective parameters\n",
    "    'objective': 'reg:logistic',\n",
    "    'evalmatrix': 'rmse',\n",
    "    'seed': 2017}\n",
    "\n",
    "watchlist = [(dtrain, 'train'), (dval, 'eval')]\n",
    "\n",
    "bst_model = xgb.train(xgb_params, dtrain, num_round, watchlist)\n",
    "pickle.dump(bst_model,open('XGBoost_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.01 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "bst_model = pickle.load(open('XGBoost_model.sav', 'rb'))\n",
    "Cali_bst = Cali_CTR(bst_model.predict(dval), 0.0075)\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.893795988128\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_bst, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_bst)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.07 minutes ---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method DMatrix.__del__ of <xgboost.core.DMatrix object at 0x152d01a20>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/xgboost-0.6-py3.6.egg/xgboost/core.py\", line 337, in __del__\n",
      "    _check_call(_LIB.XGDMatrixFree(self.handle))\n",
      "AttributeError: 'DMatrix' object has no attribute 'handle'\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "start_time = time.time()\n",
    "\n",
    "ada_model = AdaBoostClassifier(base_estimator=None, \n",
    "                               n_estimators=100, \n",
    "                               learning_rate=0.5, \n",
    "                               random_state=2017).fit(X_FM,target)\n",
    "pickle.dump(ada_model,open('AdaBoost_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.11 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "ada_model = pickle.load(open('AdaBoost_model.sav', 'rb'))\n",
    "Cali_ada = Cali_CTR(ada_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.840433782339\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_ada, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_ada)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 2.14 minutes ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "start_time = time.time()\n",
    "\n",
    "nn_model = MLPClassifier(hidden_layer_sizes=(140,70),\n",
    "                   activation = 'logistic',\n",
    "                   solver = 'lbfgs',\n",
    "                   max_iter = 150,\n",
    "                   learning_rate_init = 0.75,\n",
    "                   alpha = 0.0001, #l2 penalty\n",
    "                   random_state=2017).fit(X_FM, target)\n",
    "\n",
    "pickle.dump(nn_model,open('Neural_Network_model.sav', 'wb'))\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- time use: 0.03 minutes ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "nn_model = pickle.load(open('Neural_Network_model.sav', 'rb'))\n",
    "\n",
    "Cali_nn = Cali_CTR(nn_model.predict_proba(X_val_FM), 0.0075)[:,1]\n",
    "print(\"--- time use: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.843039813835\n",
      "RMSE on test dataset = 0.05\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_nn, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_nn)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bidding Strategy\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "Apply the pCTR to the validation set and evaluate with pay price for each impression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def evaluation (pBidprice, payprice, clicks, budget=6250):\n",
    "    win_num = 0.\n",
    "    total_cost = 0.\n",
    "    click_num = 0.\n",
    "    for i in range(0, len(pBidprice)):\n",
    "        if pBidprice[i] > payprice[i]:\n",
    "            if total_cost + payprice[i] / 1000. > budget:\n",
    "                break\n",
    "            win_num = win_num + 1\n",
    "            total_cost = total_cost + payprice[i] / 1000.\n",
    "            click_num += clicks[i]\n",
    "    ctr = click_num / win_num\n",
    "    cpc = total_cost / click_num\n",
    "    print('win_num {}, total_cost {}, click_num {}, ctr {}, cpc {}'.format\n",
    "          (win_num, total_cost, click_num, ctr, cpc))\n",
    "    return win_num, total_cost, click_num, ctr, cpc       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def pbidprice(ctr, base_bid, flag):\n",
    "    pbid = []\n",
    "    for i in range(len(ctr)):\n",
    "        if flag == 'linear':\n",
    "            p = base_bid * ((ctr[i]/average_ctr))            \n",
    "            pbid.append(p)\n",
    "        elif flag == 'squared':\n",
    "            p = base_bid * ((ctr[i]/average_ctr)**2)            \n",
    "            pbid.append(p)\n",
    "        elif flag == 'constant':\n",
    "            pbid.append(base_bid)\n",
    "        elif flag == 'random':\n",
    "            p = np.random.randint(100,200)\n",
    "            pbid.append(p)\n",
    "        elif flag == 'gated':\n",
    "            tradeoff = (ctr[i]/average_ctr)\n",
    "            if tradeoff > 1:\n",
    "                p = 124 # get from constant strategy\n",
    "            else: p = 1\n",
    "            pbid.append(p) \n",
    "            \n",
    "    if flag == 'linear':\n",
    "        print(\" **LINEAR** \")\n",
    "    elif flag == 'squared':\n",
    "        print(\"**SQUARED**\")\n",
    "    elif flag == 'constant':\n",
    "        print(\"**CONSTANT**\")\n",
    "    elif flag == \"random\":\n",
    "        print(\"**RANDOM**\")\n",
    "    elif flag == \"gated\":\n",
    "        print(\"**GATED**\")\n",
    "    return pbid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Payprice_val = np.array(Y_val[:,2], dtype=np.int32)\n",
    "clicks = np.array(Y_val[:,1], dtype=np.int32)\n",
    "#pCTR = Cali_xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Payprice_train = np.array(Y[:,2], dtype=np.int32)\n",
    "clicks_train = np.array(Y[:,1], dtype=np.int32)\n",
    "#pCTR = Cali_xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constant Bidding Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def constant_bidding(base_bid, Payprice):\n",
    "    base = 300\n",
    "    win_number = []\n",
    "    totalcost = []\n",
    "    clicknum = []\n",
    "    ctr_p = []\n",
    "    cpc_p = []\n",
    "    bid_p = []\n",
    "    Cali_constant = [[]]*len(Payprice)\n",
    "    for i in range (50):\n",
    "\n",
    "        constant_bid = pbidprice(Cali_constant, base, 'constant')\n",
    "        win_num, total_cost, click_num, ctr, cpc = evaluation(constant_bid, Payprice_train, clicks_train)\n",
    "\n",
    "        #win_num, total_cost, click_num, ctr, cpc = evaluation(constant_bid, Payprice_val, clicks)\n",
    "        base+=1\n",
    "\n",
    "        win_number.append(win_num)\n",
    "        totalcost.append(total_cost)\n",
    "        clicknum.append(click_num)\n",
    "        ctr_p.append(ctr)\n",
    "        cpc_p.append(cpc)\n",
    "        bid_p.append(constant_bid)\n",
    "        print(i+1,'/50')\n",
    "    return best_bid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**CONSTANT**\n",
      "win_num 149629.0, total_cost 5238.832000002614, click_num 58.0, ctr 0.00038762539347319034, cpc 90.32468965521748\n",
      "1 /50\n",
      "**CONSTANT**\n",
      "win_num 161979.0, total_cost 6249.966000002683, click_num 70.0, ctr 0.0004321547854968854, cpc 89.2852285714669\n",
      "2 /50\n",
      "**CONSTANT**\n",
      "win_num 161122.0, total_cost 6249.976000002561, click_num 68.0, ctr 0.0004220404414046499, cpc 91.91141176474355\n",
      "3 /50\n",
      "**CONSTANT**\n",
      "win_num 160445.0, total_cost 6249.935000002575, click_num 68.0, ctr 0.0004238212471563464, cpc 91.91080882356728\n",
      "4 /50\n",
      "**CONSTANT**\n",
      "win_num 156607.0, total_cost 6249.99400000276, click_num 69.0, ctr 0.00044059333235423704, cpc 90.5796231884458\n",
      "5 /50\n",
      "**CONSTANT**\n",
      "win_num 155792.0, total_cost 6249.99000000264, click_num 66.0, ctr 0.0004236417787819657, cpc 94.69681818185818\n",
      "6 /50\n",
      "**CONSTANT**\n",
      "win_num 155061.0, total_cost 6249.984000002532, click_num 68.0, ctr 0.00043853709185417354, cpc 91.91152941180195\n",
      "7 /50\n",
      "**CONSTANT**\n",
      "win_num 152959.0, total_cost 6249.948000002469, click_num 67.0, ctr 0.00043802587621519495, cpc 93.2828059701861\n",
      "8 /50\n",
      "**CONSTANT**\n",
      "win_num 146501.0, total_cost 6249.992000003338, click_num 76.0, ctr 0.0005187677899809557, cpc 82.23673684214918\n",
      "9 /50\n",
      "**CONSTANT**\n",
      "win_num 145927.0, total_cost 6249.979000003434, click_num 77.0, ctr 0.0005276610908193823, cpc 81.16855844160304\n",
      "10 /50\n",
      "**CONSTANT**\n",
      "win_num 145515.0, total_cost 6249.961000003404, click_num 77.0, ctr 0.0005291550699240629, cpc 81.16832467536888\n",
      "11 /50\n",
      "**CONSTANT**\n",
      "win_num 140239.0, total_cost 6249.995000002741, click_num 77.0, ctr 0.0005490626715820849, cpc 81.16876623380183\n",
      "12 /50\n",
      "**CONSTANT**\n",
      "win_num 139663.0, total_cost 6249.975000002761, click_num 79.0, ctr 0.0005656473081632215, cpc 79.11360759497165\n",
      "13 /50\n",
      "**CONSTANT**\n",
      "win_num 139223.0, total_cost 6249.943000002789, click_num 78.0, ctr 0.0005602522571701515, cpc 80.12747435901011\n",
      "14 /50\n",
      "**CONSTANT**\n",
      "win_num 138787.0, total_cost 6249.993000002715, click_num 78.0, ctr 0.0005620122922175709, cpc 80.1281153846502\n",
      "15 /50\n",
      "**CONSTANT**\n",
      "win_num 136607.0, total_cost 6249.937000002317, click_num 79.0, ctr 0.0005783012583542571, cpc 79.11312658230781\n",
      "16 /50\n",
      "**CONSTANT**\n",
      "win_num 135902.0, total_cost 6249.947000002334, click_num 79.0, ctr 0.0005813012317699519, cpc 79.11325316458651\n",
      "17 /50\n",
      "**CONSTANT**\n",
      "win_num 135245.0, total_cost 6249.970000002315, click_num 80.0, ctr 0.0005915190949757847, cpc 78.12462500002894\n",
      "18 /50\n",
      "**CONSTANT**\n",
      "win_num 134784.0, total_cost 6249.951000002372, click_num 82.0, ctr 0.00060838081671415, cpc 76.21891463417526\n",
      "19 /50\n",
      "**CONSTANT**\n",
      "win_num 132014.0, total_cost 6249.949000002158, click_num 84.0, ctr 0.00063629615040829, cpc 74.40415476193046\n",
      "20 /50\n",
      "**CONSTANT**\n",
      "win_num 130727.0, total_cost 6249.999000002071, click_num 85.0, ctr 0.0006502099795757571, cpc 73.52940000002437\n",
      "21 /50\n",
      "**CONSTANT**\n",
      "win_num 130291.0, total_cost 6249.987000002113, click_num 84.0, ctr 0.0006447106860796218, cpc 74.4046071428823\n",
      "22 /50\n",
      "**CONSTANT**\n",
      "win_num 129676.0, total_cost 6249.942000002152, click_num 84.0, ctr 0.0006477682840309695, cpc 74.40407142859705\n",
      "23 /50\n",
      "**CONSTANT**\n",
      "win_num 129384.0, total_cost 6249.991000002121, click_num 84.0, ctr 0.0006492301984789464, cpc 74.40465476193\n",
      "24 /50\n",
      "**CONSTANT**\n",
      "win_num 129038.0, total_cost 6249.957000002085, click_num 82.0, ctr 0.0006354717215083928, cpc 76.21898780490348\n",
      "25 /50\n",
      "**CONSTANT**\n",
      "win_num 128601.0, total_cost 6249.965000002099, click_num 82.0, ctr 0.0006376311226195752, cpc 76.21908536587927\n",
      "26 /50\n",
      "**CONSTANT**\n",
      "win_num 127728.0, total_cost 6249.964000002071, click_num 82.0, ctr 0.0006419892271076037, cpc 76.21907317075696\n",
      "27 /50\n",
      "**CONSTANT**\n",
      "win_num 127327.0, total_cost 6249.989000001997, click_num 80.0, ctr 0.0006283035020066443, cpc 78.12486250002496\n",
      "28 /50\n",
      "**CONSTANT**\n",
      "win_num 126997.0, total_cost 6249.967000001993, click_num 80.0, ctr 0.0006299361402237848, cpc 78.12458750002492\n",
      "29 /50\n",
      "**CONSTANT**\n",
      "win_num 126829.0, total_cost 6249.929000001978, click_num 80.0, ctr 0.00063077056509158, cpc 78.12411250002472\n",
      "30 /50\n",
      "**CONSTANT**\n",
      "win_num 126534.0, total_cost 6249.985000001995, click_num 81.0, ctr 0.0006401441509791834, cpc 77.16030864199993\n",
      "31 /50\n",
      "**CONSTANT**\n",
      "win_num 125468.0, total_cost 6249.924000002061, click_num 81.0, ctr 0.0006455829374820671, cpc 77.159555555581\n",
      "32 /50\n",
      "**CONSTANT**\n",
      "win_num 124211.0, total_cost 6249.9590000019425, click_num 81.0, ctr 0.0006521161571841463, cpc 77.15998765434497\n",
      "33 /50\n",
      "**CONSTANT**\n",
      "win_num 123920.0, total_cost 6249.977000001936, click_num 81.0, ctr 0.0006536475145255004, cpc 77.16020987656711\n",
      "34 /50\n",
      "**CONSTANT**\n",
      "win_num 123461.0, total_cost 6249.998000001961, click_num 83.0, ctr 0.0006722770753517305, cpc 75.30118072291519\n",
      "35 /50\n",
      "**CONSTANT**\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-254-5abf22a7cc82>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;31m#win_num, total_cost, click_num, ctr, cpc = evaluation(constant_bid, Payprice_train, clicks_train)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mwin_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_cost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclick_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcpc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconstant_bid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPayprice_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclicks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mbase\u001b[0m\u001b[0;34m+=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-143-45ba727807a1>\u001b[0m in \u001b[0;36mevaluation\u001b[0;34m(pBidprice, payprice, clicks, budget)\u001b[0m\n\u001b[1;32m      8\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m             \u001b[0mwin_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwin_num\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m             \u001b[0mtotal_cost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtotal_cost\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpayprice\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m1000.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m             \u001b[0mclick_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mclicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mctr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclick_num\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mwin_num\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#on training\n",
    "base = 70\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "bid_p = []\n",
    "#Cali_constant = [[]]*len(Payprice_train)\n",
    "Cali_constant = [[]]*len(Payprice_val)\n",
    "\n",
    "\n",
    "for i in range (50):\n",
    "\n",
    "    constant_bid = pbidprice(Cali_constant, base, 'constant')\n",
    "    #win_num, total_cost, click_num, ctr, cpc = evaluation(constant_bid, Payprice_train, clicks_train)\n",
    "\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(constant_bid, Payprice_val, clicks)\n",
    "    base+=1\n",
    "    \n",
    "    win_number.append(win_num)\n",
    "    totalcost.append(total_cost)\n",
    "    clicknum.append(click_num)\n",
    "    ctr_p.append(ctr)\n",
    "    cpc_p.append(cpc)\n",
    "    bid_p.append(constant_bid)\n",
    "    print(i+1,'/50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clicks 89.0 base 120\n"
     ]
    }
   ],
   "source": [
    "print('clicks',max(clicknum),'base',clicknum.index(max(clicknum))+100)\n",
    "best_clicks = max(clicknum)\n",
    "best_bid = clicknum.index(max(clicknum))+100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Bidding Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**RANDOM**\n",
      "win_num 105678.0, total_cost 6249.980000001479, click_num 88.0, ctr 0.0008327182573477924, cpc 71.0225000000168\n",
      "1 / 50\n",
      "**RANDOM**\n",
      "win_num 105669.0, total_cost 6249.965000001457, click_num 83.0, ctr 0.0007854716141914847, cpc 75.30078313254768\n",
      "2 / 50\n",
      "**RANDOM**\n",
      "win_num 105599.0, total_cost 6249.992000001482, click_num 85.0, ctr 0.0008049318648850842, cpc 73.52931764707625\n",
      "3 / 50\n",
      "**RANDOM**\n",
      "win_num 105800.0, total_cost 6249.987000001472, click_num 83.0, ctr 0.0007844990548204159, cpc 75.30104819278881\n",
      "4 / 50\n",
      "**RANDOM**\n",
      "win_num 105574.0, total_cost 6249.970000001465, click_num 83.0, ctr 0.0007861784151400914, cpc 75.30084337351163\n",
      "5 / 50\n",
      "**RANDOM**\n",
      "win_num 105812.0, total_cost 6249.986000001495, click_num 86.0, ctr 0.0008127622575889313, cpc 72.67425581397087\n",
      "6 / 50\n",
      "**RANDOM**\n",
      "win_num 105640.0, total_cost 6249.992000001489, click_num 84.0, ctr 0.0007951533510034078, cpc 74.40466666668439\n",
      "7 / 50\n",
      "**RANDOM**\n",
      "win_num 105854.0, total_cost 6249.965000001487, click_num 86.0, ctr 0.0008124397755398945, cpc 72.67401162792427\n",
      "8 / 50\n",
      "**RANDOM**\n",
      "win_num 105610.0, total_cost 6249.983000001465, click_num 84.0, ctr 0.0007953792254521352, cpc 74.40455952382698\n",
      "9 / 50\n",
      "**RANDOM**\n",
      "win_num 105454.0, total_cost 6249.991000001463, click_num 85.0, ctr 0.0008060386519240616, cpc 73.52930588237015\n",
      "10 / 50\n",
      "**RANDOM**\n",
      "win_num 105723.0, total_cost 6249.991000001471, click_num 83.0, ctr 0.0007850704198707944, cpc 75.30109638555989\n",
      "11 / 50\n",
      "**RANDOM**\n",
      "win_num 105627.0, total_cost 6249.965000001473, click_num 82.0, ctr 0.0007763166614596647, cpc 76.21908536587162\n",
      "12 / 50\n",
      "**RANDOM**\n",
      "win_num 105456.0, total_cost 6249.94500000146, click_num 82.0, ctr 0.0007775754817174936, cpc 76.21884146343244\n",
      "13 / 50\n",
      "**RANDOM**\n",
      "win_num 105663.0, total_cost 6249.957000001488, click_num 86.0, ctr 0.0008139083690601251, cpc 72.67391860466847\n",
      "14 / 50\n",
      "**RANDOM**\n",
      "win_num 105715.0, total_cost 6249.971000001485, click_num 86.0, ctr 0.000813508016837724, cpc 72.6740813953661\n",
      "15 / 50\n",
      "**RANDOM**\n",
      "win_num 105793.0, total_cost 6249.968000001488, click_num 85.0, ctr 0.0008034558052045031, cpc 73.52903529413516\n",
      "16 / 50\n",
      "**RANDOM**\n",
      "win_num 105683.0, total_cost 6249.9660000015, click_num 81.0, ctr 0.0007664430419272731, cpc 77.16007407409259\n",
      "17 / 50\n",
      "**RANDOM**\n",
      "win_num 105662.0, total_cost 6249.978000001486, click_num 88.0, ctr 0.00083284435274744, cpc 71.02247727274415\n",
      "18 / 50\n",
      "**RANDOM**\n",
      "win_num 105734.0, total_cost 6249.92600000148, click_num 81.0, ctr 0.0007660733538880588, cpc 77.15958024693185\n",
      "19 / 50\n",
      "**RANDOM**\n",
      "win_num 105828.0, total_cost 6249.949000001454, click_num 85.0, ctr 0.0008031900820198813, cpc 73.52881176472299\n",
      "20 / 50\n",
      "**RANDOM**\n",
      "win_num 105688.0, total_cost 6249.9340000014645, click_num 89.0, ctr 0.0008421012792369995, cpc 70.22397752810635\n",
      "21 / 50\n",
      "**RANDOM**\n",
      "win_num 105557.0, total_cost 6249.955000001476, click_num 86.0, ctr 0.0008147256932273558, cpc 72.67389534885437\n",
      "22 / 50\n",
      "**RANDOM**\n",
      "win_num 105629.0, total_cost 6249.985000001484, click_num 83.0, ctr 0.0007857690596332447, cpc 75.30102409640342\n",
      "23 / 50\n",
      "**RANDOM**\n",
      "win_num 105801.0, total_cost 6249.965000001486, click_num 85.0, ctr 0.0008033950529768149, cpc 73.52900000001749\n",
      "24 / 50\n",
      "**RANDOM**\n",
      "win_num 105596.0, total_cost 6249.971000001461, click_num 83.0, ctr 0.000786014621765976, cpc 75.30085542170436\n",
      "25 / 50\n",
      "**RANDOM**\n",
      "win_num 105651.0, total_cost 6249.975000001493, click_num 87.0, ctr 0.0008234659397450095, cpc 71.83879310346543\n",
      "26 / 50\n",
      "**RANDOM**\n",
      "win_num 105657.0, total_cost 6249.987000001479, click_num 82.0, ctr 0.0007760962359332557, cpc 76.21935365855462\n",
      "27 / 50\n",
      "**RANDOM**\n",
      "win_num 105587.0, total_cost 6249.974000001459, click_num 85.0, ctr 0.0008050233456770246, cpc 73.5291058823701\n",
      "28 / 50\n",
      "**RANDOM**\n",
      "win_num 105708.0, total_cost 6249.886000001462, click_num 86.0, ctr 0.0008135618874635789, cpc 72.67309302327281\n",
      "29 / 50\n",
      "**RANDOM**\n",
      "win_num 105557.0, total_cost 6249.987000001465, click_num 84.0, ctr 0.0007957785840825336, cpc 74.40460714287458\n",
      "30 / 50\n",
      "**RANDOM**\n",
      "win_num 105540.0, total_cost 6249.987000001489, click_num 84.0, ctr 0.0007959067652075042, cpc 74.40460714287487\n",
      "31 / 50\n",
      "**RANDOM**\n",
      "win_num 105759.0, total_cost 6249.974000001491, click_num 79.0, ctr 0.0007469813443773107, cpc 79.11359493672774\n",
      "32 / 50\n",
      "**RANDOM**\n",
      "win_num 105713.0, total_cost 6249.990000001469, click_num 83.0, ctr 0.0007851446841921051, cpc 75.30108433736709\n",
      "33 / 50\n",
      "**RANDOM**\n",
      "win_num 105734.0, total_cost 6249.984000001468, click_num 79.0, ctr 0.0007471579624340326, cpc 79.11372151900592\n",
      "34 / 50\n",
      "**RANDOM**\n",
      "win_num 105628.0, total_cost 6249.980000001481, click_num 80.0, ctr 0.0007573749384632863, cpc 78.12475000001851\n",
      "35 / 50\n",
      "**RANDOM**\n",
      "win_num 105846.0, total_cost 6249.995000001473, click_num 84.0, ctr 0.0007936058046595998, cpc 74.40470238096992\n",
      "36 / 50\n",
      "**RANDOM**\n",
      "win_num 105627.0, total_cost 6249.980000001482, click_num 85.0, ctr 0.0008047184905374573, cpc 73.52917647060568\n",
      "37 / 50\n",
      "**RANDOM**\n",
      "win_num 105594.0, total_cost 6249.982000001515, click_num 83.0, ctr 0.0007860295092524197, cpc 75.30098795182549\n",
      "38 / 50\n",
      "**RANDOM**\n",
      "win_num 105724.0, total_cost 6249.96500000148, click_num 87.0, ctr 0.0008228973553781545, cpc 71.83867816093655\n",
      "39 / 50\n",
      "**RANDOM**\n",
      "win_num 105733.0, total_cost 6249.967000001477, click_num 84.0, ctr 0.0007944539547728713, cpc 74.40436904763662\n",
      "40 / 50\n",
      "**RANDOM**\n",
      "win_num 105783.0, total_cost 6249.962000001472, click_num 85.0, ctr 0.0008035317584110869, cpc 73.52896470589967\n",
      "41 / 50\n",
      "**RANDOM**\n",
      "win_num 105928.0, total_cost 6249.928000001484, click_num 83.0, ctr 0.000783551091307303, cpc 75.30033734941547\n",
      "42 / 50\n",
      "**RANDOM**\n",
      "win_num 105620.0, total_cost 6249.948000001483, click_num 82.0, ctr 0.000776368112099981, cpc 76.21887804879857\n",
      "43 / 50\n",
      "**RANDOM**\n",
      "win_num 105534.0, total_cost 6249.960000001458, click_num 85.0, ctr 0.0008054276346959274, cpc 73.52894117648773\n",
      "44 / 50\n",
      "**RANDOM**\n",
      "win_num 105656.0, total_cost 6249.894000001474, click_num 87.0, ctr 0.0008234269705459226, cpc 71.83786206898246\n",
      "45 / 50\n",
      "**RANDOM**\n",
      "win_num 105707.0, total_cost 6249.9510000014825, click_num 83.0, ctr 0.0007851892495293595, cpc 75.30061445784919\n",
      "46 / 50\n",
      "**RANDOM**\n",
      "win_num 105714.0, total_cost 6249.98800000149, click_num 79.0, ctr 0.0007472993170251812, cpc 79.1137721519176\n",
      "47 / 50\n",
      "**RANDOM**\n",
      "win_num 105491.0, total_cost 6249.937000001472, click_num 86.0, ctr 0.0008152354229270743, cpc 72.67368604652874\n",
      "48 / 50\n",
      "**RANDOM**\n",
      "win_num 105601.0, total_cost 6249.9810000014695, click_num 81.0, ctr 0.0007670381909262223, cpc 77.1602592592774\n",
      "49 / 50\n",
      "**RANDOM**\n",
      "win_num 105554.0, total_cost 6249.959000001488, click_num 82.0, ctr 0.0007768535536313167, cpc 76.2190121951401\n",
      "50 / 50\n"
     ]
    }
   ],
   "source": [
    "rand_time = 50 \n",
    "\n",
    "base = 1 # in random strategy, base is not used\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "Cali_random = [[]]*len(Payprice_val)\n",
    "#Cali_random = [[]]*len(Payprice_train)\n",
    "\n",
    "\n",
    "for i in range (rand_time):\n",
    "\n",
    "    random_bid = pbidprice(Cali_random, base, 'random')\n",
    "    #win_num, total_cost, click_num, ctr, cpc = evaluation(random_bid, Payprice_train, clicks_train)\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(random_bid, Payprice_val, clicks)\n",
    "    \n",
    "    win_number.append(win_num)\n",
    "    totalcost.append(total_cost)\n",
    "    clicknum.append(click_num)\n",
    "    ctr_p.append(ctr)\n",
    "    cpc_p.append(cpc)\n",
    "    bid_p.append(random_bid)\n",
    "    print(i+1,'/', rand_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_num 105669.82, total_cost 6249.967160001479, click_num 84.0, ctr 0.0007943616372548433, cpc 74.51479996684266\n"
     ]
    }
   ],
   "source": [
    "rand_a = sum(win_number)/rand_time\n",
    "rand_b = sum(totalcost)/rand_time\n",
    "rand_c = sum(clicknum)/rand_time\n",
    "rand_d = sum(ctr_p)/rand_time\n",
    "rand_e = sum(cpc_p)/rand_time\n",
    "print('win_num {}, total_cost {}, click_num {}, ctr {}, cpc {}'.format(rand_a, rand_b, np.round(rand_c), rand_d, rand_e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gated Bidding Strategy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "1 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "2 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "3 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "4 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "5 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "6 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "7 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "8 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "9 /10\n",
      "**GATED**\n",
      "win_num 30910.0, total_cost 1818.069999999824, click_num 113.0, ctr 0.0036557748301520544, cpc 16.08911504424623\n",
      "10 /10\n"
     ]
    }
   ],
   "source": [
    "#base = 124.\n",
    "base = 100                                                                                                                                                                                                                                                            \n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "bid_p = []\n",
    "#Cali_constant = [[]]*len(Payprice_val)\n",
    "\n",
    "\n",
    "for i in range (10):\n",
    "\n",
    "    bidprice = pbidprice(Cali_bst, base, 'gated')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bidprice, Payprice_val, clicks)\n",
    "    base+=1\n",
    "    \n",
    "    win_number.append(win_num)\n",
    "    totalcost.append(total_cost)\n",
    "    clicknum.append(click_num)\n",
    "    ctr_p.append(ctr)\n",
    "    cpc_p.append(cpc)\n",
    "    bid_p.append(bidprice)\n",
    "    print(i+1,'/10')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Squared Bidding Strategy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "130.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "131.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "132.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "133.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "134.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "135.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "136.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "137.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "138.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 1973.0, total_cost 149.21900000000002, click_num 100.0, ctr 0.05068423720223011, cpc 1.4921900000000001\n",
      "139.0\n"
     ]
    }
   ],
   "source": [
    "base = 130. #139\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "\n",
    "rf_model = pickle.load(open('random_forest_model.sav', 'rb'))\n",
    "Cali_rf = Cali_CTR(rf_model.predict(X_val_FM), 0.0075)\n",
    "\n",
    "for i in range(10):\n",
    "    bid_p = pbidprice(Cali_rf, base, 'squared')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#stacking\n",
    "win_num 89099.0, total_cost 6499.991000000484, click_num 186.0, ctr 0.0020875655170091694, cpc 34.94618817204561\n",
    "224.0\n",
    "#rf\n",
    "win_num 114513.0, total_cost 6499.993000001925, click_num 185.0, ctr 0.0016155371005911992, cpc 35.1350972973077\n",
    "170.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use single model in Linear Bidding Strategy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 111324.0, total_cost 5615.257000002218, click_num 178.0, ctr 0.0015989364377852035, cpc 31.546387640461898\n",
      "200.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 111716.0, total_cost 5639.916000002247, click_num 178.0, ctr 0.001593325933617387, cpc 31.68492134832723\n",
      "201.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 112118.0, total_cost 5662.718000002259, click_num 178.0, ctr 0.0015876130505360425, cpc 31.813022471922803\n",
      "202.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 112588.0, total_cost 5687.9100000022745, click_num 178.0, ctr 0.001580985540199666, cpc 31.95455056181053\n",
      "203.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 113110.0, total_cost 5716.12900000229, click_num 178.0, ctr 0.0015736893289717974, cpc 32.11308426967579\n",
      "204.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 113584.0, total_cost 5741.1570000023075, click_num 178.0, ctr 0.0015671221298774476, cpc 32.25369101124892\n",
      "205.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 114049.0, total_cost 5766.251000002321, click_num 178.0, ctr 0.0015607326675376374, cpc 32.39466853933888\n",
      "206.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 114416.0, total_cost 5787.864000002341, click_num 178.0, ctr 0.0015557264718221227, cpc 32.5160898876536\n",
      "207.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 114847.0, total_cost 5812.302000002348, click_num 178.0, ctr 0.0015498881120098915, cpc 32.653382022485104\n",
      "208.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 115306.0, total_cost 5833.890000002365, click_num 178.0, ctr 0.0015437184535063223, cpc 32.7746629213616\n",
      "209.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 115898.0, total_cost 5863.310000002376, click_num 180.0, ctr 0.0015530897858461751, cpc 32.573944444457645\n",
      "210.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 116346.0, total_cost 5886.119000002397, click_num 181.0, ctr 0.0015557045364688086, cpc 32.519994475151364\n",
      "211.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 116743.0, total_cost 5906.752000002406, click_num 181.0, ctr 0.0015504141575940313, cpc 32.633988950289535\n",
      "212.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 117146.0, total_cost 5929.9410000024245, click_num 181.0, ctr 0.0015450804978403019, cpc 32.76210497238909\n",
      "213.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 117504.0, total_cost 5952.209000002444, click_num 181.0, ctr 0.0015403730936819173, cpc 32.88513259669858\n",
      "214.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 117882.0, total_cost 5975.292000002473, click_num 182.0, ctr 0.0015439167981540862, cpc 32.83127472528832\n",
      "215.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 118234.0, total_cost 5996.186000002491, click_num 182.0, ctr 0.0015393203308692932, cpc 32.94607692309061\n",
      "216.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 118560.0, total_cost 6014.406000002506, click_num 182.0, ctr 0.0015350877192982456, cpc 33.04618681320058\n",
      "217.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 118928.0, total_cost 6034.272000002525, click_num 182.0, ctr 0.001530337683304184, cpc 33.15534065935453\n",
      "218.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 119359.0, total_cost 6058.966000002538, click_num 182.0, ctr 0.0015248117025109126, cpc 33.29102197803592\n",
      "219.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 119684.0, total_cost 6081.244000002556, click_num 182.0, ctr 0.0015206711005648207, cpc 33.413428571442616\n",
      "220.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 120004.0, total_cost 6100.612000002559, click_num 182.0, ctr 0.0015166161127962402, cpc 33.519846153860215\n",
      "221.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 120319.0, total_cost 6120.731000002565, click_num 182.0, ctr 0.0015126455505780467, cpc 33.630390109904205\n",
      "222.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 120623.0, total_cost 6141.053000002568, click_num 182.0, ctr 0.001508833307080739, cpc 33.74204945056356\n",
      "223.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 120955.0, total_cost 6162.10600000257, click_num 183.0, ctr 0.001512959365053119, cpc 33.672710382527704\n",
      "224.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 121219.0, total_cost 6179.261000002577, click_num 183.0, ctr 0.0015096643265494682, cpc 33.76645355192665\n",
      "225.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 121486.0, total_cost 6194.849000002577, click_num 183.0, ctr 0.0015063464102859588, cpc 33.8516338797955\n",
      "226.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 121775.0, total_cost 6212.247000002578, click_num 183.0, ctr 0.0015027715048244713, cpc 33.946704918046876\n",
      "227.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 122013.0, total_cost 6227.22800000259, click_num 183.0, ctr 0.0014998401809643235, cpc 34.028568306025086\n",
      "228.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 122258.0, total_cost 6242.973000002588, click_num 183.0, ctr 0.0014968345629733841, cpc 34.11460655739119\n",
      "229.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 122582.0, total_cost 6261.939000002594, click_num 183.0, ctr 0.0014928782366089638, cpc 34.21824590165352\n",
      "230.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 122863.0, total_cost 6279.979000002598, click_num 184.0, ctr 0.0014976030212513125, cpc 34.13032065218803\n",
      "231.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 123194.0, total_cost 6300.375000002603, click_num 185.0, ctr 0.001501696511193727, cpc 34.05608108109515\n",
      "232.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 123449.0, total_cost 6316.320000002599, click_num 185.0, ctr 0.001498594561316819, cpc 34.14227027028432\n",
      "233.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 123785.0, total_cost 6334.962000002614, click_num 185.0, ctr 0.0014945268005008684, cpc 34.24303783785197\n",
      "234.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 124106.0, total_cost 6355.849000002614, click_num 185.0, ctr 0.0014906612089665287, cpc 34.35594054055467\n",
      "235.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 124391.0, total_cost 6374.359000002615, click_num 186.0, ctr 0.0014952850286596296, cpc 34.270747311842015\n",
      "236.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 124667.0, total_cost 6392.984000002617, click_num 186.0, ctr 0.0014919746203887155, cpc 34.370881720444174\n",
      "237.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 124892.0, total_cost 6407.852000002612, click_num 186.0, ctr 0.0014892867437465971, cpc 34.450817204315115\n",
      "238.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 125104.0, total_cost 6422.314000002628, click_num 186.0, ctr 0.00148676301317304, cpc 34.52856989248725\n",
      "239.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 125322.0, total_cost 6436.384000002632, click_num 186.0, ctr 0.0014841767606645281, cpc 34.604215053777594\n",
      "240.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 125572.0, total_cost 6452.016000002639, click_num 186.0, ctr 0.001481221928455388, cpc 34.68825806453032\n",
      "241.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 125839.0, total_cost 6467.93600000264, click_num 186.0, ctr 0.001478079132860242, cpc 34.773849462379786\n",
      "242.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126121.0, total_cost 6484.880000002637, click_num 186.0, ctr 0.0014747742247524203, cpc 34.86494623657332\n",
      "243.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126348.0, total_cost 6499.984000002629, click_num 186.0, ctr 0.0014721246082249027, cpc 34.94615053764854\n",
      "244.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126286.0, total_cost 6499.986000002618, click_num 185.0, ctr 0.0014649288123782524, cpc 35.13505945947361\n",
      "245.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126237.0, total_cost 6499.809000002604, click_num 185.0, ctr 0.0014654974373598867, cpc 35.13410270271678\n",
      "246.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126212.0, total_cost 6499.903000002599, click_num 184.0, ctr 0.0014578645453681108, cpc 35.32555978262282\n",
      "247.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126198.0, total_cost 6499.980000002602, click_num 183.0, ctr 0.0014501022203204487, cpc 35.51901639345684\n",
      "248.0\n",
      "this is a **SQUARED** strategy to get predict bid price\n",
      "win_num 126124.0, total_cost 6499.902000002593, click_num 182.0, ctr 0.0014430243252671973, cpc 35.7137472527615\n",
      "249.0\n"
     ]
    }
   ],
   "source": [
    "base = 200. #139\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "\n",
    "for i in range(50):\n",
    "    bid_p = pbidprice(Cali_rf, base, 'squared')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " **LINEAR** \n",
      "win_num 97976.0, total_cost 6215.379000001488, click_num 175.0, ctr 0.0017861517106230098, cpc 35.51645142857993\n",
      "400.0\n",
      " **LINEAR** \n",
      "win_num 98068.0, total_cost 6222.983000001484, click_num 175.0, ctr 0.001784476077823551, cpc 35.559902857151336\n",
      "401.0\n",
      " **LINEAR** \n",
      "win_num 98167.0, total_cost 6231.493000001492, click_num 175.0, ctr 0.0017826764595026841, cpc 35.60853142857995\n",
      "402.0\n",
      " **LINEAR** \n",
      "win_num 98275.0, total_cost 6239.987000001494, click_num 175.0, ctr 0.0017807173747138134, cpc 35.65706857143711\n",
      "403.0\n",
      " **LINEAR** \n",
      "win_num 98375.0, total_cost 6248.123000001502, click_num 175.0, ctr 0.0017789072426937739, cpc 35.703560000008586\n",
      "404.0\n",
      " **LINEAR** \n",
      "win_num 98375.0, total_cost 6249.892000001503, click_num 175.0, ctr 0.0017789072426937739, cpc 35.71366857143716\n",
      "405.0\n",
      " **LINEAR** \n",
      "win_num 98356.0, total_cost 6249.987000001495, click_num 174.0, ctr 0.0017690837366302005, cpc 35.919465517249975\n",
      "406.0\n",
      " **LINEAR** \n",
      "win_num 98328.0, total_cost 6249.9910000015, click_num 174.0, ctr 0.001769587503051013, cpc 35.919488505755744\n",
      "407.0\n",
      " **LINEAR** \n",
      "win_num 98289.0, total_cost 6249.971000001492, click_num 174.0, ctr 0.0017702896560144064, cpc 35.91937356322696\n",
      "408.0\n",
      " **LINEAR** \n",
      "win_num 98233.0, total_cost 6249.988000001486, click_num 173.0, ctr 0.0017611189722394715, cpc 36.12709826590454\n",
      "409.0\n",
      " **LINEAR** \n",
      "win_num 98201.0, total_cost 6249.932000001489, click_num 173.0, ctr 0.001761692854451584, cpc 36.126774566482595\n",
      "410.0\n",
      " **LINEAR** \n",
      "win_num 98185.0, total_cost 6249.892000001482, click_num 173.0, ctr 0.0017619799358354127, cpc 36.12654335260972\n",
      "411.0\n",
      " **LINEAR** \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-227-5c806e7600ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mbid_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpbidprice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCali_nn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'linear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mwin_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_cost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclick_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcpc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbid_p\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mPayprice_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclicks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m#     win_number.append(win_num)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-143-45ba727807a1>\u001b[0m in \u001b[0;36mevaluation\u001b[0;34m(pBidprice, payprice, clicks, budget)\u001b[0m\n\u001b[1;32m      7\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtotal_cost\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpayprice\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m1000.\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbudget\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m             \u001b[0mwin_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwin_num\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m             \u001b[0mtotal_cost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtotal_cost\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpayprice\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m1000.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m             \u001b[0mclick_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mclicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "base =400. #139\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "\n",
    "for i in range(50):\n",
    "    bid_p = pbidprice(Cali_nn, base, 'linear')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " **LINEAR** \n",
      "win_num 101710.0, total_cost 5814.7860000013015, click_num 184.0, ctr 0.0018090649886933438, cpc 31.60209782609403\n",
      "150.0\n",
      " **LINEAR** \n",
      "win_num 102129.0, total_cost 5844.401000001307, click_num 185.0, ctr 0.0018114345582547562, cpc 31.59135675676382\n",
      "151.0\n",
      " **LINEAR** \n",
      "win_num 102548.0, total_cost 5873.363000001311, click_num 185.0, ctr 0.0018040332332176152, cpc 31.747908108115194\n",
      "152.0\n",
      " **LINEAR** \n",
      "win_num 103002.0, total_cost 5904.54500000132, click_num 185.0, ctr 0.0017960816294829228, cpc 31.916459459466594\n",
      "153.0\n",
      " **LINEAR** \n",
      "win_num 103426.0, total_cost 5935.272000001313, click_num 185.0, ctr 0.0017887185040512056, cpc 32.08255135135845\n",
      "154.0\n",
      " **LINEAR** \n",
      "win_num 103833.0, total_cost 5965.851000001318, click_num 185.0, ctr 0.0017817071643889707, cpc 32.24784324325037\n",
      "155.0\n",
      " **LINEAR** \n",
      "win_num 104290.0, total_cost 5999.905000001322, click_num 185.0, ctr 0.0017738997027519417, cpc 32.431918918926065\n",
      "156.0\n",
      " **LINEAR** \n",
      "win_num 104713.0, total_cost 6033.4610000013245, click_num 185.0, ctr 0.0017667338343854153, cpc 32.61330270270986\n",
      "157.0\n",
      " **LINEAR** \n",
      "win_num 105107.0, total_cost 6062.30700000133, click_num 185.0, ctr 0.0017601111248537204, cpc 32.76922702703422\n",
      "158.0\n",
      " **LINEAR** \n",
      "win_num 105513.0, total_cost 6091.838000001334, click_num 185.0, ctr 0.0017533384511861099, cpc 32.92885405406126\n",
      "159.0\n",
      " **LINEAR** \n",
      "win_num 105919.0, total_cost 6120.7460000013425, click_num 185.0, ctr 0.0017466176984299324, cpc 33.08511351352077\n",
      "160.0\n",
      " **LINEAR** \n",
      "win_num 106329.0, total_cost 6149.736000001357, click_num 185.0, ctr 0.001739882816541113, cpc 33.24181621622355\n",
      "161.0\n",
      " **LINEAR** \n",
      "win_num 106751.0, total_cost 6182.026000001357, click_num 185.0, ctr 0.0017330048430459668, cpc 33.41635675676409\n",
      "162.0\n",
      " **LINEAR** \n",
      "win_num 107155.0, total_cost 6209.05900000136, click_num 185.0, ctr 0.0017264709999533386, cpc 33.56248108108843\n",
      "163.0\n",
      " **LINEAR** \n",
      "win_num 107563.0, total_cost 6238.26900000138, click_num 185.0, ctr 0.001719922278106784, cpc 33.720372972980435\n",
      "164.0\n",
      " **LINEAR** \n",
      "win_num 107654.0, total_cost 6249.979000001377, click_num 184.0, ctr 0.0017091794081037397, cpc 33.96727717392053\n",
      "165.0\n",
      " **LINEAR** \n",
      "win_num 107560.0, total_cost 6249.811000001381, click_num 183.0, ctr 0.0017013759761993306, cpc 34.151972677603176\n",
      "166.0\n",
      " **LINEAR** \n",
      "win_num 107526.0, total_cost 6249.93600000138, click_num 182.0, ctr 0.0016926138794338113, cpc 34.34030769231528\n",
      "167.0\n",
      " **LINEAR** \n",
      "win_num 107409.0, total_cost 6249.964000001379, click_num 181.0, ctr 0.0016851474271243564, cpc 34.53018784531149\n",
      "168.0\n",
      " **LINEAR** \n",
      "win_num 107325.0, total_cost 6249.942000001378, click_num 179.0, ctr 0.0016678313533659446, cpc 34.91587709497976\n",
      "169.0\n",
      " **LINEAR** \n",
      "win_num 107229.0, total_cost 6249.923000001386, click_num 178.0, ctr 0.0016599986943830493, cpc 35.11192696629992\n",
      "170.0\n",
      " **LINEAR** \n",
      "win_num 107129.0, total_cost 6249.941000001387, click_num 177.0, ctr 0.0016522136863034286, cpc 35.31040112995134\n",
      "171.0\n",
      " **LINEAR** \n",
      "win_num 107051.0, total_cost 6249.989000001393, click_num 176.0, ctr 0.001644076187985166, cpc 35.51130113637155\n",
      "172.0\n",
      " **LINEAR** \n",
      "win_num 106963.0, total_cost 6249.845000001381, click_num 176.0, ctr 0.0016454287931340745, cpc 35.510482954553304\n",
      "173.0\n",
      " **LINEAR** \n",
      "win_num 106847.0, total_cost 6249.781000001372, click_num 175.0, ctr 0.0016378559997005063, cpc 35.71303428572213\n",
      "174.0\n",
      " **LINEAR** \n",
      "win_num 106760.0, total_cost 6249.955000001367, click_num 175.0, ctr 0.001639190708130386, cpc 35.71402857143638\n",
      "175.0\n",
      " **LINEAR** \n",
      "win_num 106682.0, total_cost 6249.96200000136, click_num 175.0, ctr 0.001640389194053355, cpc 35.71406857143634\n",
      "176.0\n",
      " **LINEAR** \n",
      "win_num 106507.0, total_cost 6249.987000001357, click_num 174.0, ctr 0.0016336954378585445, cpc 35.91946551724918\n",
      "177.0\n",
      " **LINEAR** \n",
      "win_num 106404.0, total_cost 6249.872000001349, click_num 175.0, ctr 0.0016446750122175858, cpc 35.713554285722\n",
      "178.0\n",
      " **LINEAR** \n",
      "win_num 106316.0, total_cost 6249.947000001345, click_num 174.0, ctr 0.001636630422514015, cpc 35.919235632191636\n",
      "179.0\n"
     ]
    }
   ],
   "source": [
    "base = 150. #172\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "for i in range(30):\n",
    "    bid_p = pbidprice(Cali_xgb, base, 'linear')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**SQUARED**\n",
      "win_num 100968.0, total_cost 5874.841000001382, click_num 182.0, ctr 0.00180255130338325, cpc 32.279346153853744\n",
      "180.0\n",
      "**SQUARED**\n",
      "win_num 101278.0, total_cost 5897.121000001387, click_num 183.0, ctr 0.0018069077193467486, cpc 32.224704918040366\n",
      "181.0\n",
      "**SQUARED**\n",
      "win_num 101577.0, total_cost 5915.252000001388, click_num 183.0, ctr 0.0018015889423786881, cpc 32.32378142077261\n",
      "182.0\n",
      "**SQUARED**\n",
      "win_num 101930.0, total_cost 5935.806000001393, click_num 183.0, ctr 0.0017953497498283135, cpc 32.43609836066335\n",
      "183.0\n",
      "**SQUARED**\n",
      "win_num 102231.0, total_cost 5955.6650000014, click_num 183.0, ctr 0.0017900636793144937, cpc 32.544617486346446\n",
      "184.0\n",
      "**SQUARED**\n",
      "win_num 102506.0, total_cost 5974.681000001404, click_num 183.0, ctr 0.0017852613505550895, cpc 32.64853005465248\n",
      "185.0\n",
      "**SQUARED**\n",
      "win_num 102804.0, total_cost 5995.524000001398, click_num 183.0, ctr 0.001780086377961947, cpc 32.76242622951584\n",
      "186.0\n",
      "**SQUARED**\n",
      "win_num 103108.0, total_cost 6017.0300000014, click_num 183.0, ctr 0.0017748380339061955, cpc 32.879945355198906\n",
      "187.0\n",
      "**SQUARED**\n",
      "win_num 103414.0, total_cost 6035.884000001409, click_num 183.0, ctr 0.0017695863229349991, cpc 32.98297267760333\n",
      "188.0\n",
      "**SQUARED**\n",
      "win_num 103703.0, total_cost 6057.78200000141, click_num 183.0, ctr 0.0017646548315863571, cpc 33.102633879789124\n",
      "189.0\n",
      "**SQUARED**\n",
      "win_num 104028.0, total_cost 6079.4570000014155, click_num 183.0, ctr 0.00175914176952359, cpc 33.221076502739976\n",
      "190.0\n",
      "**SQUARED**\n",
      "win_num 104319.0, total_cost 6099.205000001407, click_num 183.0, ctr 0.0017542346073102695, cpc 33.32898907104594\n",
      "191.0\n",
      "**SQUARED**\n",
      "win_num 104605.0, total_cost 6117.972000001415, click_num 183.0, ctr 0.0017494383633669518, cpc 33.43154098361429\n",
      "192.0\n",
      "**SQUARED**\n",
      "win_num 104897.0, total_cost 6137.19400000142, click_num 183.0, ctr 0.0017445684814627683, cpc 33.53657923498044\n",
      "193.0\n",
      "**SQUARED**\n",
      "win_num 105156.0, total_cost 6156.102000001428, click_num 183.0, ctr 0.001740271596485222, cpc 33.63990163935206\n",
      "194.0\n",
      "**SQUARED**\n",
      "win_num 105412.0, total_cost 6174.523000001432, click_num 184.0, ctr 0.0017455318180093348, cpc 33.55719021739908\n",
      "195.0\n",
      "**SQUARED**\n",
      "win_num 105695.0, total_cost 6194.4030000014345, click_num 184.0, ctr 0.0017408581295236293, cpc 33.66523369565997\n",
      "196.0\n",
      "**SQUARED**\n",
      "win_num 105964.0, total_cost 6213.578000001446, click_num 184.0, ctr 0.0017364387905326338, cpc 33.76944565218177\n",
      "197.0\n",
      "**SQUARED**\n",
      "win_num 106208.0, total_cost 6230.214000001461, click_num 184.0, ctr 0.001732449532991865, cpc 33.859858695660115\n",
      "198.0\n",
      "**SQUARED**\n",
      "win_num 106475.0, total_cost 6249.994000001461, click_num 184.0, ctr 0.001728105189011505, cpc 33.96735869566012\n",
      "199.0\n",
      "**SQUARED**\n",
      "win_num 106378.0, total_cost 6249.974000001462, click_num 183.0, ctr 0.0017202805091278272, cpc 34.15286338798613\n",
      "200.0\n",
      "**SQUARED**\n",
      "win_num 106376.0, total_cost 6249.912000001463, click_num 182.0, ctr 0.0017109122358426713, cpc 34.34017582418386\n",
      "201.0\n",
      "**SQUARED**\n",
      "win_num 106342.0, total_cost 6249.993000001463, click_num 181.0, ctr 0.0017020556318293808, cpc 34.53034806630642\n",
      "202.0\n",
      "**SQUARED**\n",
      "win_num 106265.0, total_cost 6249.824000001467, click_num 180.0, ctr 0.0016938785112689974, cpc 34.721244444452594\n",
      "203.0\n",
      "**SQUARED**\n",
      "win_num 106174.0, total_cost 6249.9910000014615, click_num 180.0, ctr 0.0016953303068547856, cpc 34.72217222223034\n",
      "204.0\n",
      "**SQUARED**\n",
      "win_num 106149.0, total_cost 6249.985000001465, click_num 178.0, ctr 0.0016768881477922542, cpc 35.112275280907106\n",
      "205.0\n",
      "**SQUARED**\n",
      "win_num 106131.0, total_cost 6249.958000001467, click_num 177.0, ctr 0.0016677502332023632, cpc 35.31049717514953\n",
      "206.0\n",
      "**SQUARED**\n",
      "win_num 106090.0, total_cost 6249.927000001469, click_num 176.0, ctr 0.0016589688000754076, cpc 35.510948863644714\n",
      "207.0\n",
      "**SQUARED**\n",
      "win_num 106083.0, total_cost 6249.9890000014675, click_num 175.0, ctr 0.0016496516878293413, cpc 35.71422285715124\n",
      "208.0\n",
      "**SQUARED**\n",
      "win_num 106089.0, total_cost 6249.9430000014745, click_num 174.0, ctr 0.00164013234171309, cpc 35.919212643686635\n",
      "209.0\n"
     ]
    }
   ],
   "source": [
    "base = 180. #136\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "for i in range(30):\n",
    "    bid_p = pbidprice(Cali_bst, base, 'squared')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### nn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_num 106058.0, total_cost 6127.671000001743, click_num 178.0, ctr 0.0016783269531765637, cpc 34.42511797753788\n",
      "170.0\n",
      "win_num 106395.0, total_cost 6153.728000001744, click_num 178.0, ctr 0.0016730109497626769, cpc 34.57150561798732\n",
      "171.0\n",
      "win_num 106718.0, total_cost 6180.110000001758, click_num 178.0, ctr 0.001667947300361701, cpc 34.71971910113347\n",
      "172.0\n",
      "win_num 107074.0, total_cost 6208.473000001765, click_num 179.0, ctr 0.0016717410389076714, cpc 34.684206703920474\n",
      "173.0\n",
      "win_num 107395.0, total_cost 6231.931000001777, click_num 179.0, ctr 0.0016667442618371433, cpc 34.81525698325015\n",
      "174.0\n",
      "win_num 107756.0, total_cost 6258.709000001785, click_num 179.0, ctr 0.001661160399420914, cpc 34.96485474861333\n",
      "175.0\n",
      "win_num 108099.0, total_cost 6286.171000001791, click_num 179.0, ctr 0.0016558895086911073, cpc 35.11827374302677\n",
      "176.0\n",
      "win_num 108419.0, total_cost 6309.834000001808, click_num 179.0, ctr 0.0016510021306228612, cpc 35.25046927375312\n",
      "177.0\n",
      "win_num 108728.0, total_cost 6333.712000001817, click_num 180.0, ctr 0.0016555073210212641, cpc 35.18728888889898\n",
      "178.0\n",
      "win_num 109038.0, total_cost 6357.548000001831, click_num 180.0, ctr 0.0016508006383095801, cpc 35.31971111112128\n",
      "179.0\n",
      "win_num 109316.0, total_cost 6380.283000001836, click_num 180.0, ctr 0.0016466025101540488, cpc 35.44601666667687\n",
      "180.0\n",
      "win_num 109624.0, total_cost 6403.49600000185, click_num 180.0, ctr 0.001641976209589141, cpc 35.57497777778806\n",
      "181.0\n",
      "win_num 109972.0, total_cost 6428.8860000018585, click_num 180.0, ctr 0.001636780271341796, cpc 35.71603333334366\n",
      "182.0\n",
      "win_num 110297.0, total_cost 6454.541000001866, click_num 181.0, ctr 0.0016410237812451835, cpc 35.66044751382247\n",
      "183.0\n",
      "win_num 110597.0, total_cost 6478.621000001878, click_num 181.0, ctr 0.0016365724205900704, cpc 35.79348618785568\n",
      "184.0\n",
      "win_num 110871.0, total_cost 6498.646000001877, click_num 181.0, ctr 0.0016325278927762895, cpc 35.904121546971695\n",
      "185.0\n"
     ]
    }
   ],
   "source": [
    "base = 170. #185\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "for i in range(16):\n",
    "    bid_p = pbidprice(Cali_nn, base, 'linear')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "    base += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 615,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('feature_selection', SelectFromModel(estimator=LogisticRegression(C=0.01, class_weight='balanced', dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=200,\n",
       "          multi_class='ovr', n_jobs=-1, penalty='l2', random_state=2017,\n",
       "          solver='liblinear', tol=0.0001, v...istic', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=2017, silent=True, subsample=1))])"
      ]
     },
     "execution_count": 615,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "clf1 = Pipeline([\n",
    "  ('feature_selection', SelectFromModel(LogisticRegression(C=0.01,#9,  #0.665045541881\n",
    "                                                           penalty='l2',\n",
    "                                                           class_weight='balanced',\n",
    "                                                           fit_intercept=True,\n",
    "                                                           intercept_scaling=1,\n",
    "                                                           max_iter=200,\n",
    "                                                           multi_class='ovr',\n",
    "                                                           n_jobs = -1,\n",
    "                                                           random_state=2017,\n",
    "                                                           solver='liblinear',\n",
    "                                                           verbose=True))),\n",
    "  ('classification', xgb.xgbclassifier(base_score=0.5, colsample_bylevel=1, colsample_bytree=1,\n",
    "       gamma=0, learning_rate=0.5, max_delta_step=0, max_depth=3,\n",
    "       min_child_weight=1, missing=None, n_estimators=100, nthread=-1,\n",
    "       objective='binary:logistic', reg_alpha=0, reg_lambda=1,\n",
    "       scale_pos_weight=1, seed=2017, silent=True, subsample=1))\n",
    "])\n",
    "clf1.fit(X_FM,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 616,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb_ada = clf1.predict_proba(X_val_FM)\n",
    "Cali_xgb_ada = Cali_CTR(y_pred_xgb_ada, 0.0075)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.869427300913\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_xgb_ada, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_xgb_ada)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Stacking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " for categorical features: LR, FM\n",
    " use FM to train, get encoded features\n",
    " use linear model like LR to train new features\n",
    "\n",
    " for continuous features: \n",
    " use GBDT(XGB) to turn continuous features to categorical features\n",
    "\n",
    "\n",
    "#linear\n",
    "---lr: 0.767409702075, 0.4\n",
    "---elas: 0.801852253342, 0.1\n",
    "sgd: 0.823197564068, 0.03\n",
    "\n",
    "#boost\n",
    "booster: 0.864006668834, 0.03\n",
    "xgboost: 0.898314588633, 0.03\n",
    "adaboost: 0.839688962382, 0.03\n",
    "\n",
    "#tree\n",
    "extra tree: 0.83478449023, 0.03\n",
    "random forest tree: 0.876310841613, 0.03\n",
    "\n",
    "nn: 0.844836254246, 0.04\n",
    "\n",
    "fm: 0.821892502294, 0.03"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Wold = 0.5\n",
    "w_xgb = 1.0 - Wold\n",
    "w_xgbc = 0.75 * Wold\n",
    "w_ada = 1 * Wold\n",
    "\n",
    "w_fm = 0.51 * Wold\n",
    "w_nn = 0.37 * Wold\n",
    "\n",
    "w_elas = 0.12 * Wold\n",
    "w_sgd = 1 * Wold\n",
    "\n",
    "w_ext = 1 * Wold\n",
    "w_rft = 0.25 * Wold\n",
    "\n",
    "\n",
    "x_train_stack = np.concatenate(\n",
    "                (np.log1p(w_xgb*Cali_bst.reshape(-1,1)), \n",
    "                 np.log1p(w_rft*Cali_rf.reshape(-1,1)), \n",
    "                 np.log1p(w_xgbc*Cali_xgb.reshape(-1,1)),\n",
    "                 #np.log1p(w_elas*Cali_elas.reshape(-1,1))\n",
    "                ), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-rmse:0.460769\teval-rmse:0.456051\n",
      "[1]\ttrain-rmse:0.426627\teval-rmse:0.416922\n",
      "[2]\ttrain-rmse:0.396816\teval-rmse:0.382042\n",
      "[3]\ttrain-rmse:0.370844\teval-rmse:0.350959\n",
      "[4]\ttrain-rmse:0.348303\teval-rmse:0.323199\n",
      "[5]\ttrain-rmse:0.328774\teval-rmse:0.298434\n",
      "[6]\ttrain-rmse:0.311938\teval-rmse:0.27639\n",
      "[7]\ttrain-rmse:0.29747\teval-rmse:0.256811\n",
      "[8]\ttrain-rmse:0.285041\teval-rmse:0.239305\n",
      "[9]\ttrain-rmse:0.274458\teval-rmse:0.223594\n",
      "[10]\ttrain-rmse:0.265497\teval-rmse:0.209648\n",
      "[11]\ttrain-rmse:0.257818\teval-rmse:0.197398\n",
      "[12]\ttrain-rmse:0.251303\teval-rmse:0.186475\n",
      "[13]\ttrain-rmse:0.245819\teval-rmse:0.176787\n",
      "[14]\ttrain-rmse:0.241111\teval-rmse:0.168123\n",
      "[15]\ttrain-rmse:0.237169\teval-rmse:0.160569\n",
      "[16]\ttrain-rmse:0.233817\teval-rmse:0.153872\n",
      "[17]\ttrain-rmse:0.230905\teval-rmse:0.147859\n",
      "[18]\ttrain-rmse:0.228527\teval-rmse:0.142651\n",
      "[19]\ttrain-rmse:0.226359\teval-rmse:0.137899\n",
      "[20]\ttrain-rmse:0.22451\teval-rmse:0.133782\n",
      "[21]\ttrain-rmse:0.223004\teval-rmse:0.130187\n",
      "[22]\ttrain-rmse:0.22169\teval-rmse:0.126964\n",
      "[23]\ttrain-rmse:0.220576\teval-rmse:0.124071\n",
      "[24]\ttrain-rmse:0.219553\teval-rmse:0.121532\n",
      "[25]\ttrain-rmse:0.218735\teval-rmse:0.119169\n",
      "[26]\ttrain-rmse:0.217969\teval-rmse:0.117046\n",
      "[27]\ttrain-rmse:0.217331\teval-rmse:0.115022\n",
      "[28]\ttrain-rmse:0.216781\teval-rmse:0.113276\n",
      "[29]\ttrain-rmse:0.216296\teval-rmse:0.111772\n",
      "[30]\ttrain-rmse:0.215938\teval-rmse:0.110358\n",
      "[31]\ttrain-rmse:0.215515\teval-rmse:0.109221\n",
      "[32]\ttrain-rmse:0.21505\teval-rmse:0.10822\n",
      "[33]\ttrain-rmse:0.214675\teval-rmse:0.107069\n",
      "[34]\ttrain-rmse:0.214315\teval-rmse:0.106333\n",
      "[35]\ttrain-rmse:0.213966\teval-rmse:0.10545\n",
      "[36]\ttrain-rmse:0.213614\teval-rmse:0.104847\n",
      "[37]\ttrain-rmse:0.21334\teval-rmse:0.104143\n",
      "[38]\ttrain-rmse:0.21305\teval-rmse:0.103619\n",
      "[39]\ttrain-rmse:0.212796\teval-rmse:0.103107\n",
      "[40]\ttrain-rmse:0.212563\teval-rmse:0.102751\n",
      "[41]\ttrain-rmse:0.212306\teval-rmse:0.102365\n",
      "[42]\ttrain-rmse:0.212129\teval-rmse:0.102056\n",
      "[43]\ttrain-rmse:0.211936\teval-rmse:0.101693\n",
      "[44]\ttrain-rmse:0.211718\teval-rmse:0.101283\n",
      "[45]\ttrain-rmse:0.211557\teval-rmse:0.10098\n",
      "[46]\ttrain-rmse:0.211373\teval-rmse:0.100736\n",
      "[47]\ttrain-rmse:0.211173\teval-rmse:0.100565\n",
      "[48]\ttrain-rmse:0.21102\teval-rmse:0.100328\n",
      "[49]\ttrain-rmse:0.210868\teval-rmse:0.100241\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'Booster' object has no attribute 'fit'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-235-272cf38730b9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mstack_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train_stack\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'Booster' object has no attribute 'fit'"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "watchlist = [(dtrain, 'train'), (dval, 'eval')]\n",
    "stack_model = xgb.train(xgb_params, dtrain, num_round, watchlist)\n",
    "\n",
    "#stack_model = LinearRegression()\n",
    "\n",
    "\n",
    "\n",
    "#stack_model.fit(x_train_stack, target_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#predict_xgbssss = stack_model.predict(X_val)\n",
    "predict_xgbssss = stack_model.predict(dval)\n",
    "#Cali_lin = Cali_CTR(linreg_model.predict(X_val_FM), 0.0075)\n",
    "\n",
    "Cali_stack = Cali_CTR(predict_xgbssss, 0.0075)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.01258515,  0.35675454,  0.13332993, ...,  0.07875905,\n",
       "        0.04661544,  0.01455905], dtype=float32)"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_xgbssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.893795988128\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_stack, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_stack)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "226"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_validation[\"click\"]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**SQUARED**\n",
      "win_num 100968.0, total_cost 5874.841000001382, click_num 182.0, ctr 0.00180255130338325, cpc 32.279346153853744\n",
      "180\n",
      "**SQUARED**\n",
      "win_num 101278.0, total_cost 5897.121000001387, click_num 183.0, ctr 0.0018069077193467486, cpc 32.224704918040366\n",
      "181\n",
      "**SQUARED**\n",
      "win_num 101577.0, total_cost 5915.252000001388, click_num 183.0, ctr 0.0018015889423786881, cpc 32.32378142077261\n",
      "182\n",
      "**SQUARED**\n",
      "win_num 101930.0, total_cost 5935.806000001393, click_num 183.0, ctr 0.0017953497498283135, cpc 32.43609836066335\n",
      "183\n",
      "**SQUARED**\n",
      "win_num 102231.0, total_cost 5955.6650000014, click_num 183.0, ctr 0.0017900636793144937, cpc 32.544617486346446\n",
      "184\n",
      "**SQUARED**\n",
      "win_num 102506.0, total_cost 5974.681000001404, click_num 183.0, ctr 0.0017852613505550895, cpc 32.64853005465248\n",
      "185\n",
      "**SQUARED**\n",
      "win_num 102804.0, total_cost 5995.524000001398, click_num 183.0, ctr 0.001780086377961947, cpc 32.76242622951584\n",
      "186\n",
      "**SQUARED**\n",
      "win_num 103108.0, total_cost 6017.0300000014, click_num 183.0, ctr 0.0017748380339061955, cpc 32.879945355198906\n",
      "187\n",
      "**SQUARED**\n",
      "win_num 103414.0, total_cost 6035.884000001409, click_num 183.0, ctr 0.0017695863229349991, cpc 32.98297267760333\n",
      "188\n",
      "**SQUARED**\n",
      "win_num 103703.0, total_cost 6057.78200000141, click_num 183.0, ctr 0.0017646548315863571, cpc 33.102633879789124\n",
      "189\n",
      "**SQUARED**\n",
      "win_num 104028.0, total_cost 6079.4570000014155, click_num 183.0, ctr 0.00175914176952359, cpc 33.221076502739976\n",
      "190\n",
      "**SQUARED**\n",
      "win_num 104319.0, total_cost 6099.205000001407, click_num 183.0, ctr 0.0017542346073102695, cpc 33.32898907104594\n",
      "191\n",
      "**SQUARED**\n",
      "win_num 104605.0, total_cost 6117.972000001415, click_num 183.0, ctr 0.0017494383633669518, cpc 33.43154098361429\n",
      "192\n",
      "**SQUARED**\n",
      "win_num 104897.0, total_cost 6137.19400000142, click_num 183.0, ctr 0.0017445684814627683, cpc 33.53657923498044\n",
      "193\n",
      "**SQUARED**\n",
      "win_num 105156.0, total_cost 6156.102000001428, click_num 183.0, ctr 0.001740271596485222, cpc 33.63990163935206\n",
      "194\n",
      "**SQUARED**\n",
      "win_num 105412.0, total_cost 6174.523000001432, click_num 184.0, ctr 0.0017455318180093348, cpc 33.55719021739908\n",
      "195\n",
      "**SQUARED**\n",
      "win_num 105695.0, total_cost 6194.4030000014345, click_num 184.0, ctr 0.0017408581295236293, cpc 33.66523369565997\n",
      "196\n",
      "**SQUARED**\n",
      "win_num 105964.0, total_cost 6213.578000001446, click_num 184.0, ctr 0.0017364387905326338, cpc 33.76944565218177\n",
      "197\n",
      "**SQUARED**\n",
      "win_num 106208.0, total_cost 6230.214000001461, click_num 184.0, ctr 0.001732449532991865, cpc 33.859858695660115\n",
      "198\n",
      "**SQUARED**\n",
      "win_num 106475.0, total_cost 6249.994000001461, click_num 184.0, ctr 0.001728105189011505, cpc 33.96735869566012\n",
      "199\n",
      "**SQUARED**\n",
      "win_num 106378.0, total_cost 6249.974000001462, click_num 183.0, ctr 0.0017202805091278272, cpc 34.15286338798613\n",
      "200\n",
      "**SQUARED**\n",
      "win_num 106376.0, total_cost 6249.912000001463, click_num 182.0, ctr 0.0017109122358426713, cpc 34.34017582418386\n",
      "201\n",
      "**SQUARED**\n",
      "win_num 106342.0, total_cost 6249.993000001463, click_num 181.0, ctr 0.0017020556318293808, cpc 34.53034806630642\n",
      "202\n",
      "**SQUARED**\n",
      "win_num 106265.0, total_cost 6249.824000001467, click_num 180.0, ctr 0.0016938785112689974, cpc 34.721244444452594\n",
      "203\n",
      "**SQUARED**\n",
      "win_num 106174.0, total_cost 6249.9910000014615, click_num 180.0, ctr 0.0016953303068547856, cpc 34.72217222223034\n",
      "204\n",
      "**SQUARED**\n",
      "win_num 106149.0, total_cost 6249.985000001465, click_num 178.0, ctr 0.0016768881477922542, cpc 35.112275280907106\n",
      "205\n",
      "**SQUARED**\n",
      "win_num 106131.0, total_cost 6249.958000001467, click_num 177.0, ctr 0.0016677502332023632, cpc 35.31049717514953\n",
      "206\n",
      "**SQUARED**\n",
      "win_num 106090.0, total_cost 6249.927000001469, click_num 176.0, ctr 0.0016589688000754076, cpc 35.510948863644714\n",
      "207\n",
      "**SQUARED**\n",
      "win_num 106083.0, total_cost 6249.9890000014675, click_num 175.0, ctr 0.0016496516878293413, cpc 35.71422285715124\n",
      "208\n",
      "**SQUARED**\n",
      "win_num 106089.0, total_cost 6249.9430000014745, click_num 174.0, ctr 0.00164013234171309, cpc 35.919212643686635\n",
      "209\n"
     ]
    }
   ],
   "source": [
    "base = 180 #136\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "for i in range(30):\n",
    "    bid_p = pbidprice(Cali_stack, base, 'squared')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#save the submission\n",
    "#with open('group11_submission.csv','w') as f:\n",
    "#    f.write('id, payprice\\n')\n",
    "#    for y, x in zip(Bid_id, bidprice_test):\n",
    "#        f.write('%s,%s\\n' % (y, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating validation dataset of 0.01 of training for adaptive regularization\n",
      "-- Epoch 1\n",
      "Training log loss: 0.36794\n",
      "-- Epoch 2\n",
      "Training log loss: 0.29948\n",
      "-- Epoch 3\n",
      "Training log loss: 0.29091\n",
      "-- Epoch 4\n",
      "Training log loss: 0.28470\n",
      "-- Epoch 5\n",
      "Training log loss: 0.27944\n",
      "-- Epoch 6\n",
      "Training log loss: 0.27479\n",
      "-- Epoch 7\n",
      "Training log loss: 0.27063\n",
      "-- Epoch 8\n",
      "Training log loss: 0.26683\n",
      "-- Epoch 9\n",
      "Training log loss: 0.26331\n",
      "-- Epoch 10\n",
      "Training log loss: 0.26005\n",
      "-- Epoch 11\n",
      "Training log loss: 0.25701\n",
      "-- Epoch 12\n",
      "Training log loss: 0.25414\n",
      "-- Epoch 13\n",
      "Training log loss: 0.25147\n",
      "-- Epoch 14\n",
      "Training log loss: 0.24893\n",
      "-- Epoch 15\n",
      "Training log loss: 0.24650\n",
      "-- Epoch 16\n",
      "Training log loss: 0.24423\n",
      "-- Epoch 17\n",
      "Training log loss: 0.24203\n",
      "-- Epoch 18\n",
      "Training log loss: 0.23995\n",
      "-- Epoch 19\n",
      "Training log loss: 0.23794\n",
      "-- Epoch 20\n",
      "Training log loss: 0.23603\n",
      "-- Epoch 21\n",
      "Training log loss: 0.23417\n",
      "-- Epoch 22\n",
      "Training log loss: 0.23235\n",
      "-- Epoch 23\n",
      "Training log loss: 0.23071\n",
      "-- Epoch 24\n",
      "Training log loss: 0.22905\n",
      "-- Epoch 25\n",
      "Training log loss: 0.22743\n",
      "-- Epoch 26\n",
      "Training log loss: 0.22586\n",
      "-- Epoch 27\n",
      "Training log loss: 0.22440\n",
      "-- Epoch 28\n",
      "Training log loss: 0.22295\n",
      "-- Epoch 29\n",
      "Training log loss: 0.22155\n",
      "-- Epoch 30\n",
      "Training log loss: 0.22018\n",
      "-- Epoch 31\n",
      "Training log loss: 0.21885\n",
      "-- Epoch 32\n",
      "Training log loss: 0.21756\n",
      "-- Epoch 33\n",
      "Training log loss: 0.21627\n",
      "-- Epoch 34\n",
      "Training log loss: 0.21507\n",
      "-- Epoch 35\n",
      "Training log loss: 0.21388\n",
      "-- Epoch 36\n",
      "Training log loss: 0.21273\n",
      "-- Epoch 37\n",
      "Training log loss: 0.21158\n",
      "-- Epoch 38\n",
      "Training log loss: 0.21049\n",
      "-- Epoch 39\n",
      "Training log loss: 0.20938\n",
      "-- Epoch 40\n",
      "Training log loss: 0.20834\n",
      "-- Epoch 41\n",
      "Training log loss: 0.20729\n",
      "-- Epoch 42\n",
      "Training log loss: 0.20629\n",
      "-- Epoch 43\n",
      "Training log loss: 0.20530\n",
      "-- Epoch 44\n",
      "Training log loss: 0.20434\n",
      "-- Epoch 45\n",
      "Training log loss: 0.20337\n",
      "-- Epoch 46\n",
      "Training log loss: 0.20246\n",
      "-- Epoch 47\n",
      "Training log loss: 0.20155\n",
      "-- Epoch 48\n",
      "Training log loss: 0.20065\n",
      "-- Epoch 49\n",
      "Training log loss: 0.19974\n",
      "-- Epoch 50\n",
      "Training log loss: 0.19891\n",
      "-- Epoch 51\n",
      "Training log loss: 0.19805\n",
      "-- Epoch 52\n",
      "Training log loss: 0.19722\n",
      "-- Epoch 53\n",
      "Training log loss: 0.19639\n",
      "-- Epoch 54\n",
      "Training log loss: 0.19560\n",
      "-- Epoch 55\n",
      "Training log loss: 0.19472\n",
      "-- Epoch 56\n",
      "Training log loss: 0.19402\n",
      "-- Epoch 57\n",
      "Training log loss: 0.19322\n",
      "-- Epoch 58\n",
      "Training log loss: 0.19246\n",
      "-- Epoch 59\n",
      "Training log loss: 0.19172\n",
      "-- Epoch 60\n",
      "Training log loss: 0.19097\n",
      "-- Epoch 61\n",
      "Training log loss: 0.19023\n",
      "-- Epoch 62\n",
      "Training log loss: 0.18952\n",
      "-- Epoch 63\n",
      "Training log loss: 0.18880\n",
      "-- Epoch 64\n",
      "Training log loss: 0.18807\n",
      "-- Epoch 65\n",
      "Training log loss: 0.18739\n",
      "-- Epoch 66\n",
      "Training log loss: 0.18672\n",
      "-- Epoch 67\n",
      "Training log loss: 0.18603\n",
      "-- Epoch 68\n",
      "Training log loss: 0.18534\n",
      "-- Epoch 69\n",
      "Training log loss: 0.18464\n",
      "-- Epoch 70\n",
      "Training log loss: 0.18404\n",
      "-- Epoch 71\n",
      "Training log loss: 0.18338\n",
      "-- Epoch 72\n",
      "Training log loss: 0.18271\n",
      "-- Epoch 73\n",
      "Training log loss: 0.18208\n",
      "-- Epoch 74\n",
      "Training log loss: 0.18144\n",
      "-- Epoch 75\n",
      "Training log loss: 0.18085\n",
      "-- Epoch 76\n",
      "Training log loss: 0.18020\n",
      "-- Epoch 77\n",
      "Training log loss: 0.17961\n",
      "-- Epoch 78\n",
      "Training log loss: 0.17897\n",
      "-- Epoch 79\n",
      "Training log loss: 0.17838\n",
      "-- Epoch 80\n",
      "Training log loss: 0.17779\n",
      "-- Epoch 81\n",
      "Training log loss: 0.17719\n",
      "-- Epoch 82\n",
      "Training log loss: 0.17659\n",
      "-- Epoch 83\n",
      "Training log loss: 0.17595\n",
      "-- Epoch 84\n",
      "Training log loss: 0.17544\n",
      "-- Epoch 85\n",
      "Training log loss: 0.17486\n",
      "-- Epoch 86\n",
      "Training log loss: 0.17429\n",
      "-- Epoch 87\n",
      "Training log loss: 0.17371\n",
      "-- Epoch 88\n",
      "Training log loss: 0.17315\n",
      "-- Epoch 89\n",
      "Training log loss: 0.17258\n",
      "-- Epoch 90\n",
      "Training log loss: 0.17204\n",
      "-- Epoch 91\n",
      "Training log loss: 0.17148\n",
      "-- Epoch 92\n",
      "Training log loss: 0.17091\n",
      "-- Epoch 93\n",
      "Training log loss: 0.17036\n",
      "-- Epoch 94\n",
      "Training log loss: 0.16980\n",
      "-- Epoch 95\n",
      "Training log loss: 0.16926\n",
      "-- Epoch 96\n",
      "Training log loss: 0.16878\n",
      "-- Epoch 97\n",
      "Training log loss: 0.16825\n",
      "-- Epoch 98\n",
      "Training log loss: 0.16771\n",
      "-- Epoch 99\n",
      "Training log loss: 0.16718\n",
      "-- Epoch 100\n",
      "Training log loss: 0.16667\n",
      "-- Epoch 101\n",
      "Training log loss: 0.16615\n",
      "-- Epoch 102\n",
      "Training log loss: 0.16562\n",
      "-- Epoch 103\n",
      "Training log loss: 0.16512\n",
      "-- Epoch 104\n",
      "Training log loss: 0.16462\n",
      "-- Epoch 105\n",
      "Training log loss: 0.16411\n",
      "-- Epoch 106\n",
      "Training log loss: 0.16357\n",
      "-- Epoch 107\n",
      "Training log loss: 0.16308\n",
      "-- Epoch 108\n",
      "Training log loss: 0.16260\n",
      "-- Epoch 109\n",
      "Training log loss: 0.16209\n",
      "-- Epoch 110\n",
      "Training log loss: 0.16162\n",
      "-- Epoch 111\n",
      "Training log loss: 0.16109\n",
      "-- Epoch 112\n",
      "Training log loss: 0.16057\n",
      "-- Epoch 113\n",
      "Training log loss: 0.16015\n",
      "-- Epoch 114\n",
      "Training log loss: 0.15965\n",
      "-- Epoch 115\n",
      "Training log loss: 0.15919\n",
      "-- Epoch 116\n",
      "Training log loss: 0.15871\n",
      "-- Epoch 117\n",
      "Training log loss: 0.15824\n",
      "-- Epoch 118\n",
      "Training log loss: 0.15774\n",
      "-- Epoch 119\n",
      "Training log loss: 0.15730\n",
      "-- Epoch 120\n",
      "Training log loss: 0.15680\n",
      "-- Epoch 121\n",
      "Training log loss: 0.15635\n",
      "-- Epoch 122\n",
      "Training log loss: 0.15589\n",
      "-- Epoch 123\n",
      "Training log loss: 0.15541\n",
      "-- Epoch 124\n",
      "Training log loss: 0.15495\n",
      "-- Epoch 125\n",
      "Training log loss: 0.15452\n",
      "-- Epoch 126\n",
      "Training log loss: 0.15404\n",
      "-- Epoch 127\n",
      "Training log loss: 0.15359\n",
      "-- Epoch 128\n",
      "Training log loss: 0.15310\n",
      "-- Epoch 129\n",
      "Training log loss: 0.15269\n",
      "-- Epoch 130\n",
      "Training log loss: 0.15221\n",
      "-- Epoch 131\n",
      "Training log loss: 0.15177\n",
      "-- Epoch 132\n",
      "Training log loss: 0.15129\n",
      "-- Epoch 133\n",
      "Training log loss: 0.15091\n",
      "-- Epoch 134\n",
      "Training log loss: 0.15039\n",
      "-- Epoch 135\n",
      "Training log loss: 0.15001\n",
      "-- Epoch 136\n",
      "Training log loss: 0.14956\n",
      "-- Epoch 137\n",
      "Training log loss: 0.14914\n",
      "-- Epoch 138\n",
      "Training log loss: 0.14869\n",
      "-- Epoch 139\n",
      "Training log loss: 0.14824\n",
      "-- Epoch 140\n",
      "Training log loss: 0.14784\n",
      "-- Epoch 141\n",
      "Training log loss: 0.14739\n",
      "-- Epoch 142\n",
      "Training log loss: 0.14695\n",
      "-- Epoch 143\n",
      "Training log loss: 0.14651\n",
      "-- Epoch 144\n",
      "Training log loss: 0.14606\n",
      "-- Epoch 145\n",
      "Training log loss: 0.14569\n",
      "-- Epoch 146\n",
      "Training log loss: 0.14524\n",
      "-- Epoch 147\n",
      "Training log loss: 0.14485\n",
      "-- Epoch 148\n",
      "Training log loss: 0.14437\n",
      "-- Epoch 149\n",
      "Training log loss: 0.14398\n",
      "-- Epoch 150\n",
      "Training log loss: 0.14358\n",
      "-- Epoch 151\n",
      "Training log loss: 0.14313\n",
      "-- Epoch 152\n",
      "Training log loss: 0.14274\n",
      "-- Epoch 153\n",
      "Training log loss: 0.14229\n",
      "-- Epoch 154\n",
      "Training log loss: 0.14189\n",
      "-- Epoch 155\n",
      "Training log loss: 0.14149\n",
      "-- Epoch 156\n",
      "Training log loss: 0.14104\n",
      "-- Epoch 157\n",
      "Training log loss: 0.14067\n",
      "-- Epoch 158\n",
      "Training log loss: 0.14024\n",
      "-- Epoch 159\n",
      "Training log loss: 0.13980\n",
      "-- Epoch 160\n",
      "Training log loss: 0.13945\n",
      "-- Epoch 161\n",
      "Training log loss: 0.13903\n",
      "-- Epoch 162\n",
      "Training log loss: 0.13863\n",
      "-- Epoch 163\n",
      "Training log loss: 0.13822\n",
      "-- Epoch 164\n",
      "Training log loss: 0.13779\n",
      "-- Epoch 165\n",
      "Training log loss: 0.13740\n",
      "-- Epoch 166\n",
      "Training log loss: 0.13700\n",
      "-- Epoch 167\n",
      "Training log loss: 0.13660\n",
      "-- Epoch 168\n",
      "Training log loss: 0.13622\n",
      "-- Epoch 169\n",
      "Training log loss: 0.13582\n",
      "-- Epoch 170\n",
      "Training log loss: 0.13541\n",
      "-- Epoch 171\n",
      "Training log loss: 0.13503\n",
      "-- Epoch 172\n",
      "Training log loss: 0.13463\n",
      "-- Epoch 173\n",
      "Training log loss: 0.13422\n",
      "-- Epoch 174\n",
      "Training log loss: 0.13381\n",
      "-- Epoch 175\n",
      "Training log loss: 0.13344\n",
      "-- Epoch 176\n",
      "Training log loss: 0.13304\n",
      "-- Epoch 177\n",
      "Training log loss: 0.13268\n",
      "-- Epoch 178\n",
      "Training log loss: 0.13225\n",
      "-- Epoch 179\n",
      "Training log loss: 0.13189\n",
      "-- Epoch 180\n",
      "Training log loss: 0.13149\n",
      "-- Epoch 181\n",
      "Training log loss: 0.13110\n",
      "-- Epoch 182\n",
      "Training log loss: 0.13072\n",
      "-- Epoch 183\n",
      "Training log loss: 0.13029\n",
      "-- Epoch 184\n",
      "Training log loss: 0.12997\n",
      "-- Epoch 185\n",
      "Training log loss: 0.12959\n",
      "-- Epoch 186\n",
      "Training log loss: 0.12921\n",
      "-- Epoch 187\n",
      "Training log loss: 0.12882\n",
      "-- Epoch 188\n",
      "Training log loss: 0.12845\n",
      "-- Epoch 189\n",
      "Training log loss: 0.12803\n",
      "-- Epoch 190\n",
      "Training log loss: 0.12767\n",
      "-- Epoch 191\n",
      "Training log loss: 0.12732\n",
      "-- Epoch 192\n",
      "Training log loss: 0.12693\n",
      "-- Epoch 193\n",
      "Training log loss: 0.12653\n",
      "-- Epoch 194\n",
      "Training log loss: 0.12617\n",
      "-- Epoch 195\n",
      "Training log loss: 0.12580\n",
      "-- Epoch 196\n",
      "Training log loss: 0.12540\n",
      "-- Epoch 197\n",
      "Training log loss: 0.12503\n",
      "-- Epoch 198\n",
      "Training log loss: 0.12470\n",
      "-- Epoch 199\n",
      "Training log loss: 0.12432\n",
      "-- Epoch 200\n",
      "Training log loss: 0.12396\n",
      "auc on test:  0.850264619861\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "from pyfm import pylibfm\n",
    "\n",
    "fm = pylibfm.FM(num_factors=20, \n",
    "                num_iter=200, \n",
    "                verbose=True, \n",
    "                task=\"classification\", \n",
    "                initial_learning_rate=0.0001, \n",
    "                learning_rate_schedule=\"optimal\")\n",
    "fm.fit(X_FM,target)\n",
    "pred_fm_test = fm.predict(X_val_FM)\n",
    "Cali_fm = Cali_CTR(pred_fm_test, 0.0075)\n",
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_fm, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_fm)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00010251,  0.00849571,  0.00089773, ...,  0.00058971,\n",
       "        0.00079036,  0.00041024])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Cali_fm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predict_xgbc = xgb_model.predict_proba(X_val_FM)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_xgb=(bst_model.predict(dval))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.892427100683\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "y_pred_combine_fm_xgb = 0.1 * pred_fm_test + 0.9* predict_xgb\n",
    "Cali_combine_fm_xgb = Cali_CTR(y_pred_combine_fm_xgb, 0.0075)\n",
    "\n",
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_combine_fm_xgb, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_combine_fm_xgb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc on test:  0.894163415997\n",
      "RMSE on test dataset = 0.03\n"
     ]
    }
   ],
   "source": [
    "y_pred_combine_fm_xgb = 0.1 * predict_xgbc + 0.9* predict_xgb\n",
    "Cali_combine_fm_xgb = Cali_CTR(y_pred_combine_fm_xgb, 0.0075)\n",
    "\n",
    "fpr, tpr, thresholds = metrics.roc_curve(target_val, Cali_combine_fm_xgb, pos_label=1)\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "print('auc on test: ', auc)\n",
    "print (\"RMSE on test dataset = %.2f\" % (rmse(target_val, Cali_combine_fm_xgb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " **LINEAR** \n",
      "win_num 119560.0, total_cost 5641.977000001879, click_num 184.0, ctr 0.0015389762462361995, cpc 30.66291847827108\n",
      "120\n",
      " **LINEAR** \n",
      "win_num 120273.0, total_cost 5690.921000001902, click_num 185.0, ctr 0.0015381673359773182, cpc 30.761735135145415\n",
      "121\n",
      " **LINEAR** \n",
      "win_num 120949.0, total_cost 5735.320000001932, click_num 185.0, ctr 0.001529570314760767, cpc 31.001729729740177\n",
      "122\n",
      " **LINEAR** \n",
      "win_num 121671.0, total_cost 5784.547000001968, click_num 186.0, ctr 0.0015287126759868826, cpc 31.09971505377402\n",
      "123\n",
      " **LINEAR** \n",
      "win_num 122334.0, total_cost 5829.0050000019855, click_num 186.0, ctr 0.001520427681593016, cpc 31.33873655915046\n",
      "124\n",
      " **LINEAR** \n",
      "win_num 123084.0, total_cost 5880.015000002003, click_num 186.0, ctr 0.001511163108121283, cpc 31.61298387097851\n",
      "125\n",
      " **LINEAR** \n",
      "win_num 123710.0, total_cost 5924.143000002001, click_num 186.0, ctr 0.001503516288093121, cpc 31.850231182806457\n",
      "126\n",
      " **LINEAR** \n",
      "win_num 124404.0, total_cost 5972.6320000020205, click_num 186.0, ctr 0.0014951287739944053, cpc 32.110924731193656\n",
      "127\n",
      " **LINEAR** \n",
      "win_num 125106.0, total_cost 6021.6050000020305, click_num 187.0, ctr 0.0014947324668680958, cpc 32.20109625669535\n",
      "128\n",
      " **LINEAR** \n",
      "win_num 125786.0, total_cost 6069.6960000020545, click_num 187.0, ctr 0.0014866519326475125, cpc 32.45826737969013\n",
      "129\n",
      " **LINEAR** \n",
      "win_num 126454.0, total_cost 6116.4130000020505, click_num 187.0, ctr 0.0014787986145159504, cpc 32.70809090910188\n",
      "130\n",
      " **LINEAR** \n",
      "win_num 127090.0, total_cost 6161.69400000206, click_num 187.0, ctr 0.0014713982217326304, cpc 32.95023529412867\n",
      "131\n",
      " **LINEAR** \n",
      "win_num 127714.0, total_cost 6204.977000002051, click_num 187.0, ctr 0.001464209092190363, cpc 33.18169518717674\n",
      "132\n",
      " **LINEAR** \n",
      "win_num 128351.0, total_cost 6249.975000002096, click_num 187.0, ctr 0.0014569422910612306, cpc 33.42232620321976\n",
      "133\n",
      " **LINEAR** \n",
      "win_num 128100.0, total_cost 6249.97600000208, click_num 185.0, ctr 0.001444184231069477, cpc 33.783654054065295\n",
      "134\n",
      " **LINEAR** \n",
      "win_num 127833.0, total_cost 6249.984000002068, click_num 183.0, ctr 0.001431555232217033, cpc 34.15291803279818\n",
      "135\n",
      " **LINEAR** \n",
      "win_num 127562.0, total_cost 6249.9590000020535, click_num 180.0, ctr 0.0014110785343597622, cpc 34.72199444445585\n",
      "136\n",
      " **LINEAR** \n",
      "win_num 127336.0, total_cost 6249.971000002044, click_num 178.0, ctr 0.0013978764842621097, cpc 35.11219662922497\n",
      "137\n",
      " **LINEAR** \n",
      "win_num 127126.0, total_cost 6249.944000002048, click_num 177.0, ctr 0.001392319431115586, cpc 35.31041807910761\n",
      "138\n",
      " **LINEAR** \n",
      "win_num 126888.0, total_cost 6249.980000002013, click_num 176.0, ctr 0.0013870499968476136, cpc 35.51125000001144\n",
      "139\n",
      " **LINEAR** \n",
      "win_num 126614.0, total_cost 6249.971000002008, click_num 175.0, ctr 0.0013821536322997457, cpc 35.71412000001147\n",
      "140\n",
      " **LINEAR** \n",
      "win_num 126363.0, total_cost 6249.949000001995, click_num 174.0, ctr 0.0013769853517247928, cpc 35.91924712644825\n",
      "141\n",
      " **LINEAR** \n",
      "win_num 126098.0, total_cost 6249.83100000199, click_num 173.0, ctr 0.001371948801725642, cpc 36.12619075145659\n",
      "142\n",
      " **LINEAR** \n",
      "win_num 125806.0, total_cost 6249.959000001965, click_num 172.0, ctr 0.0013671843950209051, cpc 36.336970930243986\n",
      "143\n",
      " **LINEAR** \n",
      "win_num 125576.0, total_cost 6249.929000001967, click_num 169.0, ctr 0.0013457985602344396, cpc 36.981828402378504\n",
      "144\n",
      " **LINEAR** \n",
      "win_num 125402.0, total_cost 6249.982000001951, click_num 169.0, ctr 0.0013476659064448732, cpc 36.982142011845866\n",
      "145\n",
      " **LINEAR** \n",
      "win_num 125191.0, total_cost 6249.966000001933, click_num 169.0, ctr 0.0013499372958119993, cpc 36.98204733728954\n",
      "146\n",
      " **LINEAR** \n",
      "win_num 124943.0, total_cost 6249.971000001921, click_num 169.0, ctr 0.0013526167932577256, cpc 36.98207692308829\n",
      "147\n",
      " **LINEAR** \n",
      "win_num 124703.0, total_cost 6249.78800000191, click_num 167.0, ctr 0.0013391818961853363, cpc 37.423880239532394\n",
      "148\n",
      " **LINEAR** \n",
      "win_num 124472.0, total_cost 6249.973000001898, click_num 165.0, ctr 0.0013255993315765796, cpc 37.87862424243575\n",
      "149\n"
     ]
    }
   ],
   "source": [
    "base = 120 #136\n",
    "bidp = []\n",
    "win_number = []\n",
    "totalcost = []\n",
    "clicknum = []\n",
    "ctr_p = []\n",
    "cpc_p = []\n",
    "for i in range(30):\n",
    "    bid_p = pbidprice(Cali_combine_fm_xgb, base, 'linear')\n",
    "    win_num, total_cost, click_num, ctr, cpc = evaluation(bid_p,Payprice_val, clicks)\n",
    "    print(base)\n",
    "#     win_number.append(win_num)\n",
    "#     totalcost.append(total_cost)\n",
    "#     clicknum.append(click_num)\n",
    "#     ctr_p.append(ctr)\n",
    "#     cpc_p.append(cpc)\n",
    "#     bidp.append(bid_p)\n",
    "    base += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}