demo-dogs-vs-cats-sim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.cm as cm\n",
    "import scipy.stats\n",
    "from matplotlib import pyplot as plt \n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, Input\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "\n",
    "# packages for learning from crowds\n",
    "from crowd_layer.crowd_layers import CrowdsClassification, MaskedMultiCrossEntropy\n",
    "from crowd_layer.crowd_aggregators import CrowdsCategoricalAggregator\n",
    "\n",
    "# prevent tensorflow from allocating the entire GPU memory at once\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_PATH = \"/home/fmpr/datasets/deep-crowds-datasets/dogs_vs_cats_sim/prepared/\"\n",
    "N_CLASSES = 2\n",
    "BATCH_SIZE = 16\n",
    "N_EPOCHS = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data(filename):\n",
    "    f = open(filename)\n",
    "    data = np.load(f)\n",
    "    f.close()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Loading train data...\n",
      "(12500, 150, 150, 3)\n",
      "(12500,)\n",
      "(12500,)\n",
      "(12500,)\n",
      "\n",
      "Loading AMT data...\n",
      "(12500, 5)\n",
      "\n",
      "N_CLASSES: 2\n",
      "N_ANNOT: 5\n",
      "\n",
      "Loading test data...\n",
      "(12500, 150, 150, 3)\n",
      "(12500,)\n"
     ]
    }
   ],
   "source": [
    "print \"\\nLoading train data...\"\n",
    "\n",
    "data_train = load_data(DATA_PATH+\"data_train.npy\")\n",
    "print data_train.shape\n",
    "\n",
    "labels_train = load_data(DATA_PATH+\"labels_train.npy\")\n",
    "print labels_train.shape\n",
    "\n",
    "labels_train_mv = load_data(DATA_PATH+\"labels_train_mv.npy\")\n",
    "print labels_train_mv.shape\n",
    "\n",
    "labels_train_ds = pd.read_csv(DATA_PATH+\"labels_train_DS.txt\", header=None).as_matrix()[:,0]\n",
    "print labels_train_ds.shape\n",
    "\n",
    "# data from Amazon Mechanical Turk\n",
    "print \"\\nLoading AMT data...\"\n",
    "answers = load_data(DATA_PATH+\"labels_train_sim.npy\")\n",
    "print answers.shape\n",
    "N_ANNOT = answers.shape[1]\n",
    "print \"\\nN_CLASSES:\", N_CLASSES\n",
    "print \"N_ANNOT:\", N_ANNOT\n",
    "\n",
    "# load test data\n",
    "print \"\\nLoading test data...\"\n",
    "\n",
    "data_test = load_data(DATA_PATH+\"data_test.npy\")\n",
    "print data_test.shape\n",
    "\n",
    "labels_test = load_data(DATA_PATH+\"labels_test.npy\")\n",
    "print labels_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert data to one-hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot(target, n_classes):\n",
    "    targets = np.array([target]).reshape(-1)\n",
    "    one_hot_targets = np.eye(n_classes)[targets]\n",
    "    return one_hot_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting to one-hot encoding...\n",
      "(12500, 2)\n",
      "(12500, 2)\n",
      "(12500, 2)\n",
      "(12500, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(12500, 2, 5)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print \"\\nConverting to one-hot encoding...\"\n",
    "labels_train_bin = one_hot(labels_train, N_CLASSES)\n",
    "print labels_train_bin.shape\n",
    "labels_train_mv_bin = one_hot(labels_train_mv, N_CLASSES)\n",
    "print labels_train_mv_bin.shape\n",
    "labels_train_ds_bin = one_hot(labels_train_ds, N_CLASSES)\n",
    "print labels_train_ds_bin.shape\n",
    "labels_test_bin = one_hot(labels_test, N_CLASSES)\n",
    "print labels_test_bin.shape\n",
    "\n",
    "answers_bin_missings = []\n",
    "for i in range(len(answers)):\n",
    "    row = []\n",
    "    for r in range(N_ANNOT):\n",
    "        if answers[i,r] == -1:\n",
    "            row.append(-1 * np.ones(N_CLASSES))\n",
    "        else:\n",
    "            row.append(one_hot(answers[i,r], N_CLASSES)[0,:])\n",
    "    answers_bin_missings.append(row)\n",
    "answers_bin_missings = np.array(answers_bin_missings).swapaxes(1,2)\n",
    "answers_bin_missings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the base deep learning model\n",
    "\n",
    "Here we shall use features representation produced by the VGG16 network as the input. Our base model is then simply composed by one densely-connected layer with 128 hidden units and an output dense layer. We use 50% dropout between the two dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_base_model():\n",
    "    base_model = Sequential()\n",
    "    base_model.add(Conv2D(32, (3, 3), input_shape=(150, 150, 3)))\n",
    "    base_model.add(Activation('relu'))\n",
    "    base_model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    base_model.add(BatchNormalization(scale=False))\n",
    "\n",
    "    base_model.add(Conv2D(32, (3, 3)))\n",
    "    base_model.add(Activation('relu'))\n",
    "    base_model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    base_model.add(Conv2D(64, (3, 3)))\n",
    "    base_model.add(Activation('relu'))\n",
    "    base_model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    base_model.add(Conv2D(64, (3, 3)))\n",
    "    base_model.add(Activation('relu'))\n",
    "    base_model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    base_model.add(Flatten()) \n",
    "    base_model.add(Dense(128))\n",
    "    base_model.add(Activation('relu'))\n",
    "    base_model.add(Dropout(0.5))\n",
    "    base_model.add(Dense(N_CLASSES))\n",
    "    base_model.add(Activation('softmax'))\n",
    "    base_model.compile(optimizer='adam', loss='categorical_crossentropy')\n",
    "\n",
    "    return base_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary function for evaluating the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eval_model(model, test_data, test_labels):\n",
    "    # testset accuracy\n",
    "    preds_test = model.predict(test_data)\n",
    "    preds_test_num = np.argmax(preds_test, axis=1)\n",
    "    accuracy_test = 1.0*np.sum(preds_test_num == test_labels) / len(test_labels)\n",
    "\n",
    "    return accuracy_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the true labels (ground truth) and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "16s - loss: 0.6589\n",
      "Epoch 2/30\n",
      "15s - loss: 0.5911\n",
      "Epoch 3/30\n",
      "16s - loss: 0.5190\n",
      "Epoch 4/30\n",
      "15s - loss: 0.4552\n",
      "Epoch 5/30\n",
      "16s - loss: 0.4027\n",
      "Epoch 6/30\n",
      "15s - loss: 0.3494\n",
      "Epoch 7/30\n",
      "16s - loss: 0.3039\n",
      "Epoch 8/30\n",
      "16s - loss: 0.2719\n",
      "Epoch 9/30\n",
      "16s - loss: 0.2296\n",
      "Epoch 10/30\n",
      "16s - loss: 0.1961\n",
      "Epoch 11/30\n",
      "16s - loss: 0.1756\n",
      "Epoch 12/30\n",
      "16s - loss: 0.1449\n",
      "Epoch 13/30\n",
      "16s - loss: 0.1287\n",
      "Epoch 14/30\n",
      "16s - loss: 0.1214\n",
      "Epoch 15/30\n",
      "16s - loss: 0.1027\n",
      "Epoch 16/30\n",
      "15s - loss: 0.1134\n",
      "Epoch 17/30\n",
      "16s - loss: 0.0890\n",
      "Epoch 18/30\n",
      "16s - loss: 0.0901\n",
      "Epoch 19/30\n",
      "16s - loss: 0.0851\n",
      "Epoch 20/30\n",
      "15s - loss: 0.0785\n",
      "Epoch 21/30\n",
      "16s - loss: 0.0879\n",
      "Epoch 22/30\n",
      "15s - loss: 0.0772\n",
      "Epoch 23/30\n",
      "16s - loss: 0.0708\n",
      "Epoch 24/30\n",
      "16s - loss: 0.0751\n",
      "Epoch 25/30\n",
      "16s - loss: 0.0639\n",
      "Epoch 26/30\n",
      "15s - loss: 0.0648\n",
      "Epoch 27/30\n",
      "16s - loss: 0.0672\n",
      "Epoch 28/30\n",
      "16s - loss: 0.0738\n",
      "Epoch 29/30\n",
      "15s - loss: 0.0682\n",
      "Epoch 30/30\n",
      "15s - loss: 0.0638\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc5d5f29cd0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "model.fit(data_train, labels_train_bin, epochs=N_EPOCHS, shuffle=True, batch_size=BATCH_SIZE, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: Test: 0.841\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = eval_model(model, data_test, labels_test)\n",
    "print \"Accuracy: Test: %.3f\" % (accuracy_test,) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the output of majority voting and evaluate on testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "15s - loss: 0.6834\n",
      "Epoch 2/30\n",
      "14s - loss: 0.6793\n",
      "Epoch 3/30\n",
      "14s - loss: 0.6786\n",
      "Epoch 4/30\n",
      "14s - loss: 0.6787\n",
      "Epoch 5/30\n",
      "14s - loss: 0.6792\n",
      "Epoch 6/30\n",
      "14s - loss: 0.6802\n",
      "Epoch 7/30\n",
      "14s - loss: 0.6687\n",
      "Epoch 8/30\n",
      "14s - loss: 0.6510\n",
      "Epoch 9/30\n",
      "14s - loss: 0.6291\n",
      "Epoch 10/30\n",
      "14s - loss: 0.6018\n",
      "Epoch 11/30\n",
      "14s - loss: 0.5831\n",
      "Epoch 12/30\n",
      "14s - loss: 0.5598\n",
      "Epoch 13/30\n",
      "14s - loss: 0.5364\n",
      "Epoch 14/30\n",
      "14s - loss: 0.5064\n",
      "Epoch 15/30\n",
      "14s - loss: 0.4695\n",
      "Epoch 16/30\n",
      "14s - loss: 0.4188\n",
      "Epoch 17/30\n",
      "14s - loss: 0.3754\n",
      "Epoch 18/30\n",
      "14s - loss: 0.3316\n",
      "Epoch 19/30\n",
      "14s - loss: 0.2921\n",
      "Epoch 20/30\n",
      "14s - loss: 0.2614\n",
      "Epoch 21/30\n",
      "14s - loss: 0.2360\n",
      "Epoch 22/30\n",
      "14s - loss: 0.2222\n",
      "Epoch 23/30\n",
      "14s - loss: 0.1975\n",
      "Epoch 24/30\n",
      "14s - loss: 0.1795\n",
      "Epoch 25/30\n",
      "14s - loss: 0.1783\n",
      "Epoch 26/30\n",
      "14s - loss: 0.1629\n",
      "Epoch 27/30\n",
      "14s - loss: 0.1479\n",
      "Epoch 28/30\n",
      "14s - loss: 0.1521\n",
      "Epoch 29/30\n",
      "14s - loss: 0.1582\n",
      "Epoch 30/30\n",
      "14s - loss: 0.1462\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc5b86b72d0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "model.fit(data_train, labels_train_mv_bin, epochs=N_EPOCHS, shuffle=True, batch_size=BATCH_SIZE, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: Test: 0.702\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = eval_model(model, data_test, labels_test)\n",
    "print \"Accuracy: Test: %.3f\" % (accuracy_test,) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model on the output of Dawid & Skene [1] and evaluate on testset\n",
    "\n",
    "[1] Dawid, A.P. and Skene, A.M., 1979. Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied statistics, pp.20-28."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "15s - loss: 0.6766\n",
      "Epoch 2/30\n",
      "14s - loss: 0.6532\n",
      "Epoch 3/30\n",
      "14s - loss: 0.6221\n",
      "Epoch 4/30\n",
      "14s - loss: 0.5788\n",
      "Epoch 5/30\n",
      "14s - loss: 0.5503\n",
      "Epoch 6/30\n",
      "14s - loss: 0.5185\n",
      "Epoch 7/30\n",
      "14s - loss: 0.4837\n",
      "Epoch 8/30\n",
      "14s - loss: 0.4498\n",
      "Epoch 9/30\n",
      "14s - loss: 0.4065\n",
      "Epoch 10/30\n",
      "14s - loss: 0.3658\n",
      "Epoch 11/30\n",
      "14s - loss: 0.3206\n",
      "Epoch 12/30\n",
      "14s - loss: 0.2786\n",
      "Epoch 13/30\n",
      "14s - loss: 0.2440\n",
      "Epoch 14/30\n",
      "14s - loss: 0.2185\n",
      "Epoch 15/30\n",
      "14s - loss: 0.1962\n",
      "Epoch 16/30\n",
      "14s - loss: 0.1740\n",
      "Epoch 17/30\n",
      "14s - loss: 0.1687\n",
      "Epoch 18/30\n",
      "14s - loss: 0.1491\n",
      "Epoch 19/30\n",
      "14s - loss: 0.1446\n",
      "Epoch 20/30\n",
      "14s - loss: 0.1328\n",
      "Epoch 21/30\n",
      "14s - loss: 0.1354\n",
      "Epoch 22/30\n",
      "14s - loss: 0.1392\n",
      "Epoch 23/30\n",
      "14s - loss: 0.1180\n",
      "Epoch 24/30\n",
      "14s - loss: 0.1173\n",
      "Epoch 25/30\n",
      "14s - loss: 0.1146\n",
      "Epoch 26/30\n",
      "14s - loss: 0.1030\n",
      "Epoch 27/30\n",
      "14s - loss: 0.1088\n",
      "Epoch 28/30\n",
      "14s - loss: 0.1064\n",
      "Epoch 29/30\n",
      "14s - loss: 0.0972\n",
      "Epoch 30/30\n",
      "14s - loss: 0.1066\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc5841e4350>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "model.fit(data_train, labels_train_ds_bin, epochs=N_EPOCHS, shuffle=True, batch_size=BATCH_SIZE, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: Test: 0.772\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = eval_model(model, data_test, labels_test)\n",
    "print \"Accuracy: Test: %.3f\" % (accuracy_test,) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model using EM approach and evaluate on testset\n",
    "\n",
    "The CrowdsCategoricalAggregator class acts as a wrapper for the base model that computed the EM steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.84224\n",
      "M-step\n",
      "loss: 0.684392473335\n",
      "Epoch: 2\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.59224\n",
      "M-step\n",
      "loss: 0.0323643980789\n",
      "Epoch: 3\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.5916\n",
      "M-step\n",
      "loss: 0.0324461836386\n",
      "Epoch: 4\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.59232\n",
      "M-step\n",
      "loss: 0.0324133456659\n",
      "Epoch: 5\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.59528\n",
      "M-step\n",
      "loss: 0.0324044745845\n",
      "Epoch: 6\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.60376\n",
      "M-step\n",
      "loss: 0.0323817200661\n",
      "Epoch: 7\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.6124\n",
      "M-step\n",
      "loss: 0.0323012759167\n",
      "Epoch: 8\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.63144\n",
      "M-step\n",
      "loss: 0.0322988915277\n",
      "Epoch: 9\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.65864\n",
      "M-step\n",
      "loss: 0.0322074132478\n",
      "Epoch: 10\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.68808\n",
      "M-step\n",
      "loss: 0.0321140321147\n",
      "Epoch: 11\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.68856\n",
      "M-step\n",
      "loss: 0.0317038526654\n",
      "Epoch: 12\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.72656\n",
      "M-step\n",
      "loss: 0.031607694242\n",
      "Epoch: 13\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.77176\n",
      "M-step\n",
      "loss: 0.0311710294521\n",
      "Epoch: 14\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.73056\n",
      "M-step\n",
      "loss: 0.0308162715697\n",
      "Epoch: 15\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.8108\n",
      "M-step\n",
      "loss: 0.0309182809818\n",
      "Epoch: 16\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.852\n",
      "M-step\n",
      "loss: 0.0301130830103\n",
      "Epoch: 17\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.89312\n",
      "M-step\n",
      "loss: 0.0287455912322\n",
      "Epoch: 18\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.8732\n",
      "M-step\n",
      "loss: 0.0258560727674\n",
      "Epoch: 19\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.92888\n",
      "M-step\n",
      "loss: 0.0250166548991\n",
      "Epoch: 20\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.9452\n",
      "M-step\n",
      "loss: 0.0225298188084\n",
      "Epoch: 21\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93336\n",
      "M-step\n",
      "loss: 0.0193395829326\n",
      "Epoch: 22\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93264\n",
      "M-step\n",
      "loss: 0.0180083059943\n",
      "Epoch: 23\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.92792\n",
      "M-step\n",
      "loss: 0.0143688721249\n",
      "Epoch: 24\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.92664\n",
      "M-step\n",
      "loss: 0.0136771964675\n",
      "Epoch: 25\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93216\n",
      "M-step\n",
      "loss: 0.0124778720936\n",
      "Epoch: 26\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.92264\n",
      "M-step\n",
      "loss: 0.0121590540145\n",
      "Epoch: 27\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.9376\n",
      "M-step\n",
      "loss: 0.0121397830095\n",
      "Epoch: 28\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93352\n",
      "M-step\n",
      "loss: 0.012337784813\n",
      "Epoch: 29\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93672\n",
      "M-step\n",
      "loss: 0.00958512822129\n",
      "Epoch: 30\n",
      "E-step\n",
      "Adjusted ground truth accuracy: 0.93312\n",
      "M-step\n",
      "loss: 0.0108407262293\n"
     ]
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "crowds_agg = CrowdsCategoricalAggregator(model, data_train, answers, batch_size=BATCH_SIZE)\n",
    "for epoch in xrange(N_EPOCHS):\n",
    "    print \"Epoch:\", epoch+1\n",
    "    \n",
    "    # E-step\n",
    "    ground_truth_est = crowds_agg.e_step()\n",
    "    print \"Adjusted ground truth accuracy:\", 1.0*np.sum(np.argmax(ground_truth_est, axis=1) == labels_train) / len(labels_train)\n",
    "    \n",
    "    # M-step\n",
    "    model, pi = crowds_agg.m_step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: Test: 0.805\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = eval_model(model, data_test, labels_test)\n",
    "print \"Accuracy: Test: %.3f\" % (accuracy_test,) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model using proposed DL-MW approach and evaluate on testset\n",
    "\n",
    "We start by adding a new layer (CrowdsClassification) on top of our neural network. We then require a special loss (MaskedMultiCrossEntropy) to handle the missing labels from some of the annotators (encoded as \"-1\").\n",
    "\n",
    "Notice how the training is faster then the EM approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "15s - loss: 0.6527\n",
      "Epoch 2/30\n",
      "14s - loss: 0.6279\n",
      "Epoch 3/30\n",
      "14s - loss: 0.6152\n",
      "Epoch 4/30\n",
      "14s - loss: 0.6041\n",
      "Epoch 5/30\n",
      "14s - loss: 0.5925\n",
      "Epoch 6/30\n",
      "14s - loss: 0.5818\n",
      "Epoch 7/30\n",
      "14s - loss: 0.5704\n",
      "Epoch 8/30\n",
      "14s - loss: 0.5604\n",
      "Epoch 9/30\n",
      "14s - loss: 0.5540\n",
      "Epoch 10/30\n",
      "14s - loss: 0.5429\n",
      "Epoch 11/30\n",
      "14s - loss: 0.5362\n",
      "Epoch 12/30\n",
      "14s - loss: 0.5324\n",
      "Epoch 13/30\n",
      "14s - loss: 0.5244\n",
      "Epoch 14/30\n",
      "14s - loss: 0.5214\n",
      "Epoch 15/30\n",
      "14s - loss: 0.5147\n",
      "Epoch 16/30\n",
      "14s - loss: 0.5117\n",
      "Epoch 17/30\n",
      "14s - loss: 0.5070\n",
      "Epoch 18/30\n",
      "14s - loss: 0.5055\n",
      "Epoch 19/30\n",
      "14s - loss: 0.5049\n",
      "Epoch 20/30\n",
      "14s - loss: 0.4992\n",
      "Epoch 21/30\n",
      "14s - loss: 0.4988\n",
      "Epoch 22/30\n",
      "14s - loss: 0.4969\n",
      "Epoch 23/30\n",
      "14s - loss: 0.4938\n",
      "Epoch 24/30\n",
      "14s - loss: 0.4944\n",
      "Epoch 25/30\n",
      "14s - loss: 0.4904\n",
      "Epoch 26/30\n",
      "14s - loss: 0.4905\n",
      "Epoch 27/30\n",
      "14s - loss: 0.4893\n",
      "Epoch 28/30\n",
      "14s - loss: 0.4905\n",
      "Epoch 29/30\n",
      "14s - loss: 0.4865\n",
      "Epoch 30/30\n",
      "14s - loss: 0.4873\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc5b80daf10>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_base_model()\n",
    "\n",
    "# add crowds layer on top of the base model\n",
    "model.add(CrowdsClassification(N_CLASSES, N_ANNOT, conn_type=\"MW\"))\n",
    "\n",
    "# instantiate specialized masked loss to handle missing answers\n",
    "loss = MaskedMultiCrossEntropy().loss\n",
    "\n",
    "# compile model with masked loss and train\n",
    "model.compile(optimizer='adam', loss=loss)\n",
    "model.fit(data_train, answers_bin_missings, epochs=N_EPOCHS, shuffle=True, batch_size=BATCH_SIZE, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before evaluating our model, we need to remove the crowds layer used during training in order to expose the aggregation (bottleneck) layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: Test: 0.812\n"
     ]
    }
   ],
   "source": [
    "# save weights from crowds layer for later\n",
    "weights = model.layers[-1].get_weights()\n",
    "\n",
    "# remove crowds layer before making predictions\n",
    "model.pop() \n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "accuracy_test = eval_model(model, data_test, labels_test)\n",
    "print \"Accuracy: Test: %.3f\" % (accuracy_test,) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare the weights learned by the crowds layer for each annotator with their true confution matrices\n",
    "\n",
    "First, compute true confusion matrices:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conf_mats = np.zeros((N_CLASSES,N_CLASSES,N_ANNOT)) + 0.00000001\n",
    "num_answers = np.zeros((N_CLASSES,N_ANNOT)) + (0.00000001 * N_CLASSES)\n",
    "for i in xrange(len(answers)):\n",
    "    for r in xrange(N_ANNOT):\n",
    "        if answers[i][r] != -1:\n",
    "            num_answers[labels_train[i],r] += 1\n",
    "            conf_mats[labels_train[i],answers[i][r],r] += 1\n",
    "for r in xrange(N_ANNOT):\n",
    "    for c in xrange(N_CLASSES):\n",
    "        for c2 in xrange(N_CLASSES):\n",
    "            conf_mats[c,c2,r] = conf_mats[c,c2,r] / num_answers[c,r]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True values for sensitivity: [ 0.70528  0.79584  0.49872  0.19536  0.89824]\n"
     ]
    }
   ],
   "source": [
    "print \"True values for sensitivity:\", conf_mats[1,1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learned weights for sensitivity: [ 1.00891089  1.47721922  0.49702653 -0.16305956  2.02543855]\n"
     ]
    }
   ],
   "source": [
    "print \"Learned weights for sensitivity:\", weights[0][1,1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.97676979683 -0.872584601029\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAETCAYAAAD3WTuEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X94U/W9B/D3aUtbTnJSh5LUIt605brrmgTdHm2LSpUW\nVBjV7bo6wbspE6hXYBMHUvQipfyuFueYYLkie67FibuinfxQwyigJUx3H2lSH+dsU1Frk0qlTZNS\nSnvuH7UJpb9Dm5yk79fz+NjknJN+8iXtu9/vN/l+BVmWZRAREfkhItgFEBFR6GKIEBGR3xgiRETk\nN4YIERH5jSFCRER+U0yImM1mmM1mlJSU9Hq8636z2RzIsoiIqB+KCBGr1QqTyYSsrCw4HA7YbLYe\n55jNZixZsgTx8fEDPl5lZeVIlBmS2BY+bAsftoUP28LHn7ZQRIg4HA5YrVYAgE6ng8Ph6HFObm4u\nnnvuORgMhgEfjy8KH7aFD9vCh23hw7bw8actokagjiHLysryfm2323HTTTf1OKcraOx2O7KzswNZ\nHhER9UERPZEudrsdSUlJ0Ov1PY5lZ2fDaDTC5XL1OtxFRESBJyhp2ZPS0tJeexlmsxmSJCE1NRWl\npaVQqVTIzMz0Hq+srOzWDcvJyQlIvURE4WbPnj3er1NSUpCSktLv+YoJEbPZ7B3WslqtMBqN8Hg8\nEEURNTU10Gq1EEURxcXFmDFjRq+9lQvV1tYGoGrlkyQJLpcr2GUoAtvCh23hw7bwSUhIGPI1ihjO\nslqt2L17NxYvXox58+ZBEAQAQEFBAQBAr9ejvLwcFosF8fHxAwYIEREFhmJ6IsONPZFO/CvLh23h\nw7bwYVv4hGxPhIiIQhNDhIiI/MYQISIivzFEiIjIbwwRIiLyG0OEiIj8xhAhIiK/MUSIiMhvDBEi\nIvIbQ4SIiPzGECEiIr8xRIiIyG8MESIi8htDhIiI/MYQISIivzFEiIjIbwwRIiLyG0OEiIj8xhAh\nIiK/MUSIiMhvDBEiIvIbQ4SIiPzGECEiIr9FBbuALmazGQDgcDgwd+7cHsctFgtUKhXsdjuys7MD\nXR4REfVCET0Rq9UKk8mErKwsOBwO2Gy2bsftdjsEQYDRaIQoiqipqQlOoUREw8jjiYTNJuKdd9Sw\n2UR4PJHBLmnIFBEiDocDVqsVAKDT6eBwOLodLy8vh0ql8h6vqKgIeI1ERMPJ44lEcbGE22+Pw4MP\nanD77XEoLpZCLkgUESJZWVnIzMwE0NnrSE5O7nbc7XZDrVZ7bzc3Nwe0PiKi4VZVFYPCwrEAhO/u\nEVBYOBZVVTHBLGvIFBEiXex2O5KSkqDX64NdChHRiPr66wj4AqSLgLo6Rf1aHpBiJtaBzrmROXPm\n9LhfrVZ7ex8X90oAoLKyEpWVld7bOTk5kCRpZIsNEdHR0WyL77AtfNgWPsFqi6uuAgAZ3YNExoQJ\nQlD/bfbs2eP9OiUlBSkpKf2eL8iyLI90UYNhNpuRlZUFoDNMjEYjPB4PRFGE3W5HdXU1MjMzUVpa\nCpPJNGBvpba2NgBVK58kSXC5XMEuQxHYFj5sC59gtUXXnIhvSEvGsmUtWLDABVFsD3g9AJCQkDDk\naxTRb7Jardi9ezcWL16MefPmQRA6k7mgoAAAkJiY6D1PpVJxuIuIQp4otmPBAhcOHmzErl1NOHiw\nMagB4i/F9ESGG3sinfgXpw/bwodt4cO28AnZnggREQ2eLAMHD8Zi377YYJfCECEiCiU2WxRyci7H\npk0S4uI6gl2Ost6dRUREvXM6I7B5swSzORaPPurC3LkeRCngNzh7IkRECtbSAjz3nBrTpo1HXJyM\nI0ec+OUvlREgAHsiRESKJMtAaWks1q/XwGhsw1/+8g0SE5X3zi2GCBGRwvzf/43B6tVxaG0Ftmw5\ngylTzgW7pD4xRIiIFOKrryKwcaMG5eUxWL68Cffc04JIha/HyBAhIgoyt1vA88+rsWuXCr/8pRtH\njjihVofGR/gYIkREQdLRAbz22lhs3qxBWlor3nmnHhMmKG/eoz8MESKiILBYorF6tQZjxgDFxQ34\n0Y/agl2SXxgiREQB9PnnkVi7VoOTJ8dg5UoX7rqrBcLFK8KHEH5OhIgoAJqaBKxdq8HMmeNhMLTh\nyBEn7r47tAMEYE+EiGhEnT8P7N4toqhIwrRprfjrX53Q6YK/XMlwYYgQEY2Qo0djkJ+vwfe+14GX\nXz4Ng+F8sEsadgwRIqJh9tlnUVizRoOqqig8+WQT7rjjbMgPW/WFIUJENEy+/VbAli0S9u4di0ce\nacaOHQ2IiQl2VSOLE+tERJeorQ347/9WISNDi7Y2AWVl9cjNdYd9gADsiRAR+U2WgXffjUFBQRyu\nvvo8XnvtNL7//fCb9+gPQ4SIyA8ffxyF/Pw4OBwRyM9vxLRprcEuKSgYIkREQ1BfH4HCQglvv+3b\nHGrMmGBXFTycEyEiGoSzZ4E//EGN224bD1Hs3BzqgQdGd4AA7IkQEfVLloG33urcHOraa9tQWvoN\nkpJCa5HEkaSonojdbu/zWElJCQDAbDYHqhwiGuVOnhyDn/70cjz3nITCwjPYufNbBshFFBMiVqsV\nRUVFfR43m81YsmQJ4uPjA1gVEY1GX38dgV//+jI8+OA45OS04ODBetx8s3J3FwwmxQxnGY3GfgMi\nNzcXqampAayIiEYbj0fA9u0qvPiiGvff78bRo6GzOVSwKKYnMhCHwwGr1YrS0tJgl0JEYaajA/jz\nn8di6lQt/vnPMTh4sB55eS4GyCAopicykOzsbABARUUFbDYbDAZDkCsionBgsURi+fIrIAjAtm0N\nuOGG0NwcKlhCoidiNptx4sQJAIAkSXA4HEGuiIhCmccTCbNZjbvvHo+5c2Nx//0tKC39hgHiB0X1\nRGS5e9fR4/FAFEVMmjQJWq0WAFBXV4cZM2Z0O6+yshKVlZXe2zk5OZAkaeQLDgHR0dFsi++wLXxG\nc1t89VUHHnhAxIkTUQAEADKczmgIAiBJIfF39Yjas2eP9+uUlBSkpKT0e74gX/ybO0gsFguKi4ux\ncOFC7wR6Xl4eNmzYAKCzN6JWq+F0Or1DW/2pra0d0XpDhSRJcLlcwS5DEdgWPqOxLdrbgVdeEbFx\nowbffiugM0C6yDh4sBFGoydY5SlCQkLCkK9RTIgMN4ZIp9H4y6IvbAuf0dYWx45FIz8/DhpNB+64\now35+eoe5+za1YTp05uDUJ1y+BMiihrOIiIaTlVVkSgoiMM//tG5OdTMmWdhs4kAZFzcE4mPD58t\nawOJA4BEFHbOnBHw1FMa3HXXFbjxxnM4fNiJWbM6dxdMTm7FsmUt6AwSAJCxbFkLkpNH5yq8l4o9\nESIKG21twP/8jwrPPqvGnXeeRVlZPa64onsPQxTbsWCBC5mZ51BXF4EJEwTo9R6IIpcz8QdDhIhC\nniwDf/1rDNas0eDKKzvw6qunce21fW8OJYrtMBo9MBq75ocYIP5iiBBRSPvHP6KQn6/Bl19GYtWq\nJmRmtkIQBr6OhgdDhIhC0unTnZtD7d8fi1//uhm/+IV71O/tEQycWCeikNLaCmzfrsKtt45HTEzn\n5lC/+hUDJFjYEyGikCDLwP79sVi3ToNrrjmPvXu/waRJnMsINoYIESleRcUY5Odr0NgYgY0bz2Dq\nVO7toRQMESJSrLq6CGzapEFZWQx++1sXfv5zDyIjg10VXYhzIkSkOC0tArZsUSMzU4vx49tx9KgT\nc+cyQJSIPREiUoyODuCNN8ZiwwYJP/xhGw4cqMfVV3PeQ8kYIkSkCB9+OAarV8ehowP4wx/O4MYb\nOe8RChgiRBRUX34ZifXrJfztbzFYsaIJP/1pCyI40B4y+E9FREHR3Cxg40YJt98+HsnJnfMe99zD\nAAk17IkQUUC1twOvvTYWmzdrcPPNrXj3XScSErgMe6hiiBBRwJSXR2P16jiMHSvjxRcbcP313NM8\n1DFEiGjE2e2RWLdOA5ttDFaubMLs2We5SGKY4OgjEY2YxkYBa9ZoMHv2FbjuujaUlTmRnc0ACScM\nESIadufPA7t2iZg6VYvmZgGHD9dj0aJmxMYGuzIabhzOIqJhdfhwDPLzNdBqO7B792mkpPS9ORSF\nPoYIEQ2LTz+Nwpo1GtTURGHVqkZMn87NoUYDhggRXZKGhgg884yEv/wlFosXN2PnzgZERwe7KgoU\nRc2J2O32Po9ZLBZYrVaUlpYGsCIi6su5c8ALL6iQkTEegiCjrMyJ+fPdDJBRRjEhYrVaUVRU1Osx\nu90OQRBgNBohiiJqamoCWxwReckycPBgLG67TYv33ovB66+fxtq1TRg3Tg52aRQEihnOMhqNiI+P\n7/VYeXk5Jk+eDADQ6XSoqKiAXq8PYHVEBAA2WxRWr45DQ0ME1q9vREZGq9+P5fFEoro6BrW1EUhI\n6EBSUitEkSv2hhrFhEh/3G431Gq193Zzc3MQqyEafRyOCGzeLOHQoVgsXerCnDkeRF3Cbw+PJxLF\nxRIKC8cCEADIWLasBQsWuBgkIUYxw1lEpDwtLcBzz6kxbZoWl10m4+hRJ37xi0sLEACoqoq5IEAA\nQEBh4VhUVcVcaskUYCHRE1Gr1d7ex8W9EiIafrIMlJbGYv16DUymNuzbVw+9fvh6CF9/HQFfgHQR\nUFcXAaNx2L4NBYCiQkSWu0/MeTweiKKI9PR0VFdXw2AwwOl0wmQydTuvsrISlZWV3ts5OTmQJCkg\nNStddHQ02+I7bAuf/trigw8ikJcXi9ZWoLi4FTff3A5AHNbvf9VVACCje5DImDBBCPi/EV8X3e3Z\ns8f7dUpKClJSUvo9XzEhYrFYUF1djRMnTiA1NRUAUFBQgA0bNiAxMRHV1dWwWq1QqVQ9JtV7e6Iu\nlytQpSuaJElsi++wLXx6a4uvvorAhg0aHD8eg+XLm/Czn3Xu7TESTabXR2LZssgecyJ6vQcuV2Dn\nRPi68JEkCTk5OUO6RpAv/vM/TNTW1ga7BEXgD4gP28LnwrZwuwU8/7wau3ap8MtfuvHII81QqUb+\n14LHE4mqqhjU1UUgPr4DycnBeXcWXxc+CQkJQ75GMT0RIgqsjg7f5lDp6a145516TJgQuF/iotgO\no9HDOZAQxxAhGoXefz8Sy5dfgTFjgOLiBvzoR9wcivzDECEaRT7/PBJr12pgtcYgL+8M9/agS3bJ\nnxNxOp3DUQcRjaCmJgFr12owc+Z4GAxt+PBDN+66iwFCl25IPZHS0tJunxaXZRk1NTV44oknhr0w\nIrp0588Du3eLKCqSMG1aK/76Vyd0ug6MHSuNyLuuaPQZUogkJiZCp9N1u6+iomJYCyKi4XHkSOfm\nUOPGdeDll0/DYODmUDT8hhQiycnJEMXuHzrKysoa1oKI6NJ89lkU8vM1qK6OwpNPNuGOOzhsRSOn\nzzmR0tJSeDyebvddHCAAsHv3btTU1HB5dqIga2gQ8F//pcFPfnI5brqpc+jqzjsZIDSy+uyJ6HS6\nXkPjYllZWdBqtcNaFBEN3rlzwB//qMJzz6nx4x+fRVlZPS6/vCPYZdEo0WeIfPbZZ975DpPJ5F2K\nhIiUQZaBd9+NQUFBHCZOPI8///k0vv99zntQYPUZIiaTyTsH0rVtLd+dRaQMH38chfz8ODgcEcjP\nb8S0af5vDkV0KfoMEeMFaxEkJiZ6/893ZxEFT319BAoLJbz9diwefdSFuXM9GDMm2FXRaNbnxPqO\nHTt63JecnAytVtvtP747i2jknT0L/OEPatx223iIoowjR5x44AEGCAVfnz0RWZZhs9mQlJTknWAf\nzEQ7EQ0fWQbeeqtzc6hrr21Daek3SEri9rGkHH2GyIIFCwAAdrsdbrcbBoMhYEUREXDy5BisXq1B\nc3MECgvP4OabzwW7JKIeBvywYdd8iNVqhU6n49t5iUZYbW0ENm7U4L33YrBsmQs5OR5ERga7KqLe\n9TkncuLECe/XHo8HOp0OTqcTu3fv7vEhRCK6dB6PgKIiNaZP1yIhoR1Hjzpx330MEFK2PnsiRUVF\nUKlUcLvdUKlU0Ol0UKlUEEURZrMZ2dnZgayTKGx1dACvvz4WGzdqcMMN53DwYD0mTuS8B4WGPkNk\n/vz5MJlMcDgcva6ZRUSX7oMPovHUUxpERADbtjXghhu4ORSFlj5DpOutu1qtlpPrRMPsiy8isW6d\nBh9+GI2VK5tw990tiLjk3X2IAq/Pl+2Fm00lJibCYDDAZrNxEyqiS+ByCdiwQcIdd4zH97/fhmPH\nnPjpTxkgFLr67Ins2LEDRqMRDocDzc3N8Hg8cLlccDqdSE9Px/z58wNZJ1FIa28H/vQnEU8/LSEj\noxVmsxNXXslFEin09fthQ8C31IlarYZKpYJareb8CNEQvPdeNFavjoMkdWDXrgZMnsx5DwoffYbI\n0qVLGRZEl6CqKhJr12rwySdj8MQTTZg1i3t7UPjpcyQ20AFisVhgtVpRWlra6/GSkhIAgNlsDmRZ\nREN25oyAp57S4K67rsANN7Th8GEnfvxjBgiFJ0VM59ntdgiCAKPRCFEUe90l0Ww2Y8mSJYiPjw98\ngUSD0NYG7NypQkaGFmfPCigrq8d//mczYmODXRnRyBnSHusjpby8HJMnTwbQuaNiRUUF9Hp9t3Ny\nc3O5MRYpkiwDhw7FoKBAg4SEdrz66mn827913xzK44lEdXUMamsjkJDQgaSkVogiP1BIoU8RIeJ2\nu6FWq723L9z4qovD4YDVaoXdbuen5UkxPvkkCvn5Gnz1VSRWrWpCZmZrj2ErjycSxcUSCgvHAhAA\nyFi2rAULFrgYJBTyFDGcNRjZ2dkwGo1wuVyw2WzBLodGudOnI7BiRRxyci7H9OmtOHSoHllZPQME\nAKqqYi4IEAAQUFg4FlVVMYEsmWhEKKInolarvb2Pi3slQOd8iCRJSE1NhSRJcDgc3T49X1lZicrK\nSu/tnJwcSJIUmOIVLjo6mm3xneFoi9ZWYPv2MXj22Wjk5JzH3//uwbhxEQD6flyHQ4AvQLoIcDqj\ngvZvw9eFD9uiuz179ni/TklJQUpKSr/nKyJE0tPTUV1dDYPBAKfTCZPJBKBz9WBRFDFp0iTvEvR1\ndXWYMWNGt+t7e6IulyswxSucJElsi+9cSlvIMnDgQCzWrtXgmmvO4/XX6zFpUudQ1EAPqdOJAGR0\nDxIZWu15uFzBWRGbrwsftoWPJEnIyckZ0jWKGM66cM8SlUrlnVQvKCgAAOj1epSXl8NisSA+Pr7H\npDvRSKqoGIN77rkcRUUSNm48g127GrwBMhjJya1YtqwFnUECdM2JJCe3jki9RIEkyF0fTQ8ztbW1\nwS5BEfhXls9Q26KuLgKbNmlQVhaD3/7WhZ//3P+9PTyeSFRVxaCuLgLx8R1ITg7uu7P4uvBhW/gk\nJCQM+RpFDGcRKUlLi4Dt21V48UUV5s714OhRJyTp0v7WEsV2GI0eGI3DVCSRQjBEiL4jy8Abb4zF\nhg0Srr++Dfv3f4Orr+ZbcIn6wxAhAvDhh2OwenUcOjqArVvP4MYbzwW7JKKQwBChUe3LLyOxfr2E\nv/0tBitWNHFvD6Ih4o8LjUrNzQI2bpRwxx1XYNKk8zh61Il77mGAEA0VeyI0qrS3A6+9NhaFhRrc\ndFMr3nmnHgkJ3ByKyF8MERo1jh2LxPLl4yGKHXjxxQZcdx03hyK6VAwRCnt2eyTWrdPg449jsGLF\nGcyezb09iIYLR4ApbDU2ClizRoPZs6/Adde14YMP3MjOZoAQDSeGCIWd8+eBXbtEZGRo0dws4PDh\neixaxM2hiEYCh7MorJSVxSA/XwOttgO7d5/GD35wfuCLiMhvDBEKC59+GoU1azSoqYnCqlWNmD69\n9709iGh4MUQopDU0RODppyW89VYsFi9uxs6dDYiODnZVRKMH50QoJJ07B2zfrkJGxnhERMgoK3Ni\n/nw3A4QowNgToZAiy8Dbb8eioECDpKTzeP310/jXf+W8B1GwMEQoZNhsUVi9Og4NDRFYv74RGRnc\n1Iko2BgipHgORwQ2b5Zw6FAsli51Yc4cD6L4yiVSBM6JkGK1tAC/+50a06ZpcdllMo4edeIXv2CA\nECkJfxxJcWQZePPNsVi/XsLkyW3Yt68eej03hyJSIoYIKcrf/965OdS5c8DvfncG6encHIpIyRgi\npAhffRWJDRskHD8eg+XLm/Czn3FvD6JQwB9TCiq3W8DmzRJmzBiPq69ux9GjTtx7LwOEKFSwJ0JB\n0dHRuTnU5s0apKd3bg41YQLnPYhCjWJCxGKxQKVSwW63Izs7e8jHKXRYLNFYvVqD6Ghgx44G/PCH\n3ByKKFQpYtDAbrdDEAQYjUaIooiampohHafQUFMTifnzv4clSy7Dww834803v2GAEIU4RYRIeXk5\nVCoVAECn06GiomJIx0nZmpoEFBRo8OMfXwGjsQ1Hjjhx113cHIooHChiOMvtdkOtVntvNzc3D+k4\nKdP588Du3SKKiiRkZp7FoUP10Ok6gl0WEQ0jRYQIhZ8jRzo3hxo3rgMvv3waBgMXSSQKR4oIEbVa\n7e1dXNzrGMzxyspKVFZWem/n5ORAkqQRrjo0REdHB7QtPv00AitXxuCf/4zAunWtmDXrPARhbMC+\nf38C3RZKxrbwYVt0t2fPHu/XKSkpSElJ6fd8RYRIeno6qqurYTAY4HQ6YTKZAAAejweiKPZ5vEtv\nT9TlcgWsfiWTJCkgbdHQIGDLFglvvBGLRx5pxvbtbsTEAEoaeQxUW4QCtoUP28JHkiTk5OQM6RpF\nTKwnJiYCAKxWK1QqFfR6PQCgoKCg3+MUfOfOATt2qHDrrVq0twsoK6tHbm5ngBBR+BNkWZaDXcRI\nqK2tDXYJijBSf2XJMvDuuzFYsyYOev15rFrVhGuuUfa8B//i9GFb+LAtfBISEoZ8jSKGsyi0fPxx\nFPLz4+BwRGDNmkZMm8bNoYhGK4YIDVp9fQQKCyW8/XYsHn3UhblzPRgzJthVEVEwKWJOhJTt7Flg\n61Y1brttPERRxpEjTjzwAAOEiNgTCUseTySqq2NQWxuBiROBf/mXSIji0Bc3lGXgrbdisW6dBj/4\nQRtKS79BUhIXSSQiH4ZImPF4IlFcLKGwcCwAAYCMZcsisWCBa0hB8tFHY5Cfr0FzcwSefvoMbr6Z\nm0MRUU8czgozVVUxFwQIAAgoLByLqqrBvee2tjYCS5ZchnnzxiEnpwUHD9YzQIioTwyRMPP11xHw\nBUgXAXV1/f9TezwCnnlGwvTpWlx5ZefmUPfd50Fk5IiVSkRhgMNZYebKKzsAyOgeJDLi43tf+LCj\nA/jf/x2LTZs0uOGGczh4sB4TJ3Leg4gGhyESZpKTW7FsWctFcyItSE7u+VmOv/2tc3OoiAhg27YG\n3HAD9/YgoqFhiIQZUWzHggUuZGaeQ11dBCZMEKDXe7pNqp86FYl16zT4+9+jsXJlE+6+m3uaE5F/\nGCJhSBTbYTR6YDR2LenQGSAul4Df/16NkhIVHnqoGc8+ewZjx4blqjdEFCAMkVGgvR34059EPP20\nhIyMVpjNzu/mToiILg1DJMyVlUXi8cfHQ6PpwB//2ACTifMeRDR8GCJhqqoqEmvXavDppzFYufIM\nZs7knuZENPw4nRpmzpwR8NRTGtx11xW48cZz+OADN2bNYoAQ0chgiISJtjZg504Vpk7V4uzZzs2h\nHn6Ym0MR0cjicFaIk2Xg0KEYFBRokJDQjldfPY1rr1X25lBEFD4YIiHsk0+ikJ+vwVdfRWLVqiZk\nZrZy2IqIAoohEoJOn+7cHGr//lj85jfN+I//cHNvDyIKCs6JhJDWVmDbNhVuvXU8oqM7N4eaN48B\nQkTBw55ICJBlYP/+zs2hrrnmPPbu/QaTJnGRRCIKPoaIwlVUdG4O1dgYgY0bz2DqVO7tQUTKwRBR\nqLq6CGzapEFZWQwee8zFvT2ISJEUMydisVhgtVpRWlra6/GSkhIAgNlsDmRZAdfSImDLFjUyM7UY\nP75zc6j772eAEJEyKSJE7HY7BEGA0WiEKIqoqanpcY7ZbMaSJUsQHx8f+AIDQJaBvXvHIiNjPD75\nZAwOHKjHypUuSBJX2SUi5VLEcFZ5eTkmT54MANDpdKioqIBer+92Tm5uLlJTU4NQ3cj78MMxWL06\nDu3twO9/fwapqZz3IKLQoIgQcbvdUKvV3tvNzc09znE4HLBarbDb7cjOzg5keSPmyy8jsX69hBMn\nYrBiRRP+/d+5ORQRhZaQ+ZWVnZ0No9EIl8sFm80W7HIuSXOzgI0bJdx++3gkJ7fj2DEnfvYzBggR\nhZ6A9UTMZjOE79bkkGUZgiBAp9PBYDBApVJ5ex8X90q6rpUkCampqZAkCQ6HAwaDwXu8srISlZWV\n3ts5OTmQJCkAz2po2tuBkpIxWLs2Grfe2o7jxz2YMAEA1ANd6rfo6GhFtkUwsC182BY+bIvu9uzZ\n4/06JSUFKSkp/Z4vyLIc9Jlbu92O6upqZGZmorS0FCaTCXq9Hh6PxzvRrtVqIYoiiouLMWPGjB5z\nJherra0NTPGD9P770cjPj4ModmD16iZcd11gNofq3B7XFZDvpXRsCx+2hQ/bwichIWHI1yhiACUx\nMREAYLVaoVKpvAFRUFAAANDr9SgvL4fFYkF8fPyAAaIkdnskfvWr7+Gxxy7D4sUu7N17OmABQkQ0\n0hTRExkJwe6JNDYKePZZCa+9Nha5uW489FAzYmMDXwf/yvJhW/iwLXzYFj4h2xMJJ+fPA7t2iZg6\nVQu3W8Dhw/VYtCg4AUJENNIU8RbfcHH4cAzWrNFAq+3AK6+cxg9+wM2hiCi8MUSGwaefRmHNGg1q\naqKwalUjpk8Pz82h7rvvPsyePRtz5szp85z169fDarWisbERL7zwAiZOnNjt+OOPP45Tp05BEARs\n2rQJEydOxLZt21BaWgpBECDLMmw2G8rLy7F161ZUVFRAEATk5eXhlltuGemnSERDxBC5BA0NEXjm\nGQl/+UssFi9uxs6dDYiODnZVI+Pzzz+HIAj9BojNZoPNZsMrr7wCm82G5cuX45VXXvEeLykpgSAI\n3uMLFiyxWKJVAAALjElEQVTAgQMH8PDDD+Phhx8GAJw6dQrr169HTU0NvvjiCxw4cAA2mw3Lli3D\ngQMHRvx5EtHQMET8cO4c8NJLKmzdqsbdd7egrMyJceNG7v0JCxcuRFNTU7e/7i+8r+uv9H379uHo\n0aN47733MH/+fHz88cc4fvw4SkpKsG7dukE9BtDZW7BarQCAzZs3w2Aw4Pnnn8fJkyexf/9+zJw5\ns9c6jx07hqlTpwIADAYDKioquh2vqKhARkaG93jX97jQ448/juLiYnz++efe+xoaGrzL4hCRsjBE\nhkCWgbffjkVBgQbJyeexd+9pTJo0svMe27Ztw/XXX4/c3FzYbDYcO3YMjY2N0Ov1yMvLQ1NTE9LT\n070ftrRarXj//fexb98+2Gw2fPTRR9i8efOgH6Ort7B//340NTXhzjvvxPvvv49HHnkEX3zxBWbO\nnIl9+/Z5h5+6LFq0CN9++y2uu+66Pp+LyWRCaWkpZs6cibfeeqvb9QCwb98+ZGRkQJIkGAwGyLKM\nm266CadOncILL7wwMg1MRJeEITJINlsU8vPjcPp0BNavb0RGRmtAvu9HH32ExYsXA+j8691gMODx\nxx/H7NmzAQAajabb+RfOG3R9PZTHqKiogNVqRW5uLmRZRlxcXI+aZs2ahVmzZvW4/9ixYzh16pT3\ndlNTU7fjc+fOxeeff445c+Zg4sSJPWrfunWr99Oy27Ztg8lkwiuvvIJTp07hzjvv7LMHRETBwxAZ\ngNMZgc2bJZjNsVi61IU5czyICmCr6fV6vPfeezAYDN6hqsTERBw9ehQ333xzt2Gf4XgMk8mEyy67\nDHl5eQCA7du393i8rp5IF0EQsGjRItxyyy1Yv349cnNzYbVae0yEnzp1CrfccgtWrlwJm82GxsZG\n77GuOZeu5Se+/fZbjBs3DgB6DTIiUgaGSB9aWoAdO9QoLlbh5z9vwdGjTmg0gf9cZl5eHhYuXIj7\n7rsPjY2NKC4uxlVXXdXjvuF6jLlz53rvFwQB999/f4/H66snAgBGo9F77aZNm7y9iMrKSlx99dVY\nt24dnn/+ecTFxaGwsNB73f79+7utzrxo0SIsXLgQb775JgRBGPA5ElFw8BPrvTh+PBq/+c1lMJna\n8MQTTdDr24exssDip3F92BY+bAsftoWPP59YZ0+kF1ptO373uzNIS+PmUERE/WGI9CI5uR3JyaHb\n+yAiChSunUVERH5jiBARkd8YIkRE5DeGCBER+Y0hQkREfmOIEBGR3xgiRETkN4YIERH5jSFCRER+\nY4gQEZHfFBUidru9z2MWiwVWq7XbEuRERBRcigkRq9WKoqKiXo/Z7XYIggCj0QhRFFFTUxPY4oiI\nqFeKCRGj0Yj4+Phej5WXl0OlUgEAdDpdj727iYgoOBQTIv1xu91Qq9Xe283NzUGshoiIuoREiBAR\nkTIFbD8Rs9kMQRAAALIsQxAE6HQ6GAyGAa9Vq9Xe3sfFvRIiIgqegIVIVlbWgOdcvFOvx+OBKIpI\nT09HdXU1DAYDnE4nTCZTt/MqKytRWVnpvZ2Tk+PXNo/hSpKkYJegGGwLH7aFD9vCZ8+ePd6vU1JS\nkJKS0u/5ihnOslgsqK6uxokTJ7z3FRQUAAASExMBdL6DS6VSQa/Xd7s2JSUFOTk53v8ubITRjm3h\nw7bwYVv4sC189uzZ0+136UABAihoe9y0tDSkpaV1u2/Dhg3erzMzMwNdEhERDUAxPREiIgo9YRki\ng+mCjRZsCx+2hQ/bwodt4eNPWwjyxbPZREREgxSWPRGiwa61xrXYaDQbjvUKQz5EBnqio2nhxoGe\nq9lshtlsRklJSYArC6zBrrVmtVphtVoDW1wQDPS6sNvtsFgssFgsAa4s8Ab7++LQoUMBrizwhmu9\nwpAOkYGe6GhauHGg52q1WmEymZCVlQWHwwGbzRacQgOAa635DOZnYO/evUhLS4PT6RzVPyN2ux06\nnQ5GoxFarTas2wIYvvUKQzpEBnqio+mXyUDP1eFweP/q1ul0cDgcAa8xUAaz1prdbofRaOzxAddw\nM9DrwmKxYNKkSQCA7OzsHp/BCieD+X3Q1Ut3OBxh3RYDGcp6hSEdIgM90dG0cONAzzUrK8v7WRu7\n3Y7k5OSA1qc0brc72CUExECvi6qqKjQ3N8Nut4f9kO9AbZGYmAitVosHH3yQn2AfgpAOERo6u92O\npKSksP4ra6C11ux2u3fNtq713EYztVrtXRViNMyL9MXj8UCtVuMnP/kJXnjhBTidzmCXFDRDWa8w\npENkoCc6mhZuHOxztVqtmDNnTiBLC7j09HTvcN2Fa615PB4AnUMVJ06cgNlshsvlCuux74FeF5Ik\nQafTAQBEUUR1dXXAawyUgdrCbDbj7rvvRnZ2NhYuXDgqArW39QqBvn+GehPSITLQL4uhNESoG6gt\ngM4fkuzsbAAI63cl9bXWWtdabGlpaUhNTQXQvX3C0UCvi7S0NO9xj8cT1sOcg/kZ6ZKamuqdPwlX\nl7Je4YVC/sOGhw4dglarhdPp9I755+Xledfd6u14uOqvLaxWK7Zs2QKVSgW3242lS5cOahl+Cn2D\n+RlRqVSorq4O+17qQG1RWloKnU6H5ubmsP99MVxCPkSIiCh4Qno4i4iIgoshQkREfmOIEBGR3xgi\nRETkN4YIERH5jSFCYWnFihW49957e/0vVNntduTl5QGA93lceN9grrvwWqLhoJg91omG08aNG70f\nIlu0aBGWLl2KpKSkIFd1aXQ6HebOnTvgfYO5jmi4sCdCYUsURYiiCFmWoVarIYoimpubsXbtWpSU\nlCAvLw92ux0rVqzwXmO1WrF27Vrv7YqKCixevBjz5s3Dli1b+vyEe3FxMR588EHMmzev20KGfV3v\ndDqxYsUKlJaW4sEHH8SSJUu6Lb/S2+M5HA68/PLL3vrmzZvnvQ8A1q5d220fjJKSEuzYsaPHOV3X\nFhcXd6u1pKQEu3fvHnpD06jGEKFRx2q1oqWlBQsXLgTQcxHGrttutxtbtmzBwoULsXXrVqjVamzf\nvr3H41ksFthsNjz//PNYtWoVSkpK4PF4Bry+a3+Ll156CQaDwbsMeV+P11Xbk08+CQDYuXNnt3rT\n0tJw8uTJbs8zPT292zkXXmsymboth26xWDBlyhS/2pRGL4YIjUoPPfTQgCsZHz9+HCaTCQaDAaIo\nYs6cOf2uOVZXVwe9Xo+XXnoJoigOeL1KpcLs2bMBANOnT++xNPnFjzeQKVOmeNdBcrvdcDqd/S5t\nk5aW5q3H4XDA7XaH9erONDI4J0KjTteqtQNxOBywWCyYN28egM4VT1taWnqcl5aWBrfbjaKiIrjd\nbu9KsANd39dKy3093kBEUURSUhJsNhvq6uq8vZD+mEwmWK1W2O12ZGVlDXg+0cUYIjTqXbh83IWb\nVel0OqSlpeHRRx/13tfbHhNOpxNGoxGZmZlwOp0oKChAUlLSoK8f7OMNZlXZ9PR0nDx5Ena7Hfff\nf/+A5xuNRhw/fhzV1dXIzc0d8Hyii3E4i0adC0NDpVLBbrfD6XTC7XbjjTfe8B6bMmUKrFYrrFYr\n3G43iouLsWPHjh6PZ7FYUFRUBKfT6X1st9uNKVOmoKKios/r+1r79OLHEwRh0DsxpqWlwWw2o7q6\nutvQVF/fKy0tDcePH4fT6eRQFvmFPREKe31NnAOAVqtFVlYWFi9e7H0rrNlsBtA5PLR06VLvLncm\nk6lbr6JLdna2911YKpUKWVlZ3v1KHnvssT6v72tXxb4ez263e89JTU3Fvffei02bNnW7VqvVQpKk\nHnvnXPi9uq599dVXodVqoVarMXny5H7bkKgvXAqeaJRbsWIFcnNz2RMhv3A4i2gUq6io4FAWXRIO\nZxGNUhaLBcXFxVi6dGmwS6EQxuEsIiLyG4eziIjIbwwRIiLyG0OEiIj8xhAhIiK/MUSIiMhvDBEi\nIvLb/wNUPTmyd5xA/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc369caaa50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.style.use('ggplot')\n",
    "plt.rc('text', usetex=True)\n",
    "#plt.rc('font', family='serif')\n",
    "slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(conf_mats[1,1,:], weights[0][1,1,:])\n",
    "print slope, intercept\n",
    "plt.scatter(conf_mats[1,1,:], weights[0][1,1,:], s=40)\n",
    "x = np.arange(0.05, 0.95, 0.01)\n",
    "plt.plot(x, slope*x + intercept, 'b-')\n",
    "plt.text(0.15, -0.7, \"corrcoef=%.3f\" % (np.corrcoef(conf_mats[1,1,:], weights[0][1,1,:])[0,1]))\n",
    "plt.xlabel(\"True sensitivity\")\n",
    "plt.ylabel(r\"$W^r_{1,1}$\")\n",
    "plt.savefig(\"sensitivities.png\")\n",
    "plt.savefig(\"sensitivities.eps\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True values for specificity: [ 0.59088  0.89792  0.49712  0.8992   0.90016]\n"
     ]
    }
   ],
   "source": [
    "print \"True values for specificity:\", conf_mats[0,0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learned weights for specificity: [ 0.72044832  1.77842236  0.48955911  1.62380767  1.83825517]\n"
     ]
    }
   ],
   "source": [
    "print \"Learned weights for specificity:\", weights[0][0,0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.19455732106 -1.12836023932\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAETCAYAAAD3WTuEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X98U/W9P/DX6Y+UJjmFOU1qnd62sO26/MB5H/u2oOK0\nhU3UTtRHleLdVSa0m6AThoA/HxOlLRVQVJQi6J0WtY6Lq4AbS6fSWlKvm5qTuLspPYizNgdBaH6U\n8qP5/lGbUPo7tMlJ8no+Hj4eTc4PPnmb5JXPOefzOUIgEAiAiIgoDEnRbgAREcUuhggREYWNIUJE\nRGFjiBARUdgYIkREFDbVhIjNZoPNZkNNTU2/y3uet9lskWwWERENQhUhIkkSrFYrCgsL4Xa74XQ6\n+6xjs9lw5513IjMzc8j9uVyusWhmTGItQliLENYihLUICacWqggRt9sNSZIAAEajEW63u886ZWVl\nWLduHcxm85D745sihLUIYS1CWIsQ1iIknFqkjEE7RqywsDD4tyzLuOSSS/qs0xM0siyjqKgoks0j\nIqIBqKIn0kOWZeTm5iI7O7vPsqKiIlgsFng8nn4PdxERUeQJapr2pK6urt9ehs1mgyiKyMvLQ11d\nHXQ6HQoKCoLLXS5Xr25YcXFxRNpLRBRvamtrg3+bTCaYTKZB11dNiNhstuBhLUmSYLFY4Pf7odVq\nsW/fPhgMBmi1WlRXV2PGjBn99lZO1draGoFWq58oivB4PNFuhiqwFiGsRQhrEZKVlTXibVRxOEuS\nJGzZsgULFy7E3LlzIQgCAGDFihUAgOzsbDQ1NcFutyMzM3PIACEioshQTU9ktLEn0o2/skJYixDW\nIoS1CInZnggREcUmhggREYWNIUJERGFjiBARUdgYIkREFDaGCBERhY0hQkREYWOIEBFR2BgiREQU\nNoYIERGFjSFCRERhY4gQEVHYGCJERBQ2hggREYWNIUJERGFjiBARUdgYIkREFDaGCBERhY0hQkRE\nYUuJdgN62Gw2AIDb7cacOXP6LLfb7dDpdJBlGUVFRZFuHhER9UMVPRFJkmC1WlFYWAi32w2n09lr\nuSzLEAQBFosFWq0W+/bti05DiYioF1WEiNvthiRJAACj0Qi3291reVNTE3Q6XXC5w+GIeBuJiKgv\nVRzOKiwsDP4tyzIuueSSXst9Ph/0en3wsdfrjVjbiIhoYKroifSQZRm5ubnIzs6OdlOIiGgYVNET\n6SFJEkpKSvo8r9frg72P03slAOByueByuYKPi4uLIYri2DY2Rmg0GtbiG6xFCGsRwlr0VltbG/zb\nZDLBZDINur4QCAQCY92o4bDZbMHDWpIkwWKxwO/3Q6vVQpZltLS0oKCgAHV1dbBarUP2VlpbWyPQ\navUTRREejyfazVAF1iKEtQhhLUKysrJGvI0qDmdJkoQtW7Zg4cKFmDt3LgRBAACsWLECAJCTkxNc\nT6fT8XAXEZFKqKYnMtrYE+nGX1khrEUIaxHCWoTEbE+EiIhiE0OEiIjCpqqrs4iIEo3fn4yWljS0\ntiYhK6sLubmd0GpPDrpNIADs3p2GEyeAgoLOCLW0f+yJEBFFid+fjOpqET/5yXjcdlsGfvKT8aiu\nFuH3Jw+4zQcfpKK4+Nu4//7x+OYapKhiiBARRcnevWmoqkoH0JMGAqqq0rF3b1qfdT/5JAXz5n0L\nt99+Fq67rgNvvaXgyiuj2wsBGCJERFHzxRdJCAVIDwFtbUm91vnNb8bjhhu+jR/+8DgaG92YM8eP\nFJWcjFBJM4iIEovfn4xAIAlAAL2DJIDMzC4cOiTg6adFvPKKFrfc4kNDg4Lx49U3IoM9ESKiKNi7\nNw0vvKDBrbceQ3eQAEAAd9xxFH/6UwqmTTPA5xNQX69g+XKPKgMEYE+EiCgqvvwyCY2NGgDH8dBD\nHfD7BTidyaipScO0aZ2oq/sKubmDX6WlBgwRIqIoOPfcLgABNDamorEx9ZtnA3jySS+uvz52RtDz\ncBYRURTk5nbihhs6ceqhrCVLOvDTn/qj2awRY0+EiCjC3n8/FRUVGVCUZDzwgB+5uSdx7rldmDhx\n6IGGasMQISKKkH/8IwWVlSIkKRWLF3tw440dqrlUN1wx3nwiIvX717+S8dhjIt56Kw2/+pUX69d/\njXHjot2q0cEQISIaIwcPJmHdOj1+/3stbr21e6xHRkboUt1w5s1SG4YIEdEo83oFVFfrsGmTHrNm\n+fH22wrOOaer1zo982aFpj3pPrE+f74npoKEV2cREY2Szk5g0yYdLr3UAFlOwc6dB/DII+19AgQY\n2bxZasaeCBHRGTp5Eti2LR2PPSbiu989gZqagzCZTgy6zZdfDjxvlsUyZk0ddarqiciyPOCympoa\nAIDNZotUc4iIBhUIALt2pWHGjHPw4os6PP74Ybz44qEhAwQIDTY8bY/IzOzba1Ez1YSIJElYs2bN\ngMttNhvuvPNOZGZmRrBVRET9e+89DWbN+jYqKjKwdGk7Xn/9K+TnHxv29hMndmLJkg6cPthw4sTo\nT+8+Eqo5nGWxWAYNiLKyMuTl5UWwRUREfX38cQoqKjLwz3+mYPFiD66/vgPJA99DakBa7UnMn+9B\nQcExtLUlITOTgw3HlNvthiRJkGUZRUVF0W4OESWY/fuTUVUloqEhDQsWeLFx4yGkneE5cK32JCwW\nf0ydAzmdag5nDaWoqAgWiwUejwdOpzPazSGiBHHgQBIeeCADM2eejZycE2hsVHD77b4zDpB4ERMh\nYrPZ0NzcDAAQRRFutzvKLSKieOfxCKiqEvHjHxsgCMDbbx/AokVe6PXqvK9HtKjqcFYg0Pt/jt/v\nh1arxaRJk2AwGAAAbW1tmDFjRq/1XC4XXC5X8HFxcTFEURz7BscAjUbDWnyDtQhhLUJOr8XRo8Bz\nz6Vi7VoNCgtPoqHBj3/7twAAXfQaGUG1tbXBv00mE0wm06DrC4HTv7mjxG63o7q6GqWlpcET6MuX\nL0d5eTmA7t6IXq+HoijDOifS2to6pu2NFaIowuOJnXsTjCXWIoS1COmpxYkTwNat3WM9zObjWLrU\ng3//96Ev1Y0nWVlZI95GNSEy2hgi3fhlEcJahLAWIXq9iNdeO47KShFnndWF5cvb8aMfHY92s6Ii\nnBBR1eEsIqJI2rNHg8pKLbzeLjzwQDuuvLITwumDyGlQDBEiSjhOZ/dYj717U/DAA8fw059+jaSY\nuMxIfRgiRJQwZLl7rMeePWm46y4PNm/249vfFsEje+Fj9hJR3HO7k7Bs2Xhce+3Z+N73usd63Hqr\nHxpNtFsW+9gTIaK4deSIgPXr9XjpJR1uusmP3bsP4KyzYmuCQ7VjiBBR3OnoAF54QYdnntFj+vSj\n2LVLwXnnMTzGAkOEiOLGiRNAba0Wa9aIuOiiY9i69SC++93EGusRaQwRIop5gQCwY8c4rFolwmjs\nQnX1IVx8cWKO9Yg0hggRxbSGBg3KyzNw8iTw8MPtuPxyjvWIJIYIEcWkjz5KRXl5Bj7/PBlLl7bj\nmmuOcqxHFDBEiCim7N2bjFWrMvD++xrcdZcHs2f7kZoa7VYlLoYIEcWEL79Mwtq1It58cxxKS314\n/PHDSE+Py6n/YgpDhIhU7fBhAU8/rceWLTqUlPiwe7eCb32L4aEWDBEiUqWODgGbNumwYYMOV111\nFDabgnPP5VgPtWGIEJGqHD8OvPyyFk88IeJHPzqG11//ChMnnox2s2gADBEiUoWuLuCNN8Zh1aoM\nnH/+SWzefAiTJ3Osh9oxRIgoqgIBYPfuNJSXi0hKAioqDuOyy45Fu1k0TAwRIoqav/2te6xHW1sy\n7rmne6wHBwrGFoYIEUXcJ5+koLJSxAcfaLB4sQfFxX6k8NsoJqlqfKcsywMus9vtkCQJdXV1EWwR\nEY2mL75IxqJFE3D99d/GxRcfR2OjGyUlDJBYppoQkSQJa9as6XeZLMsQBAEWiwVarRb79u2LbOOI\n6IwcOiTgt7/NwIwZ58BgOInGRgW/+pUX6enRbhmdKdWEiMViQWZmZr/LmpqaoNPpAABGoxEOhyOS\nTSOiMPl8Ah5/XI9p0ww4elTAX/6iYNkyD8aP52DBeBETnUifzwe9Xh987PV6o9gaIhrKsWPAli3d\nYz2mTOnEG298hZwcjvWIRzERIkQUG7q6gNdfT0dVlYiJE0/gxRcPwmzmTaHiWUyEiF6vD/Y+Tu+V\nEFH0BQJAfX0aKioyMG5cAKtXH8bUqRzrkQhUFSKBQO/jpH6/H1qtFlOmTEFLSwvMZjMURYHVau21\nnsvlgsvlCj4uLi6GKIoRabPaaTQa1uIbrEXIaNaiuTkJDz2UhoMHBTz00DFcffUJCEIagLRR2f9Y\n4/uit9ra2uDfJpMJJpNp0PWFwOnf3FFit9tRXV2N0tJS5OXlAQCWL1+O8vJyAEB9fT0MBgMURUFB\nQcGQ+2ttbR3T9sYKURTh8Xii3QxVYC1CRqMW//d/3WM9XK5ULF7swY03diA5eZQaGEF8X4RkZWWN\neBvVhMhoY4h04wckhLUIOZNa/OtfyXjsMRFvvZWGO+7w4uc/92HcuFFuYATxfRESToio6nAWEanX\nwYNJeOIJPbZu1eLWW31oaFCQkRGXv0FpBBgiRDQor1dAdbUOmzfrMGtWB95+W8E55/C+HtSNIUJE\n/ersBF58UYenntJj2rRO7Nz5FS64gGM9qDeGCBH1cvIksHVrOlavFvH975/Ali0H8YMfcKwH9Y8h\nQkQAusd6/PnPaaiszIAoduHJJw/j//0/jvWgwTFEiAjNzRqsXJkBr1fA0qXtmD69k/f1oGFhiBAl\nsI8/TkF5eQY++SQFv/mNB7NmxeZYD4oehghRApJlAb/97QQ0NKRh4UIvnnvuENJiY4A5qYxqpoIn\norF34EAS7r8/A1dcoUVu7gk0Nir4xS98DBAKG3siRHHO70+GJKXh+efT8fbbqbjhhg68/74faWm8\npQKdOYYIURz7+utkLFjwLbz9dioAAUAA55wjQKPhJbs0Ong4iygOnTgBvPJKOq644pxTAgQABFRV\npePvf+dHn0YH30lEcSQQAHbuHIeCgnNQW6vFvHkdCAVIDwFffsnrd2l0MESI4sS772pw7bVnY+1a\nEQ891I6tWw9i2rTjAE6fJDGAc8/lxIk0OhgiRDFOklJRUnIWliyZgF/8woc//ekArryye7DgxImd\nWLKkA6EgCWDJkg5ceCEnUKTRMaIT67Iso6mpCW63G36/HzqdDrNmzUJ2dvYYNY+IBiLLyVi1KgN2\nuwZ33eVBSYkfGk3vdbTak5g/34OCgmNoa0tCZmYXJk7sREaGFryFBo2GEYWIoiiYM2dOr+fq6+sZ\nIkQR1NaWhMcfF7F9+zjMm+fDY48dhk438OEprfYkLBY/LJYINpISxohCpKcHotVq4ff74fV64fP5\nxqptRHSKI0cErF+vx0sv6XDzzX7s3q3grLN4boOia0QhUlhYCJvNhk8//RSCIGDixIkoLCwcq7YR\nEYCODuD55/V45hkdfvKTo9i1S8F55/GcBqnDgCFSV1eHwsJCaLXa4HNarRZFRUW91tuyZQumTp0K\nADysRTSKusd6aLF2rYiLLz6GbdsOYtIkDhIkdRnw6iyj0dgrQAZSWFiI7OzsMw4Qu90OSZJQV1fX\n7/KamhoAgM1mO6N/h0jtAgFg+/ZxuOIKA/7wh3Q899whbNz4NQOEVGnAnsinn34Kh8MBALBarcjL\nyxvw6qwzJcsyBEGAxWKB2+3Gvn37+oSSzWZDc3Mz5s+ff8b/HpFa7d6tQUVFBrq6gEceOYJp03hf\nD1K3AUPEarVi4sSJ0Gq1kGUZwNhdndXU1ITJkycD6O4BORyOPvssKytDXl7eGf07RNHi9yejpSUN\nra1JyMrqQm5uJ7Ta0P3KP/wwFeXlGfjii2Tcc087rrnmKJI4iotiwIAhYjnlesCcnBwAY3d1ls/n\ng16vDz72evvOLup2uyFJEmRZ7nNehkjN/P5kVFeLqKpKR88kiEuWdGD+fA9aW4FVqzLw179q8Otf\ne3DzzX6kpka7xUTDN2CIbNy4EfPmzev1XDSvzuoJDofDAafTCbPZHJF/l+hM7d2bdkqAAD2TIEpS\nMt57LwVlZT488cRhpKfzcl2KPQOGSCAQgNPpRG5ubvAEe39XZ40GvV4f7H2c3isBus+HiKKIvLw8\niKIIt9vdK0RcLhdcLlfwcXFxMURRHPV2xiKNRsNafCNatXC7BfQ3CWJysoAPPvDjW98SAOj72XLs\n8H0Rwlr0VltbG/zbZDLBZDINuv6AIdJzAluWZfh8vjH95T9lyhS0tLTAbDZDURRYrVYACB46mzRp\nEgwGAwCgra0NM2bM6LV9fy/UwzkdAACiKLIW34hWLYxGLbrnrjo1SAJYuNCPlBR/VKYf4fsihLUI\nEUURxcXFI9pmyFN3OTk5MJvNkCQJiqKE3bih/g0AkCQJOp0ueFJ9xYoVALrHnzQ1NcFutyMzM5Pj\nUShmHD8ONDcnQa8P4PRJECdO7Ixm04hGhRAIBPo9ENvc3By8GqrnJLqiKHA4HLjuuuuGNYYkmlpb\nW6PdBFXgr6yQSNaiqwt4441xWLUqAxdccAJ33+1Denpyr0kQT706K9L4vghhLUKysrJGvM2Ah7PW\nrFkDnU4Hn88HnU4Ho9EInU4HrVYLm83GK6SI+hEIAG+/nYby8gykpgZQWXkYl156LLickyBSvBkw\nRObNmwer1Qq32x0cL0JEA/vrX7vHeihKEpYu9WDmzKMcKEhxb8AQ6bl012AwROTkOlGs+uc/U1BZ\nKeKjjzRYtMiD4mI/UkY0tSlR7Brwra4oSvCKqJ4T306nEwaDIfg8USL74otkrF4twmZLwy9/6cVT\nT32N9PRot4oosgYdbNgzl5XX64Xf74fH44GiKJgyZUqfgYhEieLQoSQ8+aQetbVa/Od/+tDQoGD8\neA4UpMQ06GBDoLsXYjQaodfrodPpoNfreX6EEpLPJ6C6WodNm3QoKjqKv/xFgdHI+3pQYhswRBYt\nWsSwIAJw7BhQU6PFunUipk7txPbtXyE7O3qX5xKpyYAhwgChRNfVBbz+ejqqqkRMmnQCL754EGYz\n7+lBdCpeQ0J0mkAAqK9PQ0VFBsaNC2DNmsOYMuXY0BsSJSCGCNEp/vd/NSgvF/H110lYtsyDGTM4\n1oNoMAwRIgB//3sKKisz8PHHKVi82IMbb+xAcnK0W0WkfgwRSmiff56MqioR77yThgULvHj22UMY\nNy7arSKKHQwRSkhffZWEdev02LpVi7lzfWhsVCCKHOtBNFIMEUooHo+A6mo9Nm/W4frr/XjnHQVn\nn82xHkThYohQQujsBH73u1SsXm3A5Zd34s03D+CCCzjWg+hMMUQorp08CWzdmo7Vq0WYzcArrxzE\nhRdyrAfRaGGIUFwKBIBdu8ahokLE+PFdePLJwygoSIPHwwAhGk0MEYo7drsGK1dmwO8XcO+97Sgs\n7PxmrEdatJtGFHcYIhQ3XK4UVFRk4JNPUrBkiQfXXcexHkRjTTUhYrfbodPpIMtyv7feHWo5Ja7P\nPuse69HYmIaFC7147rlDSGOngygikqLdAACQZRmCIMBisUCr1WLfvn0jWk6JSVGScN9943H11Wdj\n4sQTaGxU8Itf+BggRBGkihBpamqCTqcDABiNRjgcjhEtp8TS3i6gslLEFVcYkJoawDvvHMDdd3uh\n13OwIFGkqeJwls/ng16vDz72er0jWk6JoaMD+O//1mH9ej0KCjqxa9cBnHcex3oQRZMqQoRoMCdO\nAK+9psXq1SImTz6G1147iO9/n5fqEqmBKkJEr9cHexen9zqGs9zlcsHlcgUfFxcXQxTFMW51bNBo\nNDFbi0AAeOONFDz8sAbnnBPA7353FHl5XQDSw9pfLNditLEWIaxFb7W1tcG/TSYTTCbToOurIkSm\nTJmClpYWmM1mKIoCq9UKAPD7/dBqtQMu79HfC/V4PBFrv5qJohiTtXj3XQ3KyzNw7JiABx88jB//\nuHusx5m8lFitxVhgLUJYixBRFFFcXDyibVRxYj0nJwcAIEkSdDodsrOzAQArVqwYdDnFH4cjFSUl\nZ+Geeybg9tt9+OMfD+CKKzp5YygilRICgUBcXtLS2toa7SaoQqz8ytq7NxlVVRl47z0N7rrLg9mz\n/dBoRvffiJVaRAJrEcJahGRlZY14G1X0RChxtbUlYenS8fjZz87GD35wHI2NCv7rv0Y/QIhobKji\nnAglnsOHBTzzjB4vvaTDzTf7sXu3grPOistOMVFcY4hQRHV0CNi8WYdnn9Xhpz89ij//WUFWFm8K\nRRSrGCIUEcePA6++qsXatSIuvvgYtm07iEmTONaDKNYxRGhMdXUB27ePw6pVGTjvvJPYtOkQLrro\neLSbRUSjhCFCY2b37jSUl3cP4lq58gimTeuMcouIaLQxRGjUffhhKsrLM/DFF8lYurQdV199FEm8\nDpAoLjFEaNR8+mkKKitF/O1vGvz61x7cfLMfqanRbhURjSWGCJ2x1tYkrF0r4o9/HIdf/tKHdesO\nIz2dl+sSJQKGCIXt0CEBTz8t4pVXtLjlFh8aGhRMmMDwIEokDBEaMb9fwHPP6VBdrcM11xxFfb2C\nzEyO9SBKRAwRGrbjx4EtW7R44gkReXnHUFf3FXJzeVMookTGEKEhdXUBdXXpqKoSccEFJ/DCC4dg\ntXKsBxExRGgQgQDw9ttpKC/PgEYTQGXlYVx66bFoN4uIVIQhQv16//1UVFRk4MCBJCxd6sFVVx3l\nPT2IqA+GCPXyj390j/WQpFQsXuzBjTd2IIXvEiIaAL8eCADwr38lY/VqEfX1afjVr7xYv/5rjBsX\n7VYRkdoxRBLcoUNJWLdOj9de0+LnP/ehsVFBRgbHehDR8DBEEpTPJ6C6WodNm3T42c+O4q23FBgM\nHOtBRCPDEEkwnZ1ATY0O69bpccklndi+/StkZ3OsBxGFRzVzq9rtdkiShLq6un6X19TUAABsNlsk\nmxU3Tp4Efv/7dFx+uQFvvZWGl146iKefPswAIaIzooqeiCzLEAQBFosFbrcb+/btQ3Z2dq91bDYb\nmpubMX/+/Og0MkYFAsCf/5yGysoMaLUBPP74YeTnc6wHEY0OVYRIU1MTJk+eDAAwGo1wOBx9QqSs\nrAx5eXlRaF3seu89DVatSsehQ+OwbFk7pk/v5FgPIhpVqggRn88HvV4ffOz1evus43a7IUkSZFlG\nUVFRr2Uulwsulyv4uLi4GKIojl2DVc7lSsJvf5sGlysJDz54EjfeeBTJyRoAmmg3Lao0Gk1Cvy9O\nxVqEsBa91dbWBv82mUwwmUyDrq+KEBmOnuBwOBxwOp0wm83BZf29UI/HE9H2qcH+/cl47DER77yT\nhoULvVi/3oezzxYTshb9EUXWogdrEcJahIiiiOLi4hFtE7EQsdlsEL45lhIIBCAIAoxGI8xmM3Q6\nXbD3cXqvpGdbURSRl5cHURThdrt7hUiiO3Cge6zH//yPFnPndo/1EEWO9SCisRexECksLBxw2dSp\nU9HS0gKz2QxFUWC1WgEAfr8fWq0WkyZNgsFgAAC0tbVhxowZEWmz2nk8AjZs0OP553W44QY/3nlH\nwdlnc6wHEUWOKi7xzcnJAQBIkgSdThc8qb5ixQoAQHZ2NpqammC325GZmdnnpHuiOXoU2LhRh0sv\nNeDzz5Pxxz8ewMMPtzNAiCjihEAgEJfHPVpbW6PdhFHXM9Zj9WoRF154AsuWtePCC08Mug2P94aw\nFiGsRQhrEZKVlTXibWLmxHoiCwSAXbvGoaJCxIQJXXj66cP40Y841oOIoo8honJ79miwcmUGOjoE\n3HdfOwoKONaDiNSDIaJSTmcKKioysHdvCpYs8eC66zqQpIozWEREIQwRldm3LxlVVSKamtJw550e\nbN7shyaxxwgSkYrxt61KKEoS7r13PK655mxMmnQCjY0KbruNAUJE6saeSJQdOSLgmWf0ePFFHYqL\n/di9+wDOOouX6hJRbGCIRElHB/DCCzo884weBQWd2LXrAM47j9OyE1FsYYhE2IkTQG2tFmvWiLjo\nomP4/e8P4nvfG3ysBxGRWjFEIiQQAHbuHIfKShEGQxc2bDiE//iP49FuFhHRGWGIREBjowbl5Rk4\nflzAww+34/LLOdaDiOIDQ2QMORypKC8XsX9/91iPoiKO9SCi+MIQGQN79yajqioD772nwa9/7cHs\n2X6kpka7VUREo4+/i0fRl18m4Z57xuNnPzsbJtNxNDYq+PnPGSBEFL8YIqPg8GEBK1eKKCw0YPz4\nLjQ0KFi40AutNr4mSJ49eza2bNky6DorV67E7NmzMXPmTHz++ed9li9duhSzZ89GSUkJ9u/f3+/z\nPdv1PHfJJZdgx44do/tiiGhU8HDWGejoELBpkw4bNuhw1VVHYbMpOPfc+Bwo+Nlnn0EQBJSUlAy4\njtPphNPpxMsvvwyn04l77rkHL7/8cnB5TU0NBEEILi8tLcWbb77Z5/n58+fj3nvvDT7X3t6OKVOm\n4Oqrr47ESyWiEWCIhOH4ceCVV7R4/HERF198DNu2fYVJk8ZuoGBpaSna29tx5MgRbNiwAeeff36v\n55YvX47LLrsMO3bswO7du9HY2Ih58+bh448/xp49e1BTU4NHH310WPsAunsAkiQBAFatWgWz2Yz1\n69fjo48+ws6dOzFz5sx+29nQ0IBp06YBAMxmMxwOR6/lDocDl19+eXC50+ns93lJkpCdnY077rgD\nAJCRkYEJEyaMclWJaDQwREagqwvYvn0cVq3KwHe+cxKbNh3CRReN7ViPZ555Bj/84Q9RVlYGp9OJ\nhoYGHDlyBNnZ2Vi+fHnwV7rL5QLQfXfId999Fzt27IDT6cSHH36IVatWDXsfPb2CnTt3or29HVdd\ndRXeffdd3HHHHfj8888xc+ZM7NixA3V1dRBOuU55wYIF+Prrr3HRRRcN+FqsVivq6uowc+ZMbN++\nfcDnBUHA+eefD6C7B1RWVoYFCxaMUYWJ6EyoKkRkWQ7eKvd0drsdOp0OsiyjqKgoou0KBIDdu9NQ\nXi5CEIABoGa9AAAL70lEQVSVKw9j2rTI3BTqww8/xMKFCwF0/0o3m81YunQprr32WgDdv9JP1dOb\nOPXvkezD4XBAkiSUlZUhEAhg/Pjxfdp09dVX93toqaGhodd5jvb29l7L58yZg88++wwlJSU4//zz\ng//uQM+vX78e27dvx2OPPQaTyTScchFRhKkmRCRJQnV1NZ588sk+y2RZhiAIsFgscLvd2LdvX8Tu\ns/7BB6koL8/Al18mY+nSdlx99dGIDhTMzs5GY2MjzGZz8FBVTk4Odu/ejUsvvRSfffbZqO7DarVi\nwoQJWL58OQDg2Wef7bO/np5ID0EQsGDBAlx22WVYuXIlysrKIElSr0ADgP379+Oyyy7DvffeC6fT\niSNHjgz4fENDAxoaGrBz586w6kZEkaGaELFYLMjMzOx3WVNTEyZPngwAMBqNcDgcYx4in36agspK\nEX/7mwaLFnlw001+pEShWsuXL0dpaSlmz56NI0eOoLq6Gt/5znf6PDda+5gzZ07weUEQcMstt/TZ\n30A9EaD7/2PPtpWVldi/fz+uuuoquFwuXHDBBXj00Uexfv16jB8/HlVVVQDQ7/NPPvkkHA4HLrnk\nkuC+33333XDLSERjRAgEAqq5DvXRRx/Ffffd1+f56upqzJgxA9nZ2ZAkCZIkDXqVEAC0traG3Y66\nunG4//7x+OUvvbj1Vh/S08PeVdSJogiPxxPtZqgCaxHCWoSwFiFZWVkj3kY1PRE1ufzyTjQ0KBg/\nXjX5SkSkShELEZvNFryaJxAIQBAEGI1GmM3mIbfV6/Xwer0AAJ/PB71e32u5y+UKXp0EAMXFxRBF\nMey2nsGmqqPRaM6oFvGEtQhhLUJYi95qa2uDf5tMpiEvaolYiBQWFg65zulH1vx+P7RaLaZMmYKW\nlhaYzWYoigKr1dprvf5eKLun3dhVD2EtQliLENYiRBRFFBcXj2gb1Ux7Yrfb0dLSgubm5uBzK1as\nAIDgZb+SJEGn00XsyiwiIhqcqk6sj6YzObEeT/grK4S1CGEtQliLkHBOrKumJ0JERLGHIUJERGFj\niBARUdgYIkREFDaGCBERhY0hQkREYWOIEBFR2BgiREQUNoYIERGFjSFCRERhY4gQEVHYGCJERBQ2\nhggREYWNIUJERGFjiBARUdgYIkREFDaGCBERhY0hQkREYWOIEBFR2FQVIrIsD7ispqYGAGCz2SLV\nHCIiGoJqQkSSJKxZs2bA5TabDXfeeScyMzMj2CoiIhpMSrQb0MNisQwaEGVlZcjLy4tgi4iIaCiq\n6YkMxe12Q5Ik1NXVRbspRET0jZgJkaKiIlgsFng8Hjidzmg3h4iIEMHDWTabDYIgAAACgQAEQYDR\naITZbB7WtqIoIi8vD6Iowu1299rO5XLB5XIFHxcXFyMrK2v0X0SMEkUx2k1QDdYihLUIYS1Camtr\ng3+bTCaYTKZB149YiBQWFg65TiAQ6PXY7/dDq9Vi0qRJMBgMAIC2tjbMmDGj13qnv9Da2loUFxeP\nQqtjH2sRwlqEsBYhrEVIOLVQzeEsu92OlpYWNDc3B59bsWIFACA7OxtNTU2w2+3IzMxEdnZ2lFpJ\nRESnUs3VWfn5+cjPz+/1XHl5efDv4fRkiIgoslTTExlNQx3DSySsRQhrEcJahLAWIeHUQgicfiKC\niIhomOKyJ0JERJHBEKG4ZLfbhzU4lYNXKZENNl/hcD9DMR8iQ73Q4RYiHgz1Wm02G2w2W3Ayy3gl\nyzIEQYDFYoFWq8W+ffv6XU+SJEiSFNnGRcFQ7wtZlmG322G32yPcssgb7vdFfX19hFsWeYPNVzjc\nzxAQ4yEy1AsdSSFi3VCvVZIkWK1WFBYWwu12x/Wo/6amJuh0OgCA0WiEw+GIcouiZzifgW3btiE/\nPx+KoiT0Z0SWZRiNRlgsFhgMhriuBTD4fIUj+QzFdIgM9UIT6ctkqNfaM/dYz3K32x3xNkaKz+eD\nXq8PPvZ6vX3WkWUZFoulzwDXeDPU+8Jut2PSpEkAuqcWiucxWMP5Pujppbvd7riuxVCG8xnqEdMh\nMtQLHUkhYt1Qr7WwsBAFBQUAur9AJ06cGNH2qY3P54t2EyJiqPfF3r174fV6Icty3B/yHaoWOTk5\nMBgMuO222zgNygjEdIjQyMmyjNzc3Lj+laXX64NfEKd/cQDdNeiZe61nPrdEptfrkZOTAwAJcV5k\nIH6/H3q9HrNmzcKGDRugKEq0mxQ1Q32GThXTITLUCx1JIWLdcF+rJEkoKSmJZNMibsqUKcHDdYqi\nwGq1Auj+kgC6D1U0NzfDZrPB4/HE9bHvod4XoijCaDQCALRaLVpaWiLexkgZqhY2mw3XXXcdioqK\nUFpamhCB2t98hcDAn6H+xHSIDPVlMZJCxLqhagF0f0iKiooAIK6vSur5VS1JEnQ6XbDX1TMXW35+\nfvAGZ6fWJx4N9b7Iz88PLvf7/XF9mHM4n5EeeXl5wfMn8Wqw+QoH+gz1J+ZHrNfX18NgMEBRlOAx\n/+XLlwfn3epvebwarBaSJGHt2rXQ6XTw+XxYtGjRsKbhp9g3nM+ITqdDS0tL3PdSh6pFXV0djEYj\nvF5v3H9fjJaYDxEiIoqemD6cRURE0cUQISKisDFEiIgobAwRIiIKG0OEiIjCxhAhIqKwMUQoLi1b\ntgw33XRTv//FKlmWsXz5cgBAdXV18LUsW7Zs0FH3p24HIKZrQOqTEu0GEI2FioqK4EjkBQsWYNGi\nRcjNzY1yq86M0WjEnDlzAHQPmnv11VcBALNmzYLBYBjWdkSjjSFCcUur1QLonh9Ir9dDq9VCURSs\nWbMGOTk5cDqdmD9/PjZs2ICKigoA3dM8/OEPf8D9998PAHA4HNi4cSN8Ph8sFgtKS0uD+z1VdXU1\n9uzZA0EQgvMv2e12OBwOeL1eNDc3Izc3F3fffXfwC3+wfdvtdtTU1MDn8yE/Px/z58+H2+3GSy+9\nFPw3586di82bN2Pbtm0wGo3Izs4edLuKigo88sgjwW3z8/ORmZkZnAqnpqYGgiDE/ah1Gl08nEUJ\nR5IkdHR0oLS0FEDfmXx7Hvt8PqxduxalpaV46qmnoNfr8eyzz/bZn91uh9PpxPr16/Hggw+ipqYm\n2Auqr6/HRRddhOeffx45OTnBO8kNtm+3240NGzZg8eLFqKiogNPpDM5vJAhCMPA2b97cqx2Kogy6\nHYBgOG7evBlWq7XXPTXsdjumTp0ablkpQTFEKCHdfvvtQ06Hv2fPHlitVpjNZmi1WpSUlAw6cWVb\nWxuys7Px/PPPB3sUubm5uPLKK6HVajF//nzIsgy/3z/ovpubmzF9+nRkZ2fDYDBg0aJFwZl2B2O3\n20e0XX5+fvDfdLvd8Pl8cX2LABobPJxFCWc4X8hA9xer3W7H3LlzAXQfFuvo6OizXn5+Pnw+H9as\nWQOfzxc8nAWgz7mKnsn/Btt3W1tbr9l0e77YZVketL3hbGe1WiFJEmRZRmFh4aD7J+oPQ4QS3qlz\nkJ56x0Oj0Yj8/Hzcfffdwef6u1GRoiiwWCwoKCiAoihYsWJF8CT+6bchVhQFBoNh0H3rdDq0tbUF\nn5dlGW63e8jwC2c7i8WCPXv2oKWlBWVlZYPun6g/PJxFCefU0NDpdJBlGYqiwOfz4fXXXw8umzp1\nKiRJgiRJ8Pl8qK6uxsaNG/vsz263Y82aNVAUBYFAAIIgBMNIlmXU19fD7/ejuroaubm50Gq1g+57\n+vTpqK+vD4ZATw9nKNOnT4fNZut3u4Em687Pz8eePXugKAoPZVFY2BOhuDfQiXOg+/BSYWEhFi5c\nGLwU1mazAei+umvRokXBW6VardZePYceRUVFcDgcWLhwIXQ6HQoLC5GXlwe73R48ed0TID3bD7Zv\ng8GAOXPm4OGHH4bf78f06dNRUFAw4GGpntdjMBhwyy239Lvdqa85Ly8PN910E1599VUYDAbo9XpM\nnjz5DCpMiYz3EyEaI3a7HXv27Ok3eNRk2bJlKCsrY0+EwsLDWUQJzOFw8FAWnREeziJKUHa7HdXV\n1Vi0aFG0m0IxjIeziIgobDycRUREYWOIEBFR2BgiREQUNoYIERGFjSFCRERh+//4Q5wf/qWQCQAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc369f0c950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.style.use('ggplot')\n",
    "slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(conf_mats[0,0,:], weights[0][0,0,:])\n",
    "print slope, intercept\n",
    "plt.scatter(conf_mats[0,0,:], weights[0][0,0,:], s=40)\n",
    "x = np.arange(0.05, 0.95, 0.01)\n",
    "plt.plot(x, slope*x + intercept, 'b-')\n",
    "plt.text(0.15, -0.95, \"corrcoef=%.3f\" % (np.corrcoef(conf_mats[0,0,:], weights[0][0,0,:])[0,1]))\n",
    "plt.xlabel(\"True specificity\")\n",
    "plt.ylabel(r\"$W^r_{0,0}$\")\n",
    "plt.savefig(\"specificities.png\")\n",
    "plt.savefig(\"specificities.eps\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make comparison for various annotators:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}