newmodel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dawnstear/desktop/tensorflow_update/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/dawnstear/desktop/tensorflow_update/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/dawnstear/desktop/tensorflow_update/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow import contrib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp \n",
    "\n",
    "from sklearn import cross_validation #, model_selection\n",
    "#from sklearn.utils import shuffle\n",
    "#import matplotlib as plt\n",
    "#from matplotlib import pyplot\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "logs_path = '/Users/dawnstear/desktop/tmp/tensorflow_logs/example_1/'\n",
    "data  =  pd.read_csv('/Users/dawnstear/desktop/chop_cellpred/data.csv')\n",
    "\n",
    "np.random.seed(42) # need to set multiple random seeds? # dont need to shuffle bc next_batch does already\n",
    "#data_shuffled = shuffle(data)\n",
    "data_shuffled = data\n",
    "\n",
    "# Extract data and labels from gene expresssion matrix\n",
    "X_ = data_shuffled.drop(['Labels','TYPE'],axis=1)\n",
    "X_ = X_.iloc[:,:50]\n",
    "y_ = data_shuffled['Labels']  # X_ and y_ are used b/c X and y are also standard name of placeholders in TF... \n",
    "\n",
    "cellcount, genecount = np.shape(X_)\n",
    "\n",
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X_.values,y_.values,test_size=0.2)\n",
    "                                                                # pass in X_.values so its a numpy.ndarray & not dframe\n",
    "def next_batch(num, data, labels):  # from stack overflow, \n",
    "    # Return a total of `num` random samples and labels. \n",
    "    idx = np.arange(0 , len(data))\n",
    "    np.random.shuffle(idx)\n",
    "    idx = idx[:num]\n",
    "    data_shuffle = [data[ i] for i in idx]\n",
    "    labels_shuffle = [labels[ i] for i in idx]\n",
    "    return np.asarray(data_shuffle), np.asarray(labels_shuffle)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# *-------Define architecture parameters--------*\n",
    "num_input = genecount # *must preserve gene order when inputing data, take np.shape(X_) so no need for genecount -1 \n",
    "n_hidden_1 = 64   \n",
    "n_hidden_2 = 32\n",
    "n_hidden_3 = 16\n",
    "n_hidden_4 = 256\n",
    "n_hidden_5 = 128\n",
    "num_classes = 10 # == len(celldistro) # specify how many cell types ( # of labels in dataset)\n",
    "\n",
    "# Create placeholders for train X,y and test X,y. Shape = (None, n_inputs)...\n",
    "# bc we dont know how many samples we will have per batch yet\n",
    "X = tf.placeholder(tf.float32, [None,num_input],name=\"X_data\") # use -1 or None for dynamic batch size\n",
    "y = tf.placeholder(tf.int64, [num_classes], name=\"y_labels\")\n",
    "\n",
    "# Build Architecture ''' Set weights to 0 option/reinit them '''\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    #with \n",
    "    #tf.reset_default_graph() #if condition, set to false\n",
    "    layer1 = tf.layers.dense(inputs=X, units=n_hidden_1, activation=tf.nn.elu, name='Layer_1',reuse=tf.AUTO_REUSE) \n",
    "    dropout1 = tf.layers.dropout(inputs=layer1, rate=0.3,name='Dropout_1')\n",
    "    layer2 = tf.layers.dense(inputs=dropout1,units=n_hidden_2, activation=tf.nn.elu,name='Layer_2',reuse=tf.AUTO_REUSE)#,reuse=True)\n",
    "    dropout2 = tf.layers.dropout(inputs=layer2, rate=0.3,name='Dropout_2')\n",
    "    layer3 = tf.layers.dense(inputs=dropout2, units=n_hidden_3, activation=tf.nn.elu,name='Layer_3',reuse=tf.AUTO_REUSE)#,reuse=True)\n",
    "    output = tf.layers.dense(inputs=layer3,units=num_classes,activation=tf.nn.softmax,name='Output',reuse=tf.AUTO_REUSE)#,reuse=True)\n",
    "    #output_tensor = tf.Print(output,[output]) # tf.Print it !!!!!!\n",
    "    \n",
    "######### Cost fn = softmax xentropy\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,logits=output) # check sparse_xentropy docs\n",
    "    loss = tf.reduce_mean(xentropy,name=\"loss_\")          \n",
    "######### Optimizer = SGD    \n",
    "lr=0.0005\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(lr)\n",
    "    training_op = optimizer.minimize(loss)  # pass in only variables you want to change !! *******\n",
    "    \n",
    "### Now how to evaluate...\n",
    "with tf.name_scope('eval'):\n",
    "    correct = tf.nn.in_top_k(output,y,1)  # check top_k docs\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct,tf.float32)) # compute roc metrics here?\n",
    "# *------------End of Construction phase ----------*   \n",
    "\n",
    "#*------------Begin Execution Stage----------------*\n",
    "#saver = tf.train.Saver()\n",
    "\n",
    "# *--------Set Some hyperparameters & initialize Metic Vectors --------*\n",
    "n_epochs = 12\n",
    "batch_size = 10\n",
    "epochvec = range(1,n_epochs+1)\n",
    "total_batch = (cellcount//batch_size)\n",
    "display_epoch = 1\n",
    "accTrain = []\n",
    "accVal = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 Loss= 2.430806626\n",
      "Epoch: 0002 Loss= 2.434556658\n",
      "Epoch: 0003 Loss= 2.377173179\n",
      "Epoch: 0004 Loss= 2.364881500\n",
      "Epoch: 0005 Loss= 2.369656815\n",
      "Epoch: 0006 Loss= 2.361259474\n",
      "Epoch: 0007 Loss= 2.338480397\n",
      "Epoch: 0008 Loss= 2.360881072\n",
      "Epoch: 0009 Loss= 2.343766300\n",
      "Epoch: 0010 Loss= 2.356025803\n",
      "Epoch: 0011 Loss= 2.338512150\n",
      "Epoch: 0012 Loss= 2.355979108\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'str' object has no attribute 'SerializeToString'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-caee4a5a582d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'%04d'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Loss=\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"{:.9f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mavg_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m     \u001b[0mlayer1_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer2_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer3_weights\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Layer_1/kernel:0'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Layer_2/kernel:0'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Layer_3/kernel:0'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m     \u001b[0;31m#layer1_weights = sess.run('Layer_1/kernel:0') # save weight/biases for later inspection (must do sess.run() to get them)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;31m#layer2_weights = sess.run('Layer_2/kernel:0')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/desktop/tensorflow_update/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \"\"\"\n\u001b[1;32m    894\u001b[0m     options_ptr = tf_session.TF_NewBufferFromString(\n\u001b[0;32m--> 895\u001b[0;31m         compat.as_bytes(options.SerializeToString())) if options else None\n\u001b[0m\u001b[1;32m    896\u001b[0m     \u001b[0mrun_metadata_ptr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_NewBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute 'SerializeToString'"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer() # always do this right before you run sess or make sure all variables have been declared \n",
    "# clear out folder, or create new folder name with date/time/params eeach time, histogram of weights ????\n",
    "\n",
    "tf.summary.scalar(\"Loss\", loss)            # Create a summary to monitor cost (loss) tensor for tensorboard\n",
    "tf.summary.scalar(\"Accuracy\", accuracy)    # Create a summary to monitor accuracy tensor for tensorboard\n",
    "merged_summary_op = tf.summary.merge_all() # Merge all summaries into a single op\n",
    "\n",
    "# *--------------- Begin Run Session -------------------*\n",
    "                              # ///config=tf.ConfigProto(log_device_placement=True)\n",
    "with tf.Session() as sess:    # log_device checks if gpu or cpu is being used...print(sess.run(training_op,feed_dict={X:X_batch, y:y_batch}))\n",
    "    tf.set_random_seed(1234)\n",
    "    sess.run(init)\n",
    "    \n",
    "    summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph()) # op to write logs to Tensorboard\n",
    "    \n",
    "    for epoch in range(n_epochs):\n",
    "        #print('Starting Epoch %d ' % epoch)\n",
    "        avg_loss = 0\n",
    "        for iteration in range(cellcount//batch_size):\n",
    "            \n",
    "            X_batch, y_batch = next_batch(batch_size, X_train, y_train)\n",
    "            _, c, summary = sess.run([training_op, loss, merged_summary_op],feed_dict={X:X_batch, y:y_batch}) # c for cost\n",
    "            summary_writer.add_summary(summary, epoch * total_batch + iteration) # Write logs at every iteration\n",
    "            avg_loss += c / total_batch # Compute average loss\n",
    "                                                                    # Add these for Validation accuracy:\n",
    "        acc_train = accuracy.eval(feed_dict={X:X_batch, y:y_batch}) # acc_val = accuracy.eval(feed_dict={X:X_test, y:y_test})\n",
    "        accTrain = np.append(accTrain,acc_train)                    # accVal = np.append(accVal,acc_val) \n",
    "                                                                    # print('Epoch:%d - Train Acc :%f - Validation Acc: %f' % (epoch, acc_train,acc_val))\n",
    "        # Display logs per epoch step\n",
    "        if (epoch+1) % display_epoch == 0:\n",
    "            print(\"Epoch:\", '%04d' % (epoch+1), \"Loss=\", \"{:.9f}\".format(avg_loss))\n",
    "\n",
    "    layer1_weights = sess.run('Layer_1/kernel:0') # save weight/biases for later inspection (must do sess.run() to get them)\n",
    "    layer2_weights = sess.run('Layer_2/kernel:0') \n",
    "    layer3_weights = sess.run('Layer_3/kernel:0') # get output weights? \n",
    "    #layer1_bias = sess.run('Layer_1/bias:0')\n",
    "         \n",
    "\n",
    "# *--------------- Plot Results -------------------*   \n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(epochvec,accTrain)\n",
    "#ax.plot(epochvec,accVal)\n",
    "ax.set(xlabel='Epoch', ylabel='Accuracies',title='Accuracy Over Training Phase') # include time//epoch//batch size in title\n",
    "ax.grid()\n",
    "#fig.savefig(\"/Users/dawnstear/desktop/tmp/\")\n",
    "\n",
    "# TENSORBOARD LINUX COMMAND (for macOS/safari, add the local host part): \n",
    "# tensorboard --logdir=\"/Users/dawnstear/desktop/tmp/tensorflow_logs/example_1/\" --host localhost --port 8088\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Variable 'Layer_1/kernel:0' shape=(50, 64) dtype=float32_ref>, <tf.Variable 'Layer_1/bias:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'Layer_2/kernel:0' shape=(64, 32) dtype=float32_ref>, <tf.Variable 'Layer_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'Layer_3/kernel:0' shape=(32, 16) dtype=float32_ref>, <tf.Variable 'Layer_3/bias:0' shape=(16,) dtype=float32_ref>, <tf.Variable 'Output/kernel:0' shape=(16, 10) dtype=float32_ref>, <tf.Variable 'Output/bias:0' shape=(10,) dtype=float32_ref>]\n",
      "\n",
      "\n",
      "Tensor(\"report_uninitialized_variables/boolean_mask/GatherV2:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "(50, 64)\n",
      "-0.00011928856\n"
     ]
    }
   ],
   "source": [
    "# check weights\n",
    "print(tf.trainable_variables())\n",
    "print(\"\\n\")\n",
    "print(tf.report_uninitialized_variables())\n",
    "print(np.shape(layer1_weights))\n",
    "print(np.mean(layer1_weights)) # check reproducibility, use scatter or histo in tb "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.18922128  0.18344417 -0.12067178 ...  0.02953382  0.11341352\n",
      "   0.17954761]\n",
      " [-0.21845238  0.00102474 -0.01227096 ... -0.05355166 -0.06845389\n",
      "   0.00173333]\n",
      " [-0.07991266 -0.11191159 -0.04416826 ...  0.04888756  0.11220614\n",
      "  -0.04430334]\n",
      " ...\n",
      " [-0.21056946 -0.01913771 -0.14971414 ... -0.0214503  -0.21465547\n",
      "  -0.19767147]\n",
      " [-0.0603504  -0.02652836  0.1475838  ...  0.12779285  0.17272927\n",
      "   0.07958131]\n",
      " [-0.22618413 -0.11698591 -0.22707945 ... -0.21974349 -0.14670625\n",
      "   0.09180107]]\n",
      "<class 'numpy.ndarray'>\n",
      "0.22929198 -0.2293377\n"
     ]
    }
   ],
   "source": [
    "print(layer1_weights)\n",
    "# To analyze weights from trial to trial, make histogram, #plt.hist(x,bins)\n",
    "# Get max, min, median to create bins for histogram\n",
    "print(type(layer1_weights))\n",
    "max_1 = np.amax(layer1_weights) \n",
    "min_1 = np.amin(layer1_weights) \n",
    "mean_1 = np.median(layer1_weights)\n",
    "print(max_1,min_1)\n",
    "rank_1 = np.sort(layer1_weights) # no need  to sort for spearman rank coeff.\n",
    "sp.stats.spearmanr()\n",
    "\n",
    "# Every one of the 50 rows represents a gene and each column is a node, 1:64, if we want to silence a\n",
    "# genes input then we need to set its whole row to zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.22592145 -0.22432405 -0.21538211 ...  0.2137316   0.22062325\n",
      "   0.22267553]\n",
      " [-0.2224403  -0.21845238 -0.21715821 ...  0.2232803   0.22488384\n",
      "   0.22732513]\n",
      " [-0.22487088 -0.21608467 -0.21276365 ...  0.2196325   0.22100468\n",
      "   0.22236815]\n",
      " ...\n",
      " [-0.22444175 -0.22347625 -0.21990612 ...  0.20545705  0.20990367\n",
      "   0.22656004]\n",
      " [-0.21821412 -0.21415478 -0.20512629 ...  0.22503082  0.22690813\n",
      "   0.22698866]\n",
      " [-0.22707945 -0.22618413 -0.21974349 ...  0.2006243   0.2224522\n",
      "   0.22537404]]\n"
     ]
    }
   ],
   "source": [
    "#layer1.name\n",
    "#sess.run('dnn_4/Layer_1/Elu:0')\n",
    "#tf.boolean_mask\n",
    "print(rank_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#rand shuffle seed needed \n",
    "#scatter plot for weights\n",
    "#how does perturbation effect metabolism\n",
    "# trey atticker paper\n",
    "# mitochondria\n",
    "# spearman correlation for correlation of ranking weights of a layer, just compare flattened np.arrays? closer to -1 or 1 means high pos or neg correlation\n",
    "\n",
    "'''\n",
    "  TODO:\n",
    "      \n",
    " -check that labels were applied correctly    \n",
    " -next batch method, ok, use tf version?\n",
    " -use coo sparse dframe/matrix\n",
    " -softmax logits needs labels to be ints 0-9 so change labels to 0 through 9. DONE  \n",
    " -change shuffle method    \n",
    " -save trainedmodelwith date/time/params title \n",
    " -implement Tensorboard DONE\n",
    " -Implement multithreading/queueing for cluster\n",
    " - implement pie chart of cell types and Venn diagram\n",
    " - plot accuracy as function of n epochs or n genes\n",
    " -kfold cross validation'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}