0.expression-download/2C.process-gtex-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process GTEx Gene Expression Data\n",
    "\n",
    "Retrieve the downloaded expression data, update gene identifiers to entrez, and curate sample IDs. The script will also identify a balanced hold-out test set to compare projection performance into learned latent spaces across algorithms.\n",
    "\n",
    "**Note:** GTEx version 7 was downloaded from https://www.gtexportal.org/home/datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Phenotype Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15598, 63)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAMPID</th>\n",
       "      <th>SMATSSCR</th>\n",
       "      <th>SMCENTER</th>\n",
       "      <th>SMPTHNTS</th>\n",
       "      <th>SMRIN</th>\n",
       "      <th>SMTS</th>\n",
       "      <th>SMTSD</th>\n",
       "      <th>SMUBRID</th>\n",
       "      <th>SMTSISCH</th>\n",
       "      <th>SMTSPAX</th>\n",
       "      <th>...</th>\n",
       "      <th>SME1ANTI</th>\n",
       "      <th>SMSPLTRD</th>\n",
       "      <th>SMBSMMRT</th>\n",
       "      <th>SME1SNSE</th>\n",
       "      <th>SME1PCTS</th>\n",
       "      <th>SMRRNART</th>\n",
       "      <th>SME1MPRT</th>\n",
       "      <th>SMNUM5CD</th>\n",
       "      <th>SMDPMPRT</th>\n",
       "      <th>SME2PCTS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEX-1117F-0003-SM-58Q7G</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Blood</td>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>0013756</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEX-1117F-0003-SM-5DWSB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Blood</td>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>0013756</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEX-1117F-0003-SM-6WBT7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Blood</td>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>0013756</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     SAMPID  SMATSSCR SMCENTER SMPTHNTS  SMRIN   SMTS  \\\n",
       "0  GTEX-1117F-0003-SM-58Q7G       NaN       B1      NaN    NaN  Blood   \n",
       "1  GTEX-1117F-0003-SM-5DWSB       NaN       B1      NaN    NaN  Blood   \n",
       "2  GTEX-1117F-0003-SM-6WBT7       NaN       B1      NaN    NaN  Blood   \n",
       "\n",
       "         SMTSD  SMUBRID  SMTSISCH  SMTSPAX    ...    SME1ANTI SMSPLTRD  \\\n",
       "0  Whole Blood  0013756    1188.0      NaN    ...         NaN      NaN   \n",
       "1  Whole Blood  0013756    1188.0      NaN    ...         NaN      NaN   \n",
       "2  Whole Blood  0013756    1188.0      NaN    ...         NaN      NaN   \n",
       "\n",
       "  SMBSMMRT SME1SNSE SME1PCTS SMRRNART SME1MPRT SMNUM5CD  SMDPMPRT  SME2PCTS  \n",
       "0      NaN      NaN      NaN      NaN      NaN      NaN       NaN       NaN  \n",
       "1      NaN      NaN      NaN      NaN      NaN      NaN       NaN       NaN  \n",
       "2      NaN      NaN      NaN      NaN      NaN      NaN       NaN       NaN  \n",
       "\n",
       "[3 rows x 63 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'GTEx_v7_Annotations_SampleAttributesDS.txt')\n",
    "pheno_df = pd.read_table(path)\n",
    "\n",
    "print(pheno_df.shape)\n",
    "pheno_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Entrez ID Curation Information\n",
    "\n",
    "Load curated gene names from versioned resource. See https://github.com/cognoma/genes for more details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commit from https://github.com/cognoma/genes\n",
    "genes_commit = 'ad9631bb4e77e2cdc5413b0d77cb8f7e93fc5bee'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20395, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>symbol</th>\n",
       "      <th>description</th>\n",
       "      <th>chromosome</th>\n",
       "      <th>gene_type</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>aliases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>alpha-1-B glycoprotein</td>\n",
       "      <td>19</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A1B|ABG|GAB|HYST2477</td>\n",
       "      <td>alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A2M</td>\n",
       "      <td>alpha-2-macroglobulin</td>\n",
       "      <td>12</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A2MD|CPAMD5|FWP007|S863-7</td>\n",
       "      <td>alpha-2-macroglobulin|C3 and PZP-like alpha-2-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_gene_id symbol             description chromosome       gene_type  \\\n",
       "0               1   A1BG  alpha-1-B glycoprotein         19  protein-coding   \n",
       "1               2    A2M   alpha-2-macroglobulin         12  protein-coding   \n",
       "\n",
       "                    synonyms  \\\n",
       "0       A1B|ABG|GAB|HYST2477   \n",
       "1  A2MD|CPAMD5|FWP007|S863-7   \n",
       "\n",
       "                                             aliases  \n",
       "0  alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...  \n",
       "1  alpha-2-macroglobulin|C3 and PZP-like alpha-2-...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/genes.tsv'.format(genes_commit)\n",
    "gene_df = pd.read_table(url)\n",
    "\n",
    "# Only consider protein-coding genes\n",
    "gene_df = (\n",
    "    gene_df.query(\"gene_type == 'protein-coding'\")\n",
    ")\n",
    "\n",
    "print(gene_df.shape)\n",
    "gene_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load gene updater - old to new Entrez gene identifiers\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/updater.tsv'.format(genes_commit)\n",
    "updater_df = pd.read_table(url)\n",
    "old_to_new_entrez = dict(zip(updater_df.old_entrez_gene_id,\n",
    "                             updater_df.new_entrez_gene_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Gene Expression Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(56202, 11689)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>GTEX-1117F-0226-SM-5GZZ7</th>\n",
       "      <th>GTEX-111CU-1826-SM-5GZYN</th>\n",
       "      <th>GTEX-111FC-0226-SM-5N9B8</th>\n",
       "      <th>GTEX-111VG-2326-SM-5N9BK</th>\n",
       "      <th>GTEX-111YS-2426-SM-5GZZQ</th>\n",
       "      <th>GTEX-1122O-2026-SM-5NQ91</th>\n",
       "      <th>GTEX-1128S-2126-SM-5H12U</th>\n",
       "      <th>GTEX-113IC-0226-SM-5HL5C</th>\n",
       "      <th>GTEX-117YX-2226-SM-5EGJJ</th>\n",
       "      <th>...</th>\n",
       "      <th>GTEX-ZVE2-0006-SM-51MRW</th>\n",
       "      <th>GTEX-ZVP2-0005-SM-51MRK</th>\n",
       "      <th>GTEX-ZVT2-0005-SM-57WBW</th>\n",
       "      <th>GTEX-ZVT3-0006-SM-51MT9</th>\n",
       "      <th>GTEX-ZVT4-0006-SM-57WB8</th>\n",
       "      <th>GTEX-ZVTK-0006-SM-57WBK</th>\n",
       "      <th>GTEX-ZVZP-0006-SM-51MSW</th>\n",
       "      <th>GTEX-ZVZQ-0006-SM-51MR8</th>\n",
       "      <th>GTEX-ZXES-0005-SM-57WCB</th>\n",
       "      <th>GTEX-ZXG5-0005-SM-57WCN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Description</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DDX11L1</th>\n",
       "      <td>ENSG00000223972.4</td>\n",
       "      <td>0.1082</td>\n",
       "      <td>0.1158</td>\n",
       "      <td>0.02104</td>\n",
       "      <td>0.02329</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.04641</td>\n",
       "      <td>0.03076</td>\n",
       "      <td>0.09358</td>\n",
       "      <td>0.121</td>\n",
       "      <td>...</td>\n",
       "      <td>0.09012</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.1045</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.6603</td>\n",
       "      <td>0.695</td>\n",
       "      <td>0.1213</td>\n",
       "      <td>0.4169</td>\n",
       "      <td>0.2355</td>\n",
       "      <td>0.145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WASH7P</th>\n",
       "      <td>ENSG00000227232.4</td>\n",
       "      <td>21.4000</td>\n",
       "      <td>11.0300</td>\n",
       "      <td>16.75000</td>\n",
       "      <td>8.17200</td>\n",
       "      <td>7.658</td>\n",
       "      <td>9.37200</td>\n",
       "      <td>10.08000</td>\n",
       "      <td>13.56000</td>\n",
       "      <td>9.889</td>\n",
       "      <td>...</td>\n",
       "      <td>3.92600</td>\n",
       "      <td>13.1300</td>\n",
       "      <td>5.5370</td>\n",
       "      <td>5.789</td>\n",
       "      <td>8.4390</td>\n",
       "      <td>7.843</td>\n",
       "      <td>12.3900</td>\n",
       "      <td>12.5300</td>\n",
       "      <td>8.0270</td>\n",
       "      <td>12.760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 11689 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Name  GTEX-1117F-0226-SM-5GZZ7  \\\n",
       "Description                                                \n",
       "DDX11L1      ENSG00000223972.4                    0.1082   \n",
       "WASH7P       ENSG00000227232.4                   21.4000   \n",
       "\n",
       "             GTEX-111CU-1826-SM-5GZYN  GTEX-111FC-0226-SM-5N9B8  \\\n",
       "Description                                                       \n",
       "DDX11L1                        0.1158                   0.02104   \n",
       "WASH7P                        11.0300                  16.75000   \n",
       "\n",
       "             GTEX-111VG-2326-SM-5N9BK  GTEX-111YS-2426-SM-5GZZQ  \\\n",
       "Description                                                       \n",
       "DDX11L1                       0.02329                     0.000   \n",
       "WASH7P                        8.17200                     7.658   \n",
       "\n",
       "             GTEX-1122O-2026-SM-5NQ91  GTEX-1128S-2126-SM-5H12U  \\\n",
       "Description                                                       \n",
       "DDX11L1                       0.04641                   0.03076   \n",
       "WASH7P                        9.37200                  10.08000   \n",
       "\n",
       "             GTEX-113IC-0226-SM-5HL5C  GTEX-117YX-2226-SM-5EGJJ  \\\n",
       "Description                                                       \n",
       "DDX11L1                       0.09358                     0.121   \n",
       "WASH7P                       13.56000                     9.889   \n",
       "\n",
       "                      ...             GTEX-ZVE2-0006-SM-51MRW  \\\n",
       "Description           ...                                       \n",
       "DDX11L1               ...                             0.09012   \n",
       "WASH7P                ...                             3.92600   \n",
       "\n",
       "             GTEX-ZVP2-0005-SM-51MRK  GTEX-ZVT2-0005-SM-57WBW  \\\n",
       "Description                                                     \n",
       "DDX11L1                       0.1462                   0.1045   \n",
       "WASH7P                       13.1300                   5.5370   \n",
       "\n",
       "             GTEX-ZVT3-0006-SM-51MT9  GTEX-ZVT4-0006-SM-57WB8  \\\n",
       "Description                                                     \n",
       "DDX11L1                        0.000                   0.6603   \n",
       "WASH7P                         5.789                   8.4390   \n",
       "\n",
       "             GTEX-ZVTK-0006-SM-57WBK  GTEX-ZVZP-0006-SM-51MSW  \\\n",
       "Description                                                     \n",
       "DDX11L1                        0.695                   0.1213   \n",
       "WASH7P                         7.843                  12.3900   \n",
       "\n",
       "             GTEX-ZVZQ-0006-SM-51MR8  GTEX-ZXES-0005-SM-57WCB  \\\n",
       "Description                                                     \n",
       "DDX11L1                       0.4169                   0.2355   \n",
       "WASH7P                       12.5300                   8.0270   \n",
       "\n",
       "             GTEX-ZXG5-0005-SM-57WCN  \n",
       "Description                           \n",
       "DDX11L1                        0.145  \n",
       "WASH7P                        12.760  \n",
       "\n",
       "[2 rows x 11689 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.join('download', 'GTEx_Analysis_2016-01-15_v7_RNASeQCv1.1.8_gene_tpm.gct.gz')\n",
    "expr_df = pd.read_table(file, sep='\\t', skiprows=2, index_col=1)\n",
    "\n",
    "print(expr_df.shape)\n",
    "\n",
    "expr_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get GTEx gene mapping\n",
    "expr_gene_ids = (\n",
    "    expr_df\n",
    "    .loc[:, ['Name']]\n",
    "    .reset_index()\n",
    "    .drop_duplicates(subset='Description')\n",
    ")\n",
    "\n",
    "# Inner merge gene df to get ensembl to entrez mapping\n",
    "map_df = expr_gene_ids.merge(gene_df, how='inner', left_on='Description', right_on='symbol')\n",
    "\n",
    "symbol_to_entrez = dict(zip(map_df.symbol, map_df.entrez_gene_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process gene expression matrix\n",
    "\n",
    "This involves updating Entrez gene ids, sorting and subsetting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11688, 18356)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>symbol</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>101180976</th>\n",
       "      <th>101241878</th>\n",
       "      <th>101362076</th>\n",
       "      <th>101928601</th>\n",
       "      <th>102723547</th>\n",
       "      <th>102724231</th>\n",
       "      <th>102724473</th>\n",
       "      <th>102724928</th>\n",
       "      <th>105375355</th>\n",
       "      <th>105378803</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GTEX-1117F-0226-SM-5GZZ7</th>\n",
       "      <td>4.9610</td>\n",
       "      <td>356.1</td>\n",
       "      <td>2.0350</td>\n",
       "      <td>0.2791</td>\n",
       "      <td>48.12</td>\n",
       "      <td>1.4010</td>\n",
       "      <td>174.40</td>\n",
       "      <td>0.4212</td>\n",
       "      <td>71.11</td>\n",
       "      <td>7.744</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.6460</td>\n",
       "      <td>0.1025</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.521</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.5300</td>\n",
       "      <td>0.03187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTEX-1117F-0426-SM-5EGHI</th>\n",
       "      <td>0.6198</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0.2491</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5.08</td>\n",
       "      <td>0.1115</td>\n",
       "      <td>78.17</td>\n",
       "      <td>0.1274</td>\n",
       "      <td>53.43</td>\n",
       "      <td>6.029</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4977</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.525</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8182</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 18356 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "symbol                    1          2          9          10         \\\n",
       "sample_id                                                              \n",
       "GTEX-1117F-0226-SM-5GZZ7     4.9610      356.1     2.0350     0.2791   \n",
       "GTEX-1117F-0426-SM-5EGHI     0.6198      110.0     0.2491     0.0000   \n",
       "\n",
       "symbol                    12         13         14         15         \\\n",
       "sample_id                                                              \n",
       "GTEX-1117F-0226-SM-5GZZ7      48.12     1.4010     174.40     0.4212   \n",
       "GTEX-1117F-0426-SM-5EGHI       5.08     0.1115      78.17     0.1274   \n",
       "\n",
       "symbol                    16         18           ...      101180976  \\\n",
       "sample_id                                         ...                  \n",
       "GTEX-1117F-0226-SM-5GZZ7      71.11      7.744    ...            0.0   \n",
       "GTEX-1117F-0426-SM-5EGHI      53.43      6.029    ...            0.0   \n",
       "\n",
       "symbol                    101241878  101362076  101928601  102723547  \\\n",
       "sample_id                                                              \n",
       "GTEX-1117F-0226-SM-5GZZ7        0.0     1.6460     0.1025        0.0   \n",
       "GTEX-1117F-0426-SM-5EGHI        0.0     0.4977     0.0000        0.0   \n",
       "\n",
       "symbol                    102724231  102724473  102724928  105375355  \\\n",
       "sample_id                                                              \n",
       "GTEX-1117F-0226-SM-5GZZ7      2.521        0.0        0.0    10.5300   \n",
       "GTEX-1117F-0426-SM-5EGHI      1.525        0.0        0.0     0.8182   \n",
       "\n",
       "symbol                    105378803  \n",
       "sample_id                            \n",
       "GTEX-1117F-0226-SM-5GZZ7    0.03187  \n",
       "GTEX-1117F-0426-SM-5EGHI    0.00000  \n",
       "\n",
       "[2 rows x 18356 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expr_df = (expr_df\n",
    " .drop(['Name'], axis='columns')\n",
    " .dropna(axis='rows')\n",
    " .groupby(level=0).mean()\n",
    " .reindex(map_df.symbol)\n",
    " .rename(index=symbol_to_entrez)\n",
    " .rename(index=old_to_new_entrez)\n",
    " .transpose()\n",
    " .sort_index(axis='rows')\n",
    " .sort_index(axis='columns')\n",
    ")\n",
    "\n",
    "expr_df.index.rename('sample_id', inplace=True)\n",
    "\n",
    "print(expr_df.shape)\n",
    "expr_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratify Balanced Training and Testing Sets in GTEx Gene Expression\n",
    "\n",
    "Output training and testing gene expression datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "strat = pheno_df.set_index('SAMPID').reindex(expr_df.index).SMTSD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tissuetype</th>\n",
       "      <th>n =</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Muscle - Skeletal</td>\n",
       "      <td>564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Skin - Sun Exposed (Lower leg)</td>\n",
       "      <td>473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thyroid</td>\n",
       "      <td>446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Artery - Tibial</td>\n",
       "      <td>441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lung</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Nerve - Tibial</td>\n",
       "      <td>414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Whole Blood</td>\n",
       "      <td>407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Esophagus - Mucosa</td>\n",
       "      <td>407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Skin - Not Sun Exposed (Suprapubic)</td>\n",
       "      <td>387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Esophagus - Muscularis</td>\n",
       "      <td>370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Adipose - Visceral (Omentum)</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Cells - Transformed fibroblasts</td>\n",
       "      <td>343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Heart - Left Ventricle</td>\n",
       "      <td>303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Artery - Aorta</td>\n",
       "      <td>299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Heart - Atrial Appendage</td>\n",
       "      <td>297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Breast - Mammary Tissue</td>\n",
       "      <td>290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Colon - Transverse</td>\n",
       "      <td>274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Stomach</td>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Testis</td>\n",
       "      <td>259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Pancreas</td>\n",
       "      <td>248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Esophagus - Gastroesophageal Junction</td>\n",
       "      <td>244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Colon - Sigmoid</td>\n",
       "      <td>233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Adrenal Gland</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Pituitary</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Liver</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Brain - Cerebellum</td>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Artery - Coronary</td>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Spleen</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Brain - Caudate (basal ganglia)</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Brain - Cortex</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Prostate</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Brain - Nucleus accumbens (basal ganglia)</td>\n",
       "      <td>147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Small Intestine - Terminal Ileum</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Brain - Cerebellar Hemisphere</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Ovary</td>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Cells - EBV-transformed lymphocytes</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Brain - Frontal Cortex (BA9)</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Brain - Putamen (basal ganglia)</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Brain - Hippocampus</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Brain - Hypothalamus</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Brain - Anterior cingulate cortex (BA24)</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Vagina</td>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Uterus</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Brain - Amygdala</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Minor Salivary Gland</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Brain - Spinal cord (cervical c-1)</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Brain - Substantia nigra</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Kidney - Cortex</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Bladder</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Fallopian Tube</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Cervix - Ectocervix</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Cervix - Endocervix</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   tissuetype  n =\n",
       "0                           Muscle - Skeletal  564\n",
       "1              Skin - Sun Exposed (Lower leg)  473\n",
       "2                                     Thyroid  446\n",
       "3                      Adipose - Subcutaneous  442\n",
       "4                             Artery - Tibial  441\n",
       "5                                        Lung  427\n",
       "6                              Nerve - Tibial  414\n",
       "7                                 Whole Blood  407\n",
       "8                          Esophagus - Mucosa  407\n",
       "9         Skin - Not Sun Exposed (Suprapubic)  387\n",
       "10                     Esophagus - Muscularis  370\n",
       "11               Adipose - Visceral (Omentum)  355\n",
       "12            Cells - Transformed fibroblasts  343\n",
       "13                     Heart - Left Ventricle  303\n",
       "14                             Artery - Aorta  299\n",
       "15                   Heart - Atrial Appendage  297\n",
       "16                    Breast - Mammary Tissue  290\n",
       "17                         Colon - Transverse  274\n",
       "18                                    Stomach  262\n",
       "19                                     Testis  259\n",
       "20                                   Pancreas  248\n",
       "21      Esophagus - Gastroesophageal Junction  244\n",
       "22                            Colon - Sigmoid  233\n",
       "23                              Adrenal Gland  190\n",
       "24                                  Pituitary  183\n",
       "25                                      Liver  175\n",
       "26                         Brain - Cerebellum  173\n",
       "27                          Artery - Coronary  173\n",
       "28                                     Spleen  162\n",
       "29            Brain - Caudate (basal ganglia)  160\n",
       "30                             Brain - Cortex  158\n",
       "31                                   Prostate  152\n",
       "32  Brain - Nucleus accumbens (basal ganglia)  147\n",
       "33           Small Intestine - Terminal Ileum  137\n",
       "34              Brain - Cerebellar Hemisphere  136\n",
       "35                                      Ovary  133\n",
       "36        Cells - EBV-transformed lymphocytes  130\n",
       "37               Brain - Frontal Cortex (BA9)  129\n",
       "38            Brain - Putamen (basal ganglia)  124\n",
       "39                        Brain - Hippocampus  123\n",
       "40                       Brain - Hypothalamus  121\n",
       "41   Brain - Anterior cingulate cortex (BA24)  121\n",
       "42                                     Vagina  115\n",
       "43                                     Uterus  111\n",
       "44                           Brain - Amygdala  100\n",
       "45                       Minor Salivary Gland   97\n",
       "46         Brain - Spinal cord (cervical c-1)   91\n",
       "47                   Brain - Substantia nigra   88\n",
       "48                            Kidney - Cortex   45\n",
       "49                                    Bladder   11\n",
       "50                             Fallopian Tube    7\n",
       "51                        Cervix - Ectocervix    6\n",
       "52                        Cervix - Endocervix    5"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissuetype_count_df = (\n",
    "    pd.DataFrame(strat.value_counts())\n",
    "    .reset_index()\n",
    "    .rename({'index': 'tissuetype', 'SMTSD': 'n ='}, axis='columns')\n",
    ")\n",
    "\n",
    "file = os.path.join('data', 'gtex_sample_counts.tsv')\n",
    "tissuetype_count_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "tissuetype_count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, test_df = train_test_split(expr_df,\n",
    "                                     test_size=0.1,\n",
    "                                     random_state=123,\n",
    "                                     stratify=strat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10519, 18356)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1169, 18356)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_df.shape)\n",
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = os.path.join('data', 'train_gtex_expression_matrix_processed.tsv.gz')\n",
    "train_df.to_csv(train_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file = os.path.join('data', 'test_gtex_expression_matrix_processed.tsv.gz')\n",
    "test_df.to_csv(test_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort genes based on median absolute deviation and output to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine most variably expressed genes and subset\n",
    "mad_genes_df = pd.DataFrame(train_df.mad(axis=0).sort_values(ascending=False)).reset_index()\n",
    "mad_genes_df.columns = ['gene_id', 'median_absolute_deviation']\n",
    "\n",
    "file = os.path.join('data', 'gtex_mad_genes.tsv')\n",
    "mad_genes_df.to_csv(file, sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}