3.0_best_validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e0a7ea0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import omnipath as op\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import scipy\n",
    "import torch\n",
    "import torch_geometric\n",
    "from torch_geometric.data import Data\n",
    "from sklearnex import patch_sklearn\n",
    "import sklearn\n",
    "patch_sklearn()\n",
    "from run.run_ae_default_config import *\n",
    "from tqdm import tqdm\n",
    "import scprep, phate\n",
    "import warnings; warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e90c0aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'Texas'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a11aaf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = np.load(f'data/{dataset}_curated_interactions.npz')\n",
    "train, val, test = splits['train'], splits['val'], splits['test']\n",
    "all_data = np.vstack((train, val,test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffad647f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_neg_pairs = []\n",
    "for pair in train:\n",
    "    if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "        train_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "train_neg_pairs = np.array(train_neg_pairs).reshape(-1,2)\n",
    "\n",
    "val_neg_pairs = []\n",
    "for pair in val:\n",
    "    if len(set(np.where(all_data[:, 0] == pair[1])[0]).intersection(np.where(all_data[:, 1] == pair[0])[0])) == 0:\n",
    "        val_neg_pairs.append([pair[1], pair[0]])\n",
    "\n",
    "val_neg_pairs = np.array(val_neg_pairs).reshape(-1,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc71365e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_neg_pairs = train_neg_pairs.astype(str)\n",
    "val_neg_pairs = val_neg_pairs.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db96664f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def directed_classifier(embedding, train_pairs, val_pairs):\n",
    "    embedding.index = embedding.index.astype(str)\n",
    "    train_pairs = train_pairs.astype(str)\n",
    "    val_pairs = val_pairs.astype(str)\n",
    "\n",
    "    X_train_pos = np.hstack((embedding.loc[train_pairs[:, 0]], embedding.loc[train_pairs[:, 1]]))\n",
    "    X_train_neg = np.hstack((embedding.loc[train_neg_pairs[:, 0]], embedding.loc[train_neg_pairs[:, 1]]))\n",
    "    X_train = np.vstack((X_train_pos, X_train_neg))\n",
    "    y_train = [1]*(X_train_pos.shape[0]) + [0]*(X_train_neg.shape[0])\n",
    "\n",
    "    X_val_pos = np.hstack((embedding.loc[val_pairs[:, 0]], embedding.loc[val_pairs[:, 1]]))\n",
    "    X_val_neg = np.hstack((embedding.loc[val_neg_pairs[:, 0]], embedding.loc[val_neg_pairs[:, 1]]))\n",
    "    X_val = np.vstack((X_val_pos, X_val_neg))\n",
    "    y_val = [1]*(X_val_pos.shape[0]) + [0]*(X_val_neg.shape[0])\n",
    "\n",
    "    clf = sklearn.linear_model.RidgeClassifier(random_state=0)\n",
    "    clf.fit(X_train, y_train, )\n",
    "    y_score_val = clf.decision_function(X_val)\n",
    "\n",
    "    return ((sklearn.metrics.roc_auc_score(y_val, y_score_val), \n",
    "            sklearn.metrics.average_precision_score(y_val, y_score_val)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adbd48fd",
   "metadata": {},
   "source": [
    "### DS-AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "618cb7a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/DS-AE/{dataset}')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da6cc8ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_performance = -1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/DS-AE/{dataset}/{i}_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    try:\n",
    "        res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    except:\n",
    "        continue\n",
    "    auroc = directed_classifier(res, train, val)[0]\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6467c7ec",
   "metadata": {},
   "source": [
    "### DS-PM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e08e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/DS-PM/{dataset}')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b418fa4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in range(maxrun+1):\n",
    "    try:\n",
    "        res = np.load(f'results/DS-PM/{dataset}/{i}_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2fad431",
   "metadata": {},
   "source": [
    "### Node2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1487482",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/Node2Vec/') if x.endswith(f'_{dataset}_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c345858e",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in range(maxrun+1):\n",
    "    res = np.load(f'results/Node2Vec/{i}_{dataset}_results.npz', allow_pickle=True)\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    auroc = directed_classifier(res, train, val)[0]\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5482298f",
   "metadata": {},
   "source": [
    "### MagNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcb7003f",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/MagNet/') if x.endswith('npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e487d86e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "performances = {}\n",
    "for i in range(maxrun+1):\n",
    "    try:\n",
    "        res = np.load(f'results/MagNet/{i}_{dataset}_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    auroc = directed_classifier(res, train, val)[0]\n",
    "    performances[i] = auroc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "470f5c71",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in dict(sorted(performances.items(), key=lambda item: item[1], reverse=True)).items():\n",
    "    config = np.load(f'results/MagNet/{k}_{dataset}_results.npz', allow_pickle=True)['config'][()]\n",
    "    if config['weight_decay'] == 0.001:\n",
    "        continue\n",
    "    else:\n",
    "        print (k,v)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f834cda4",
   "metadata": {},
   "source": [
    "### UDS-AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a258432",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/UDS-AE/{dataset}') if x.endswith('npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "287a1f56",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in range(maxrun+1):\n",
    "    try:\n",
    "        res = np.load(f'results/UDS-AE/{dataset}/{i}_{dataset}_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    auroc = directed_classifier(res, train, val)[0]\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cfcef04",
   "metadata": {},
   "source": [
    "### TransE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91abb5f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/TransE/') if x.endswith(f'_{dataset}_noedgeatt_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d69db216",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in range(maxrun+1):\n",
    "    try:\n",
    "        res = np.load(f'results/TransE/{i}_{dataset}_noedgeatt_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aef217e0",
   "metadata": {},
   "source": [
    "### HGCN undirected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c8d25d",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/HGCN/') if x.endswith(f'_{dataset}_undirected_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0030c9be",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/HGCN/{i}_{dataset}_undirected_results.npz', allow_pickle=True)\n",
    "    except:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d38ffaef",
   "metadata": {},
   "source": [
    "### HGCN directed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ef5cd54",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/HGCN/') if x.endswith(f'_{dataset}_directed_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b71b64a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/HGCN/{i}_{dataset}_directed_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f0d4332",
   "metadata": {},
   "source": [
    "### PM undirected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b03c72c",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/Shallow/') if x.endswith(f'_{dataset}_undirected_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81b6c6dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/Shallow/{i}_{dataset}_undirected_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7149129",
   "metadata": {},
   "source": [
    "### PM directed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddbbfe41",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/Shallow/') if x.endswith(f'_{dataset}_directed_results.npz')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d64f4de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/Shallow/{i}_{dataset}_directed_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f58621c",
   "metadata": {},
   "source": [
    "### GAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0b3e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "maxrun = max([int(x.split('_')[0]) for x in os.listdir(f'results/GAE/{dataset}')])\n",
    "maxrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c06aa976",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/GAE/{dataset}/{i}_{dataset}_undirected_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1313e53",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_performance=-1\n",
    "for i in tqdm(range(maxrun+1)):\n",
    "    try:\n",
    "        res = np.load(f'results/GAE/{dataset}/{i}_{dataset}_directed_results.npz', allow_pickle=True)\n",
    "    except FileNotFoundError:\n",
    "        continue\n",
    "    config = res['config']\n",
    "    res = pd.DataFrame(data=res['embedding'], index=res['names'])\n",
    "    try:\n",
    "        auroc = directed_classifier(res, train, val)[0]\n",
    "    except ValueError:\n",
    "        continue\n",
    "    if auroc > best_performance:\n",
    "        best_performance = auroc\n",
    "        best_config = i\n",
    "best_config, best_performance"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}