upload data and notebooks

VCG · May 25, 2023 · 4d838b7 · 4d838b7
1 parent f85d592
commit 4d838b7
Show file tree

Hide file tree

Showing 7 changed files with 111,033 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 .env
 
 .idea/
+.ipynb_checkpoints/
diff --git a/data/dbcells-dump.csv b/data/dbcells-dump.csv
diff --git a/data/dbcells_dump_table_data.h5 b/data/dbcells_dump_table_data.h5
diff --git a/data/h01_cell_body_nucleus_matrix_may16_2023.csv b/data/h01_cell_body_nucleus_matrix_may16_2023.csv
diff --git a/data/h01_nucleus_table_data.h5 b/data/h01_nucleus_table_data.h5
diff --git a/notebooks/Create Tables.ipynb b/notebooks/Create Tables.ipynb
@@ -0,0 +1,368 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "6KTnmzuJPA5k",
+    "outputId": "5f3df2ae-3d42-4f6b-c555-76c3e252818a"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: caveclient in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (5.4.0)\n",
+      "Requirement already satisfied: urllib3 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (1.26.9)\n",
+      "Requirement already satisfied: pandas in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (1.4.2)\n",
+      "Requirement already satisfied: requests in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (2.27.1)\n",
+      "Requirement already satisfied: pyarrow>=3 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages/pyarrow-7.0.0-py3.9-macosx-10.9-x86_64.egg (from caveclient) (7.0.0)\n",
+      "Requirement already satisfied: cachetools>=4 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (4.2.2)\n",
+      "Requirement already satisfied: attrs>=21.3.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (21.4.0)\n",
+      "Requirement already satisfied: numpy in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (1.21.5)\n",
+      "Requirement already satisfied: jsonschema in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (4.4.0)\n",
+      "Requirement already satisfied: ipython in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (8.2.0)\n",
+      "Requirement already satisfied: networkx in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from caveclient) (2.7.1)\n",
+      "Requirement already satisfied: backcall in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.2.0)\n",
+      "Requirement already satisfied: stack-data in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.2.0)\n",
+      "Requirement already satisfied: jedi>=0.16 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.18.1)\n",
+      "Requirement already satisfied: appnope in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.1.2)\n",
+      "Requirement already satisfied: pexpect>4.3 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (4.8.0)\n",
+      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (3.0.20)\n",
+      "Requirement already satisfied: decorator in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (5.1.1)\n",
+      "Requirement already satisfied: pickleshare in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.7.5)\n",
+      "Requirement already satisfied: pygments>=2.4.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (2.11.2)\n",
+      "Requirement already satisfied: setuptools>=18.5 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (61.2.0)\n",
+      "Requirement already satisfied: matplotlib-inline in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (0.1.2)\n",
+      "Requirement already satisfied: traitlets>=5 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from ipython->caveclient) (5.1.1)\n",
+      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from jedi>=0.16->ipython->caveclient) (0.8.3)\n",
+      "Requirement already satisfied: ptyprocess>=0.5 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from pexpect>4.3->ipython->caveclient) (0.7.0)\n",
+      "Requirement already satisfied: wcwidth in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->caveclient) (0.2.5)\n",
+      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from jsonschema->caveclient) (0.18.0)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from pandas->caveclient) (2.8.2)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from pandas->caveclient) (2021.3)\n",
+      "Requirement already satisfied: six>=1.5 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas->caveclient) (1.16.0)\n",
+      "Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from requests->caveclient) (2.0.4)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from requests->caveclient) (2021.10.8)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from requests->caveclient) (3.3)\n",
+      "Requirement already satisfied: pure-eval in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from stack-data->ipython->caveclient) (0.2.2)\n",
+      "Requirement already satisfied: executing in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from stack-data->ipython->caveclient) (0.8.3)\n",
+      "Requirement already satisfied: asttokens in /Users/jinhan/opt/anaconda3/lib/python3.9/site-packages (from stack-data->ipython->caveclient) (2.0.5)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install caveclient"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "id": "X1A9BoWNMjdT"
+   },
+   "outputs": [],
+   "source": [
+    "from caveclient import CAVEclient\n",
+    "import numpy as np\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# CAVEclient"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "id": "9FjgR8bxPI2K"
+   },
+   "outputs": [],
+   "source": [
+    "url = \"https://global.brain-wire-test.org/\"\n",
+    "datastack = \"h01_c3_flat\"\n",
+    "token = \"3ac8d9f0de1b192dba9c085114f0f811\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "7RJI8vz7PLF8",
+    "outputId": "54e3014c-45bf-4206-81cb-2b99389a174b"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Updated datastack-to-server cache — 'https://global.brain-wire-test.org/' will now be used by default for datastack 'h01_c3_flat'\n"
+     ]
+    }
+   ],
+   "source": [
+    "client = CAVEclient(datastack_name=datastack, server_address=url, auth_token=token)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "xgBR6GFVP3S1",
+    "outputId": "de723a5f-c698-4e36-9e5a-3e2cd09b1f86"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['h01_c3_flat']"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client.info.get_datastacks()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "SZOWBNUQPL4M",
+    "outputId": "0f74678e-205c-4b3b-b055-806d8d42f7cf"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['nucleus_test_table2', 'nucleus_test_table', 'test_table', 'test_table2']"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get existing tables\n",
+    "all_tables = client.annotation.get_tables()\n",
+    "all_tables"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## create a nucleus table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "id": "Qu-2roASRZwi"
+   },
+   "outputs": [],
+   "source": [
+    "# create a new table\n",
+    "table_name = \"nucleus_table\"\n",
+    "description = \"\"\"\n",
+    "This is a table that contains nucleus id, volume, position and bounding box's position.\n",
+    "\n",
+    "About Columns:\n",
+    "- id: ID in the VAST segmentation file (for neurons/glia this should be consistent with the 'dbid' value in other tables)\n",
+    "- volume: volume estimate of cell body or nucleus in cubic micrometers, at _very low resolution_ (mip6 every 128th section for neurons/glia, mip7 every 16th section for bv nuclei, manually painted)\n",
+    "- pt_position: X,Y,Z anchor point within cell body or nucleus, in pixels at 8x8x33 nm\n",
+    "- bb_start_position: Xmin,Ymin,Zmin of bounding box of cell body or nucleus, in pixels at 8x8x33 nm\n",
+    "- bb_end_position: Xmax,Ymax,Zmax of bounding box of cell body or nucleus, in pixels at 8x8x33 nm\n",
+    "The data in this table was compiled on May 16, 2023, by Daniel Berger.\n",
+    "\n",
+    "This table was create by Jinhan Choi.\n",
+    "\"\"\"\n",
+    "schema_name = \"nucleus_detection\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "Kb3182KOPP5Q",
+    "outputId": "571c608a-ea65-40fe-9c3e-e2539b06de62"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [200]>"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client.annotation.create_table(table_name=table_name, \n",
+    "                               schema_name=schema_name, \n",
+    "                               description=description, \n",
+    "                               voxel_resolution=[1, 1, 1])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create annotation table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "id": "SxM2NTsu5Epw"
+   },
+   "outputs": [],
+   "source": [
+    "# create a new table\n",
+    "table_name = \"dbcells_dump\"\n",
+    "description = \"\"\"\n",
+    "This is a table that contains cell's position, types, and sub-types.\n",
+    "\n",
+    "About Columns:\n",
+    "- pt_position: X,Y,Z anchor point within cell body or nucleus, in pixels at 8x8x33 nm\n",
+    "- classification_system: comma-seperated string joined from 'dbsubcelltype', 'dbsubcelltype2', and 'dbsubcelltype3' \n",
+    "- cell_type:  cell type (manually classified):\n",
+    "% dbcelltypes are:\n",
+    "%   0:  [U] unknown,\n",
+    "%   1:  [P  pyramidal,\n",
+    "%   2:  [I  interneuron,\n",
+    "%   3:  [N  unclassified neuron,\n",
+    "%   4:  [A] astrocyte,\n",
+    "%   5:  [O] oligodendrocyte,\n",
+    "%   6:  [G] mg/opc,\n",
+    "%   7:  [C] 'c-shaped' cell (also mg/opc),\n",
+    "%   8:  [B] blood vessel cell (does not occur in this list)\n",
+    "%   9:  [S  Spiny stellate\n",
+    "%   10: [E  Excitatory/spiny with atypical tree\n",
+    "% Evelinas blood vessel nucleus types are:\n",
+    "%   11: 'Pericytes'\n",
+    "%   12: 'vSMC'\n",
+    "%   13: 'fibroblast'\n",
+    "%   14: 'perivascular lymphocytes'\n",
+    "%   15: 'perivascular macrophages'\n",
+    "%   16: 'undecided' (unspecified blood vessel cell)\n",
+    "%   17: 'Endothelial cells'\n",
+    "%   18: 'circulating imm'\n",
+    "\n",
+    "The data in this table was compiled on May 16, 2023. \n",
+    "Source: console.cloud.google.com/bigquery?ws=!1m5!1m4!4m3!1slcht-goog-connectomics!2sgoog14r0seg1!3sdbcells\n",
+    "\n",
+    "This table was create by Jinhan Choi.\n",
+    "\"\"\"\n",
+    "schema_name = \"cell_type_local\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [200]>"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client.annotation.create_table(table_name=table_name, \n",
+    "                               schema_name=schema_name, \n",
+    "                               description=description, \n",
+    "                               voxel_resolution=[1, 1, 1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['nucleus_test_table2',\n",
+       " 'nucleus_test_table',\n",
+       " 'nucleus_table',\n",
+       " 'dbcells_dump',\n",
+       " 'test_table',\n",
+       " 'test_table2']"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_tables = client.annotation.get_tables()\n",
+    "all_tables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}