1_Data_extration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0",
   "metadata": {},
   "source": [
    "# Extract and align data from Onix, Harp, Sleap, and photometry"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import harp\n",
    "\n",
    "from harp_resources import process, utils\n",
    "from sleap import load_and_process as lp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3",
   "metadata": {},
   "source": [
    "# Extracting Noras data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#photometry_path = '/Volumes/RanczLab/Nora_Cohort1_training/Training_day4/B6J2717-2024-11-28T09-37-55/photometry'\n",
    "#data_path = '/Volumes/RanczLab/Nora_Cohort1_training/Training_day4/B6J2717-2024-11-28T09-37-55/'\n",
    "\n",
    "#photometry_path = '/Volumes/RanczLab/Nora_Cohort1_training/Visual_mismatch_day1/B6J2717/photometry_processed'\n",
    "#data_path = '/Volumes/RanczLab/Nora_Cohort1_training/Visual_mismatch_day1/B6J2717/'\n",
    "\n",
    "data_path = Path('/Users/rancze/Documents/Data/Mismatch_experiments/Cohort0_GCaMP_example/2024-08-08T10-05-26_B3M3')\n",
    "photometry_path = Path('/Users/rancze/Documents/Data/Mismatch_experiments/Cohort0_GCaMP_example/photometry/B3M3_MMclosedOpen_day2/2024_08_08-12_08_29')\n",
    "\n",
    "#h1_datafolder = data_path+'HarpDataH1' #this failed for Ede\n",
    "#h2_datafolder = data_path+'HarpDataH2'\n",
    "\n",
    "h1_datafolder = data_path / 'HarpDataH1'\n",
    "h2_datafolder = data_path / 'HarpDataH2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_csv(str(photometry_path)+'/Processed_fluorescence.csv') #Processed_fluorescence.csv #Fluorescence.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from andrew:\n",
    "#Changed didgital output file\n",
    "h1_reader = harp.create_reader('harp_resources/h1-device.yml', epoch=harp.REFERENCE_EPOCH)\n",
    "h2_reader = harp.create_reader('harp_resources/h2-device.yml', epoch=harp.REFERENCE_EPOCH)\n",
    "session_data_reader = utils.SessionData(\"SessionSettings\")\n",
    "experiment_events_reader = utils.TimestampedCsvReader(\"ExperimentEvents\", columns=[\"Event\"])\n",
    "framecount_reader = utils.TimestampedCsvReader(\"OnixAnalogFrameCount\", columns=[\"Index\"])\n",
    "photometry_reader = utils.PhotometryReader(\"Processed_fluorescence\")\n",
    "video_reader = utils.Video(\"VideoData1\")\n",
    "onix_digital_reader = utils.TimestampedCsvReader(\"OnixDigital\", columns=[\"Clock\", \"HubClock\", \n",
    "                                                                         \"DigitalInputs0\",\n",
    "                                                                         \"DigitalInputs1\",\n",
    "                                                                         \"DigitalInputs2\",\n",
    "                                                                         \"DigitalInputs3\",\n",
    "                                                                         \"DigitalInputs4\",\n",
    "                                                                         \"DigitalInputs5\"\n",
    "                                                                         \"DigitalInputs6\",\n",
    "                                                                         \"DigitalInputs7\",\n",
    "                                                                         \"DigitalInputs8\",\n",
    "                                                                         \"Buttons\"])\n",
    "onix_harp_reader = utils.TimestampedCsvReader(\"OnixHarp\", columns=[\"Clock\", \"HubClock\", \"HarpTime\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read metadata\n",
    "session_settings = utils.load_2(session_data_reader, data_path)\n",
    "\n",
    "print(session_settings.iloc[0]['metadata'].blocks[0].haltProtocol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read harp streams, experiment events, video\n",
    "flow_sensor = utils.load_harp(h1_reader.OpticalTrackingRead, h1_datafolder)\n",
    "camera_triggers = utils.load_harp(h1_reader.Cam0Event, h1_datafolder)\n",
    "experiment_events = utils.load_2(experiment_events_reader, root_folder)\n",
    "video_data = utils.load_2(video_reader, root_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "10",
   "metadata": {},
   "source": [
    "# Bulk extraction\n",
    "## old data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "#data_path = Path('/home/ikharitonov/RANCZLAB-NAS/data/ONIX/20240730_Mismatch_Experiment/MMclosed&Regular_120824/2024-08-12T16-51-16_B3M3')\n",
    "#photometry_path = Path('/home/ikharitonov/RANCZLAB-NAS/data/ONIX/20240730_Mismatch_Experiment/MMclosed&Regular_120824/photometry/B3M3_MMclosed&Regular_day1/2024_08_12-18_57_17')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import harp\n",
    "\n",
    "from harp_resources import process, utils\n",
    "from sleap import load_and_process as lp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "rootdir = '/Users/nora/Desktop/Cohort0_GCaMP_example/'\n",
    "#rootdir = '/Volumes/RanczLab/20240730_Mismatch_Experiment/GRAB_MMclosed-and-Regular_220824'\n",
    "\n",
    "#initialize sets to ensure uniqueness\n",
    "data_paths_set = set()\n",
    "photometry_paths_set = set()\n",
    "\n",
    "data_paths = []\n",
    "photometry_paths = []\n",
    "\n",
    "for dirpath, subdirs, files in os.walk(rootdir):\n",
    "    #data paths\n",
    "    if 'ExperimentEvents' in dirpath:\n",
    "        trimmed_path = dirpath[:-17]\n",
    "        if trimmed_path not in data_paths_set:\n",
    "            data_paths_set.add(trimmed_path)\n",
    "            data_paths.append(trimmed_path)\n",
    "\n",
    "    #photometry paths\n",
    "    for x in files:\n",
    "        if 'Processed_fluorescence.csv' in x:\n",
    "            if dirpath not in photometry_paths_set:\n",
    "                photometry_paths_set.add(dirpath)\n",
    "                photometry_paths.append(dirpath)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "photometry_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_extraction_makeh5(data_path, photometry_path, make_h5 = False, eyes = False):\n",
    "    print(f'\\n Running extraction for {data_path.split('/')[-1][20:]} for session {data_path.split('/')[-2]}')\n",
    "    #data_path = Path(data_path)\n",
    "    #photometry_path = Path(photometry_path)\n",
    "    SessionSettings = utils.read_SessionSettings(Path(data_path), print_contents=True)\n",
    "    conversions = process.calculate_conversions_second_approach(Path(data_path), Path(photometry_path), verbose=False)\n",
    "    streams = utils.load_registers(Path(data_path))\n",
    "\n",
    "    Photometry = utils.read_fluorescence(Path(photometry_path))\n",
    "    Photometry['HARP Timestamps'] = conversions['photometry_to_harp_time'](Photometry.index)\n",
    "    \n",
    "    OnixAnalogClock = utils.read_OnixAnalogClock(Path(data_path))\n",
    "    OnixAnalogData = utils.read_OnixAnalogData(Path(data_path), binarise=True)\n",
    "    ExperimentEvents = utils.read_ExperimentEvents(Path(data_path)) \n",
    "    \n",
    "    photodiode_series = pd.Series(OnixAnalogData[:,0], index=conversions['onix_to_harp_timestamp'](OnixAnalogClock))\n",
    "\n",
    "    print('Adding Photometry, Eye Movements and Photodiode to the streams')\n",
    "    streams = process.reformat_and_add_many_streams(streams, Photometry, 'Photometry', ['470_dfF', 'z_470'], index_column_name='HARP Timestamps')\n",
    "    streams = process.add_stream(streams, 'ONIX', photodiode_series, 'Photodiode')\n",
    "    \n",
    "    \n",
    "    _ = process.get_timepoint_info(streams, print_all=True)\n",
    "    resampled_streams = process.pad_and_resample(streams, resampling_period='1 ms', method='linear')\n",
    "    _ = process.get_timepoint_info(resampled_streams, print_all=True)\n",
    "\n",
    "    print('Applying linear and angular conversion to Optical tracking sensor streams (cm / sec and degrees / sec)')\n",
    "    resampled_streams['H1']['OpticalTrackingRead0X(46)'] = process.running_unit_conversion(resampled_streams['H1']['OpticalTrackingRead0X(46)']*100)\n",
    "    resampled_streams['H1']['OpticalTrackingRead0Y(46)'] = process.rotation_unit_conversion(resampled_streams['H1']['OpticalTrackingRead0Y(46)'])\n",
    "        \n",
    "    print('Streams are extracted and can be used or made to h5')\n",
    "    if make_h5:\n",
    "        process.save_streams_as_h5(Path(data_path), resampled_streams, streams_to_save_pattern, SessionSetting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_and_add_many_streams(streams, dataframe, source_name, stream_names, index_column_name='Seconds'):\n",
    "    for stream_name in stream_names:\n",
    "        print(dataframe)\n",
    "        new_stream = process.reformat_dataframe(dataframe, stream_name, index_column_name, data_column_name=stream_name)\n",
    "        streams = process.add_stream(streams, source_name, new_stream, stream_name)\n",
    "    return streams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_extraction_makeh5(data_paths[0], photometry_paths[0]) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_extraction_makeh5(data_path, photometry_path, photometry_traces = ['470_dfF', 'z_470'],make_h5=False, eyes=False):\n",
    "    print(f'\\n Running extraction for {data_path.split(\"/\")[-1][20:]} for session {data_path.split(\"/\")[-2]}')\n",
    "    \n",
    "    SessionSettings = utils.read_SessionSettings(Path(data_path), print_contents=True)\n",
    "    conversions = process.calculate_conversions_second_approach(Path(data_path), Path(photometry_path), verbose=False)\n",
    "    streams = utils.load_registers(Path(data_path))\n",
    "\n",
    "    Photometry = utils.read_fluorescence(Path(photometry_path))\n",
    "    Photometry['HARP Timestamps'] = conversions['photometry_to_harp_time'](Photometry.index)\n",
    "    \n",
    "    OnixAnalogClock = utils.read_OnixAnalogClock(Path(data_path))\n",
    "    OnixAnalogData = utils.read_OnixAnalogData(Path(data_path), binarise=True)\n",
    "    ExperimentEvents = utils.read_ExperimentEvents(Path(data_path)) \n",
    "    \n",
    "    photodiode_series = pd.Series(OnixAnalogData[:, 0], index=conversions['onix_to_harp_timestamp'](OnixAnalogClock))\n",
    "\n",
    "    print('Adding Photometry, Eye Movements and Photodiode to the streams')\n",
    "    streams = process.reformat_and_add_many_streams(streams, Photometry, 'Photometry', photometry_traces,\n",
    "                                                    index_column_name='HARP Timestamps')\n",
    "    streams = process.add_stream(streams, 'ONIX', photodiode_series, 'Photodiode')\n",
    "\n",
    "    # Handle Eye Movement Data if specified\n",
    "    if eyes:\n",
    "        print('  Checking for and processing videography data...')\n",
    "        try:\n",
    "            # Load videography data\n",
    "            VideoData1, VideoData2, VideoData1_Has_Sleap, VideoData2_Has_Sleap = lp.load_videography_data(data_path)\n",
    "\n",
    "            if VideoData2_Has_Sleap:\n",
    "                print('  Processing VideoData2 with SLEAP data...')\n",
    "                \n",
    "                # Interpolate missing data\n",
    "                VideoData2 = VideoData2.interpolate()\n",
    "\n",
    "                # Extract coordinates and compute transformations\n",
    "                columns_of_interest = [\n",
    "                    'left.x', 'left.y', 'center.x', 'center.y', 'right.x', 'right.y',\n",
    "                    'p1.x', 'p1.y', 'p2.x', 'p2.y', 'p3.x', 'p3.y', 'p4.x', 'p4.y',\n",
    "                    'p5.x', 'p5.y', 'p6.x', 'p6.y', 'p7.x', 'p7.y', 'p8.x', 'p8.y'\n",
    "                ]\n",
    "                coordinates_dict = lp.get_coordinates_dict(VideoData2, columns_of_interest)\n",
    "\n",
    "                # Calculate transformations\n",
    "                theta = lp.find_horizontal_axis_angle(VideoData2, 'left', 'center')\n",
    "                center_point = lp.get_left_right_center_point(coordinates_dict)\n",
    "\n",
    "                reformatted_coordinates_dict = lp.get_reformatted_coordinates_dict(coordinates_dict, ['left', 'right', 'center'] + [f'p{i}' for i in range(1, 9)])\n",
    "                centered_coordinates_dict = lp.get_centered_coordinates_dict(reformatted_coordinates_dict, center_point)\n",
    "                rotated_coordinates_dict = lp.get_rotated_coordinates_dict(centered_coordinates_dict, theta)\n",
    "\n",
    "                # Extract ellipse parameters\n",
    "                columns_of_interest = [f'p{i}' for i in range(1, 9)]\n",
    "                ellipse_parameters_data, ellipse_center_points_data = lp.get_fitted_ellipse_parameters(\n",
    "                    rotated_coordinates_dict, columns_of_interest\n",
    "                )\n",
    "\n",
    "                # Compute additional metrics\n",
    "                average_diameter = np.mean([ellipse_parameters_data[:, 0], ellipse_parameters_data[:, 1]], axis=0)\n",
    "\n",
    "                # Prepare SLEAP video data for streams\n",
    "                SleapVideoData2 = process.convert_arrays_to_dataframe(\n",
    "                    ['Seconds', 'Ellipse.Diameter', 'Ellipse.Angle', 'Ellipse.Center.X', 'Ellipse.Center.Y'],\n",
    "                    [VideoData2['Seconds'].values, average_diameter, ellipse_parameters_data[:, 2],\n",
    "                     ellipse_center_points_data[:, 0], ellipse_center_points_data[:, 1]]\n",
    "                )\n",
    "\n",
    "                streams = process.reformat_and_add_many_streams(\n",
    "                    streams, SleapVideoData2, 'SleapVideoData2',\n",
    "                    ['Ellipse.Diameter', 'Ellipse.Angle', 'Ellipse.Center.X', 'Ellipse.Center.Y']\n",
    "                )\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Error processing videography data: {e}\")\n",
    "\n",
    "    # Display timepoint info for streams\n",
    "    _ = process.get_timepoint_info(streams, print_all=True)\n",
    "\n",
    "    # Resample streams\n",
    "    resampled_streams = process.pad_and_resample(streams, resampling_period='1 ms', method='linear')\n",
    "    _ = process.get_timepoint_info(resampled_streams, print_all=True)\n",
    "\n",
    "    # Apply unit conversions for optical tracking sensor streams\n",
    "    print('  Applying linear and angular conversion to Optical tracking sensor streams (cm/sec and degrees/sec)')\n",
    "    resampled_streams['H1']['OpticalTrackingRead0X(46)'] = process.running_unit_conversion(\n",
    "        resampled_streams['H1']['OpticalTrackingRead0X(46)'] * 100\n",
    "    )\n",
    "    resampled_streams['H1']['OpticalTrackingRead0Y(46)'] = process.rotation_unit_conversion(\n",
    "        resampled_streams['H1']['OpticalTrackingRead0Y(46)']\n",
    "    )\n",
    "\n",
    "    print(' - Streams are extracted and can be used or made to h5')\n",
    "\n",
    "    if make_h5:\n",
    "        # Define streams to save, including SLEAP data if processed\n",
    "        streams_to_save_pattern = {\n",
    "            'Photometry':photometry_traces,\n",
    "            'ONIX': ['Photodiode'],\n",
    "        }\n",
    "        if eyes:\n",
    "            streams_to_save_pattern['SleapVideoData2'] = ['Ellipse.Diameter', 'Ellipse.Center.X', 'Ellipse.Center.Y']\n",
    "\n",
    "        process.save_streams_as_h5(Path(data_path), resampled_streams, streams_to_save_pattern)\n",
    "        print('Streams saved as h5 file \\n')\n",
    "\n",
    "    return data_path, resampled_streams\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path, resampled_streams= run_extraction_makeh5(data_paths[0], photometry_paths[0], make_h5=True, eyes=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21",
   "metadata": {},
   "source": [
    "## Loading and Synchronisation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "photometry_path = Path(photometry_paths[3])\n",
    "data_path = Path(data_paths[3])\n",
    "photometry_paths[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23",
   "metadata": {},
   "outputs": [],
   "source": [
    "photometry_path = Path('/Volumes/RanczLab/Photometry_recordings/August_Mismatch_Experiment_G8m/B3M3_MMclosed_and_Regular_day2')\n",
    "data_path = Path('/Volumes/RanczLab/20240730_Mismatch_Experiment/G8_MMclosed-and-regular_130824/2024-08-13T12-53-01_B3M3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24",
   "metadata": {},
   "outputs": [],
   "source": [
    "photometry_path = Path('/Volumes/RanczLab/Photometry_recordings/August_Mismatch_Experiment_G8m/MM_closed-and-open_day2/B2M4/photometry')\n",
    "data_path = Path('/Volumes/RanczLab/20240730_Mismatch_Experiment/G8_MMclosed-and-open_080824/2024-08-08T09-20-54_B2M4')\n",
    "#works"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25",
   "metadata": {},
   "source": [
    "### TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27",
   "metadata": {},
   "outputs": [],
   "source": [
    "photometry_paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28",
   "metadata": {},
   "outputs": [],
   "source": [
    "#OnixDigital = utils.read_OnixDigital(Path(data_path))\n",
    "#PhotometryEvents = utils.read_fluorescence_events(Path(photometry_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29",
   "metadata": {},
   "outputs": [],
   "source": [
    "OnixDigital = utils.read_OnixDigital(Path(data_paths[2]))\n",
    "PhotometryEvents = utils.read_fluorescence_events(Path(photometry_paths[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30",
   "metadata": {},
   "outputs": [],
   "source": [
    "OnixDigital"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31",
   "metadata": {},
   "outputs": [],
   "source": [
    "PhotometryEvents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32",
   "metadata": {},
   "outputs": [],
   "source": [
    "onix_digital_array = OnixDigital[\"Seconds\"].values\n",
    "photometry_events_array = PhotometryEvents['TimeStamp'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example time series (replace with your actual data)\n",
    "from scipy.signal import correlate\n",
    "\n",
    "\n",
    "time_series_1 = np.diff(onix_digital_array) # First time series\n",
    "time_series_2 = np.diff(photometry_events_array)   # Second time series\n",
    "\n",
    "# Cross-correlation\n",
    "correlation = correlate(time_series_1, time_series_2, mode='full')\n",
    "offset = np.argmax(correlation) - (len(time_series_2) - 1)\n",
    "\n",
    "# Results\n",
    "print(\"Offset at maximum correlation:\", offset)\n",
    "\n",
    "# Optional: Visualize the correlation\n",
    "\n",
    "lags = np.arange(-len(time_series_2) + 1, len(time_series_1))\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(lags, correlation)\n",
    "plt.title(\"Cross-Correlation\")\n",
    "plt.xlabel(\"Lag\")\n",
    "plt.ylabel(\"Correlation\")\n",
    "plt.axvline(x=offset, color='red', linestyle='--', label=f'Max Offset: {offset}')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34",
   "metadata": {},
   "outputs": [],
   "source": [
    "onix_digital_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(np.unique(np.diff(photometry_events_array)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.diff(photometry_events_array)[8:])\n",
    "ax2.plot()\n",
    "#plt.xlim([0,len(np.diff(photometry_events_array))])\n",
    "#plt.xlim([168,200+168])\n",
    "#plt.xlim([0,200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.diff(onix_digital_array))\n",
    "#plt.xlim([0, len(np.diff(onix_digital_array))-168])\n",
    "#plt.xlim([0,200])\n",
    "#plt.xlim([26,226])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax1 = plt.subplots(figsize=(15,10))\n",
    "ax2 = ax1.twinx()\n",
    "if offset > 0:\n",
    "    ax2.plot(np.diff(onix_digital_array[:offset]))\n",
    "    ax1.plot(np.diff(photometry_events_array), color = 'r')\n",
    "if offset < 0:\n",
    "    ax2.plot(np.diff(onix_digital_array))\n",
    "    ax1.plot(np.diff(photometry_events_array[abs(offset):]), color = 'r')\n",
    "if offset == 0:\n",
    "    ax2.plot(np.diff(onix_digital_array))\n",
    "    ax1.plot(np.diff(photometry_events_array), color = 'r')\n",
    "#plt.xlim([0,200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42",
   "metadata": {},
   "outputs": [],
   "source": [
    "conversions = process.calculate_conversions_second_approach(data_paths[0], photometry_paths[0], verbose=False)\n",
    "# After hardware ONIX clock implementation - this will have to be adapted\n",
    "# Only photometry will need to be converted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44",
   "metadata": {},
   "outputs": [],
   "source": [
    "streams = utils.load_registers(data_path)\n",
    "\n",
    "Photometry = utils.read_fluorescence(photometry_path)\n",
    "Photometry['HARP Timestamps'] = conversions['photometry_to_harp_time'](Photometry['TimeStamp'])\n",
    "\n",
    "OnixAnalogClock = utils.read_OnixAnalogClock(data_path)\n",
    "OnixAnalogData = utils.read_OnixAnalogData(data_path, binarise=True)\n",
    "ExperimentEvents = utils.read_ExperimentEvents(data_path) \n",
    "\n",
    "\n",
    "\n",
    "photodiode_series = pd.Series(OnixAnalogData[:,0], index=conversions['onix_to_harp_timestamp'](OnixAnalogClock))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding Photometry, Eye Movements and Photodiode to the streams\n",
    "streams = process.reformat_and_add_many_streams(streams, Photometry, 'Photometry', ['CH1-410', 'CH1-470', 'CH1-560'], index_column_name='HARP Timestamps')\n",
    "streams = process.add_stream(streams, 'ONIX', photodiode_series, 'Photodiode')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = process.get_timepoint_info(streams, print_all=True)\n",
    "resampled_streams = process.pad_and_resample(streams, resampling_period='1 ms', method='linear')\n",
    "_ = process.get_timepoint_info(resampled_streams, print_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying linear and angular conversion to Optical tracking sensor streams\n",
    "# OpticalTrackingRead0X(46) converted to centimeters per second\n",
    "# OpticalTrackingRead0Y(46) covnerted to degrees per second\n",
    "resampled_streams['H1']['OpticalTrackingRead0X(46)'] = process.running_unit_conversion(resampled_streams['H1']['OpticalTrackingRead0X(46)']*100)\n",
    "resampled_streams['H1']['OpticalTrackingRead0Y(46)'] = process.rotation_unit_conversion(resampled_streams['H1']['OpticalTrackingRead0Y(46)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(type(resampled_streams['Photometry']['CH1-470']))\n",
    "resampled_streams['Photometry']['CH1-470']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49",
   "metadata": {},
   "outputs": [],
   "source": [
    "streams_to_save_pattern = {'H1': ['OpticalTrackingRead0X(46)', 'OpticalTrackingRead0Y(46)'], 'H2': ['Encoder(38)'], 'Photometry': ['CH1-410', 'CH1-470', 'CH1-560'], 'ONIX': ['Photodiode']}\n",
    "streams_to_save_pattern\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "aeon",
   "language": "python",
   "name": "aeon"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}