diff --git a/CHANGELOG.md b/CHANGELOG.md index 9770d9d48..bb7e26900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Change Log All notable changes to this project will be documented in this file. +## [2.13.0] = 2021-09-22 +- Major internal refactor to BehaviorSession, BehaviorOphysExperiment classes. Implements DataObject pattern for fetching and serialization of data. + ## [2.12.4] = 2021-09-21 - Documentation changes ahead of SWDB 2021 - Bugfix to CloudCache; it is now possible for multiple users to share a cache. diff --git a/allensdk/__init__.py b/allensdk/__init__.py index ea675c49a..1bc7205ce 100644 --- a/allensdk/__init__.py +++ b/allensdk/__init__.py @@ -35,7 +35,7 @@ # import logging -__version__ = '2.12.4' +__version__ = '2.13.0' try: diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py index e6428a6cd..f13171ff5 100644 --- a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py +++ b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py @@ -1,33 +1,241 @@ +from typing import Optional + import numpy as np import pandas as pd -from typing import Any +from pynwb import NWBFile from allensdk.brain_observatory.behavior.behavior_session import ( BehaviorSession) -from allensdk.brain_observatory.session_api_utils import ParamsMixin -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi, BehaviorOphysLimsApi) +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_files.eye_tracking_file import \ + EyeTrackingFile +from allensdk.brain_observatory.behavior.data_files\ + .rigid_motion_transform_file import \ + RigidMotionTransformFile +from allensdk.brain_observatory.behavior.data_objects import \ + BehaviorSessionId, StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.cell_specimens \ + .cell_specimens import \ + CellSpecimens, EventsParams +from allensdk.brain_observatory.behavior.data_objects.eye_tracking\ + .eye_tracking_table import \ + EyeTrackingTable +from allensdk.brain_observatory.behavior.data_objects.eye_tracking\ + .rig_geometry import \ + RigGeometry as EyeTrackingRigGeometry +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.date_of_acquisition import \ + DateOfAcquisitionOphys, DateOfAcquisition +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_ophys_metadata import \ + BehaviorOphysMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group \ + import \ + ImagingPlaneGroup +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata.multi_plane_metadata \ + import \ + MultiplaneMetadata +from allensdk.brain_observatory.behavior.data_objects.motion_correction \ + import \ + MotionCorrection +from allensdk.brain_observatory.behavior.data_objects.projections import \ + Projections +from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \ + calculate_monitor_delay +from allensdk.brain_observatory.behavior.data_objects.timestamps \ + .ophys_timestamps import \ + OphysTimestamps, OphysTimestampsMultiplane +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP from allensdk.deprecated import legacy -from allensdk.brain_observatory.behavior.image_api import Image, ImageApi +from allensdk.brain_observatory.behavior.image_api import Image +from allensdk.internal.api import db_connection_creator -class BehaviorOphysExperiment(BehaviorSession, ParamsMixin): +class BehaviorOphysExperiment(BehaviorSession): """Represents data from a single Visual Behavior Ophys imaging session. - Can be initialized with an api that fetches data, or by using class methods - `from_lims` and `from_nwb_path`. + Initialize by using class methods `from_lims` or `from_nwb_path`. """ - def __init__(self, api=None, + def __init__(self, + behavior_session: BehaviorSession, + projections: Projections, + ophys_timestamps: OphysTimestamps, + cell_specimens: CellSpecimens, + metadata: BehaviorOphysMetadata, + motion_correction: MotionCorrection, + eye_tracking_table: Optional[EyeTrackingTable], + eye_tracking_rig_geometry: Optional[EyeTrackingRigGeometry], + date_of_acquisition: DateOfAcquisition): + super().__init__( + behavior_session_id=behavior_session._behavior_session_id, + licks=behavior_session._licks, + metadata=behavior_session._metadata, + raw_running_speed=behavior_session._raw_running_speed, + rewards=behavior_session._rewards, + running_speed=behavior_session._running_speed, + running_acquisition=behavior_session._running_acquisition, + stimuli=behavior_session._stimuli, + stimulus_timestamps=behavior_session._stimulus_timestamps, + task_parameters=behavior_session._task_parameters, + trials=behavior_session._trials, + date_of_acquisition=date_of_acquisition + ) + + self._metadata = metadata + self._projections = projections + self._ophys_timestamps = ophys_timestamps + self._cell_specimens = cell_specimens + self._motion_correction = motion_correction + self._eye_tracking = eye_tracking_table + self._eye_tracking_rig_geometry = eye_tracking_rig_geometry + + def to_nwb(self) -> NWBFile: + nwbfile = super().to_nwb(add_metadata=False) + + self._metadata.to_nwb(nwbfile=nwbfile) + self._projections.to_nwb(nwbfile=nwbfile) + self._cell_specimens.to_nwb(nwbfile=nwbfile, + ophys_timestamps=self._ophys_timestamps) + self._motion_correction.to_nwb(nwbfile=nwbfile) + self._eye_tracking.to_nwb(nwbfile=nwbfile) + self._eye_tracking_rig_geometry.to_nwb(nwbfile=nwbfile) + + return nwbfile + # ==================== class and utility methods ====================== + + @classmethod + def from_lims(cls, + ophys_experiment_id: int, + eye_tracking_z_threshold: float = 3.0, + eye_tracking_dilation_frames: int = 2, + events_filter_scale: float = 2.0, + events_filter_n_time_steps: int = 20, + exclude_invalid_rois=True, + skip_eye_tracking=False) -> \ + "BehaviorOphysExperiment": + """ + Parameters + ---------- + ophys_experiment_id + eye_tracking_z_threshold + See `BehaviorOphysExperiment.from_nwb` + eye_tracking_dilation_frames + See `BehaviorOphysExperiment.from_nwb` + events_filter_scale + See `BehaviorOphysExperiment.from_nwb` + events_filter_n_time_steps + See `BehaviorOphysExperiment.from_nwb` + exclude_invalid_rois + Whether to exclude invalid rois + skip_eye_tracking + Used to skip returning eye tracking data + """ + def _is_multi_plane_session(): + imaging_plane_group_meta = ImagingPlaneGroup.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + return cls._is_multi_plane_session( + imaging_plane_group_meta=imaging_plane_group_meta) + + def _get_motion_correction(): + rigid_motion_transform_file = RigidMotionTransformFile.from_lims( + ophys_experiment_id=ophys_experiment_id, db=lims_db + ) + return MotionCorrection.from_data_file( + rigid_motion_transform_file=rigid_motion_transform_file) + + def _get_eye_tracking_table(sync_file: SyncFile): + eye_tracking_file = EyeTrackingFile.from_lims( + db=lims_db, ophys_experiment_id=ophys_experiment_id) + eye_tracking_table = EyeTrackingTable.from_data_file( + data_file=eye_tracking_file, + sync_file=sync_file, + z_threshold=eye_tracking_z_threshold, + dilation_frames=eye_tracking_dilation_frames + ) + return eye_tracking_table + + lims_db = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP + ) + sync_file = SyncFile.from_lims(db=lims_db, + ophys_experiment_id=ophys_experiment_id) + stimulus_timestamps = StimulusTimestamps.from_sync_file( + sync_file=sync_file) + behavior_session_id = BehaviorSessionId.from_lims( + db=lims_db, ophys_experiment_id=ophys_experiment_id) + is_multiplane_session = _is_multi_plane_session() + meta = BehaviorOphysMetadata.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db, + is_multiplane=is_multiplane_session + ) + monitor_delay = calculate_monitor_delay( + sync_file=sync_file, equipment=meta.behavior_metadata.equipment) + date_of_acquisition = DateOfAcquisitionOphys.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + behavior_session = BehaviorSession.from_lims( + lims_db=lims_db, + behavior_session_id=behavior_session_id.value, + stimulus_timestamps=stimulus_timestamps, + monitor_delay=monitor_delay, + date_of_acquisition=date_of_acquisition + ) + if is_multiplane_session: + ophys_timestamps = OphysTimestampsMultiplane.from_sync_file( + sync_file=sync_file, + group_count=meta.ophys_metadata.imaging_plane_group_count, + plane_group=meta.ophys_metadata.imaging_plane_group + ) + else: + ophys_timestamps = OphysTimestamps.from_sync_file( + sync_file=sync_file) + + projections = Projections.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + cell_specimens = CellSpecimens.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db, + ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=projections.max_projection.spacing, + events_params=EventsParams( + filter_scale=events_filter_scale, + filter_n_time_steps=events_filter_n_time_steps), + exclude_invalid_rois=exclude_invalid_rois + ) + motion_correction = _get_motion_correction() + if skip_eye_tracking: + eye_tracking_table = None + eye_tracking_rig_geometry = None + else: + eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file) + eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + + return BehaviorOphysExperiment( + behavior_session=behavior_session, + cell_specimens=cell_specimens, + ophys_timestamps=ophys_timestamps, + metadata=meta, + projections=projections, + motion_correction=motion_correction, + eye_tracking_table=eye_tracking_table, + eye_tracking_rig_geometry=eye_tracking_rig_geometry, + date_of_acquisition=date_of_acquisition + ) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile, eye_tracking_z_threshold: float = 3.0, eye_tracking_dilation_frames: int = 2, events_filter_scale: float = 2.0, - events_filter_n_time_steps: int = 20): + events_filter_n_time_steps: int = 20, + exclude_invalid_rois=True + ) -> "BehaviorOphysExperiment": """ + Parameters ---------- - api : object, optional - The backend api used by the session object to get behavior ophys - data, by default None. + nwbfile eye_tracking_z_threshold : float, optional The z-threshold when determining which frames likely contain outliers for eye or pupil areas. Influences which frames @@ -42,72 +250,161 @@ def __init__(self, api=None, by default 2.0 events_filter_n_time_steps : int, optional Number of time steps to use for convolution of ophys events + exclude_invalid_rois + Whether to exclude invalid rois """ - - BehaviorSession.__init__(self, api=api) - ParamsMixin.__init__(self, ignore={'api'}) - - # eye_tracking processing params - self._eye_tracking_z_threshold = eye_tracking_z_threshold - self._eye_tracking_dilation_frames = eye_tracking_dilation_frames - - # events processing params - self._events_filter_scale = events_filter_scale - self._events_filter_n_time_steps = events_filter_n_time_steps - - # LazyProperty constructor provided by LazyPropertyMixin - LazyProperty = self.LazyProperty - - # Initialize attributes to be lazily evaluated - self._ophys_session_id = LazyProperty( - self.api.get_ophys_session_id) - self._ophys_experiment_id = LazyProperty( - self.api.get_ophys_experiment_id) - self._max_projection = LazyProperty(self.api.get_max_projection, - wrappers=[ImageApi.deserialize]) - self._average_projection = LazyProperty( - self.api.get_average_projection, wrappers=[ImageApi.deserialize]) - self._ophys_timestamps = LazyProperty(self.api.get_ophys_timestamps, - settable=True) - self._dff_traces = LazyProperty(self.api.get_dff_traces, settable=True) - self._events = LazyProperty(self.api.get_events, settable=True) - self._cell_specimen_table = LazyProperty( - self.api.get_cell_specimen_table, settable=True) - self._corrected_fluorescence_traces = LazyProperty( - self.api.get_corrected_fluorescence_traces, settable=True) - self._motion_correction = LazyProperty(self.api.get_motion_correction, - settable=True) - self._segmentation_mask_image = LazyProperty( - self.get_segmentation_mask_image) - self._eye_tracking = LazyProperty( - self.api.get_eye_tracking, settable=True, - z_threshold=self._eye_tracking_z_threshold, - dilation_frames=self._eye_tracking_dilation_frames) - self._eye_tracking_rig_geometry = LazyProperty( - self.api.get_eye_tracking_rig_geometry) - - # ==================== class and utility methods ====================== + def _is_multi_plane_session(): + imaging_plane_group_meta = ImagingPlaneGroup.from_nwb( + nwbfile=nwbfile) + return cls._is_multi_plane_session( + imaging_plane_group_meta=imaging_plane_group_meta) + + behavior_session = BehaviorSession.from_nwb(nwbfile=nwbfile) + projections = Projections.from_nwb(nwbfile=nwbfile) + cell_specimens = CellSpecimens.from_nwb( + nwbfile=nwbfile, + segmentation_mask_image_spacing=projections.max_projection.spacing, + events_params=EventsParams( + filter_scale=events_filter_scale, + filter_n_time_steps=events_filter_n_time_steps + ), + exclude_invalid_rois=exclude_invalid_rois + ) + eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_nwb( + nwbfile=nwbfile) + eye_tracking_table = EyeTrackingTable.from_nwb( + nwbfile=nwbfile, z_threshold=eye_tracking_z_threshold, + dilation_frames=eye_tracking_dilation_frames) + motion_correction = MotionCorrection.from_nwb(nwbfile=nwbfile) + is_multiplane_session = _is_multi_plane_session() + metadata = BehaviorOphysMetadata.from_nwb( + nwbfile=nwbfile, is_multiplane=is_multiplane_session) + if is_multiplane_session: + ophys_timestamps = OphysTimestampsMultiplane.from_nwb( + nwbfile=nwbfile) + else: + ophys_timestamps = OphysTimestamps.from_nwb(nwbfile=nwbfile) + date_of_acquisition = DateOfAcquisitionOphys.from_nwb(nwbfile=nwbfile) + + return BehaviorOphysExperiment( + behavior_session=behavior_session, + cell_specimens=cell_specimens, + eye_tracking_rig_geometry=eye_tracking_rig_geometry, + eye_tracking_table=eye_tracking_table, + motion_correction=motion_correction, + metadata=metadata, + ophys_timestamps=ophys_timestamps, + projections=projections, + date_of_acquisition=date_of_acquisition + ) @classmethod - def from_lims(cls, ophys_experiment_id: int, + def from_json(cls, + session_data: dict, eye_tracking_z_threshold: float = 3.0, - eye_tracking_dilation_frames: int = 2 - ) -> "BehaviorOphysExperiment": - return cls(api=BehaviorOphysLimsApi(ophys_experiment_id), - eye_tracking_z_threshold=eye_tracking_z_threshold, - eye_tracking_dilation_frames=eye_tracking_dilation_frames) + eye_tracking_dilation_frames: int = 2, + events_filter_scale: float = 2.0, + events_filter_n_time_steps: int = 20, + exclude_invalid_rois=True, + skip_eye_tracking=False) -> \ + "BehaviorOphysExperiment": + """ - @classmethod - def from_nwb_path( - cls, nwb_path: str, **api_kwargs: Any) -> "BehaviorOphysExperiment": # noqa: E501 - api_kwargs["filter_invalid_rois"] = api_kwargs.get( - "filter_invalid_rois", True) - return cls(api=BehaviorOphysNwbApi.from_path( - path=nwb_path, **api_kwargs)) + Parameters + ---------- + session_data + eye_tracking_z_threshold + See `BehaviorOphysExperiment.from_nwb` + eye_tracking_dilation_frames + See `BehaviorOphysExperiment.from_nwb` + events_filter_scale + See `BehaviorOphysExperiment.from_nwb` + events_filter_n_time_steps + See `BehaviorOphysExperiment.from_nwb` + exclude_invalid_rois + Whether to exclude invalid rois + skip_eye_tracking + Used to skip returning eye tracking data + + """ + def _is_multi_plane_session(): + imaging_plane_group_meta = ImagingPlaneGroup.from_json( + dict_repr=session_data) + return cls._is_multi_plane_session( + imaging_plane_group_meta=imaging_plane_group_meta) + + def _get_motion_correction(): + rigid_motion_transform_file = RigidMotionTransformFile.from_json( + dict_repr=session_data) + return MotionCorrection.from_data_file( + rigid_motion_transform_file=rigid_motion_transform_file) + + def _get_eye_tracking_table(sync_file: SyncFile): + eye_tracking_file = EyeTrackingFile.from_json( + dict_repr=session_data) + eye_tracking_table = EyeTrackingTable.from_data_file( + data_file=eye_tracking_file, + sync_file=sync_file, + z_threshold=eye_tracking_z_threshold, + dilation_frames=eye_tracking_dilation_frames + ) + return eye_tracking_table + + sync_file = SyncFile.from_json(dict_repr=session_data) + is_multiplane_session = _is_multi_plane_session() + meta = BehaviorOphysMetadata.from_json( + dict_repr=session_data, is_multiplane=is_multiplane_session) + monitor_delay = calculate_monitor_delay( + sync_file=sync_file, equipment=meta.behavior_metadata.equipment) + behavior_session = BehaviorSession.from_json( + session_data=session_data, + monitor_delay=monitor_delay + ) + + if is_multiplane_session: + ophys_timestamps = OphysTimestampsMultiplane.from_sync_file( + sync_file=sync_file, + group_count=meta.ophys_metadata.imaging_plane_group_count, + plane_group=meta.ophys_metadata.imaging_plane_group + ) + else: + ophys_timestamps = OphysTimestamps.from_sync_file( + sync_file=sync_file) + + projections = Projections.from_json(dict_repr=session_data) + cell_specimens = CellSpecimens.from_json( + dict_repr=session_data, + ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=projections.max_projection.spacing, + events_params=EventsParams( + filter_scale=events_filter_scale, + filter_n_time_steps=events_filter_n_time_steps), + exclude_invalid_rois=exclude_invalid_rois + ) + motion_correction = _get_motion_correction() + if skip_eye_tracking: + eye_tracking_table = None + eye_tracking_rig_geometry = None + else: + eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file) + eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_json( + dict_repr=session_data) + + return BehaviorOphysExperiment( + behavior_session=behavior_session, + cell_specimens=cell_specimens, + ophys_timestamps=ophys_timestamps, + metadata=meta, + projections=projections, + motion_correction=motion_correction, + eye_tracking_table=eye_tracking_table, + eye_tracking_rig_geometry=eye_tracking_rig_geometry, + date_of_acquisition=behavior_session._date_of_acquisition + ) # ========================= 'get' methods ========================== - def get_segmentation_mask_image(self): + def get_segmentation_mask_image(self) -> Image: """a 2D binary image of all valid cell masks Returns @@ -116,16 +413,7 @@ def get_segmentation_mask_image(self): array-like interface to segmentation_mask image data and metadata """ - mask_data = np.sum(self.roi_masks['roi_mask'].values).astype(int) - - max_projection_image = self.max_projection - - mask_image = Image( - data=mask_data, - spacing=max_projection_image.spacing, - unit=max_projection_image.unit - ) - return mask_image + return self._cell_specimens.segmentation_mask_image @legacy('Consider using "dff_traces" instead.') def get_dff_traces(self, cell_specimen_ids=None): @@ -159,47 +447,81 @@ def get_cell_specimen_ids(self): f"for {self.ophys_experiment_id}") return cell_specimen_ids - # ====================== properties and setters ======================== + # ====================== properties ======================== @property def ophys_experiment_id(self) -> int: """Unique identifier for this experimental session. :rtype: int """ - return self._ophys_experiment_id + return self._metadata.ophys_metadata.ophys_experiment_id @property def ophys_session_id(self) -> int: """Unique identifier for this ophys session. :rtype: int """ - return self._ophys_session_id + return self._metadata.ophys_metadata.ophys_session_id + + @property + def metadata(self): + behavior_meta = super()._get_metadata( + behavior_metadata=self._metadata.behavior_metadata) + ophys_meta = { + 'indicator': self._cell_specimens.meta.imaging_plane.indicator, + 'emission_lambda': self._cell_specimens.meta.emission_lambda, + 'excitation_lambda': + self._cell_specimens.meta.imaging_plane.excitation_lambda, + 'experiment_container_id': + self._metadata.ophys_metadata.experiment_container_id, + 'field_of_view_height': + self._metadata.ophys_metadata.field_of_view_shape.height, + 'field_of_view_width': + self._metadata.ophys_metadata.field_of_view_shape.width, + 'imaging_depth': self._metadata.ophys_metadata.imaging_depth, + 'imaging_plane_group': + self._metadata.ophys_metadata.imaging_plane_group + if isinstance(self._metadata.ophys_metadata, + MultiplaneMetadata) else None, + 'imaging_plane_group_count': + self._metadata.ophys_metadata.imaging_plane_group_count + if isinstance(self._metadata.ophys_metadata, + MultiplaneMetadata) else 0, + 'ophys_experiment_id': + self._metadata.ophys_metadata.ophys_experiment_id, + 'ophys_frame_rate': + self._cell_specimens.meta.imaging_plane.ophys_frame_rate, + 'ophys_session_id': self._metadata.ophys_metadata.ophys_session_id, + 'project_code': self._metadata.ophys_metadata.project_code, + 'targeted_structure': + self._cell_specimens.meta.imaging_plane.targeted_structure + } + return { + **behavior_meta, + **ophys_meta + } @property def max_projection(self) -> Image: """2D max projection image. :rtype: allensdk.brain_observatory.behavior.image_api.Image """ - return self._max_projection + return self._projections.max_projection @property - def average_projection(self) -> pd.DataFrame: + def average_projection(self) -> Image: """2D image of the microscope field of view, averaged across the experiment - :rtype: pandas.DataFrame + :rtype: allensdk.brain_observatory.behavior.image_api.Image """ - return self._average_projection + return self._projections.avg_projection @property def ophys_timestamps(self) -> np.ndarray: """Timestamps associated with frames captured by the microscope :rtype: numpy.ndarray """ - return self._ophys_timestamps - - @ophys_timestamps.setter - def ophys_timestamps(self, value): - self._ophys_timestamps = value + return self._ophys_timestamps.value @property def dff_traces(self) -> pd.DataFrame: @@ -223,11 +545,7 @@ def dff_traces(self) -> pd.DataFrame: (arbitrary units) """ - return self._dff_traces - - @dff_traces.setter - def dff_traces(self, value): - self._dff_traces = value + return self._cell_specimens.dff_traces @property def events(self) -> pd.DataFrame: @@ -261,20 +579,7 @@ def events(self) -> pd.DataFrame: estimated noise standard deviation for the events trace """ - params = {'events_filter_scale', 'events_filter_n_time_steps'} - - if self.needs_data_refresh(params): - self._events = self.LazyProperty( - self.api.get_events, - filter_scale=self._events_filter_scale, - filter_n_time_steps=self._events_filter_n_time_steps) - self.clear_updated_params(params) - - return self._events - - @events.setter - def events(self, value): - self._events = value + return self._cell_specimens.events @property def cell_specimen_table(self) -> pd.DataFrame: @@ -320,11 +625,7 @@ def cell_specimen_table(self) -> pd.DataFrame: y position of ROI in field of view in pixels (top left corner) """ - return self._cell_specimen_table - - @cell_specimen_table.setter - def cell_specimen_table(self, value): - self._cell_specimen_table = value + return self._cell_specimens.table @property def corrected_fluorescence_traces(self) -> pd.DataFrame: @@ -349,11 +650,7 @@ def corrected_fluorescence_traces(self) -> pd.DataFrame: fluorescence values (arbitrary units) """ - return self._corrected_fluorescence_traces - - @corrected_fluorescence_traces.setter - def corrected_fluorescence_traces(self, value): - self._corrected_fluorescence_traces = value + return self._cell_specimens.corrected_fluorescence_traces @property def motion_correction(self) -> pd.DataFrame: @@ -369,24 +666,14 @@ def motion_correction(self) -> pd.DataFrame: y: (int) frame shift along y axis """ - return self._motion_correction - - @motion_correction.setter - def motion_correction(self, value): - self._motion_correction = value + return self._motion_correction.value @property def segmentation_mask_image(self) -> Image: """A 2d binary image of all valid cell masks :rtype: allensdk.brain_observatory.behavior.image_api.Image """ - if self._segmentation_mask_image is None: - self._segmentation_mask_image = self.get_segmentation_mask_image() - return self._segmentation_mask_image - - @segmentation_mask_image.setter - def segmentation_mask_image(self, value): - self._segmentation_mask_image = value + return self._cell_specimens.segmentation_mask_image @property def eye_tracking(self) -> pd.DataFrame: @@ -424,20 +711,7 @@ def eye_tracking(self) -> pd.DataFrame: :rtype: pandas.DataFrame """ - params = {'eye_tracking_dilation_frames', 'eye_tracking_z_threshold'} - - if self.needs_data_refresh(params): - self._eye_tracking = self.LazyProperty( - self.api.get_eye_tracking, - z_threshold=self._eye_tracking_z_threshold, - dilation_frames=self._eye_tracking_dilation_frames) - self.clear_updated_params(params) - - return self._eye_tracking - - @eye_tracking.setter - def eye_tracking(self, value): - self._eye_tracking = value + return self._eye_tracking.value @property def eye_tracking_rig_geometry(self) -> dict: @@ -455,8 +729,27 @@ def eye_tracking_rig_geometry(self) -> dict: monitor_position_mm (array of float) monitor_rotation_deg (array of float) """ - return self.api.get_eye_tracking_rig_geometry() + return self._eye_tracking_rig_geometry.to_dict()['rig_geometry'] @property def roi_masks(self) -> pd.DataFrame: return self.cell_specimen_table[['cell_roi_id', 'roi_mask']] + + def _get_identifier(self) -> str: + return str(self.ophys_experiment_id) + + @staticmethod + def _is_multi_plane_session( + imaging_plane_group_meta: ImagingPlaneGroup) -> bool: + """Returns whether this experiment is part of a multiplane session""" + return imaging_plane_group_meta is not None and \ + imaging_plane_group_meta.plane_group_count > 1 + + def _get_session_type(self) -> str: + return self._metadata.behavior_metadata.session_type + + @staticmethod + def _get_keywords(): + """Keywords for NWB file""" + return ["2-photon", "calcium imaging", "visual cortex", + "behavior", "task"] diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py index 360e901ff..0979334db 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py @@ -271,7 +271,8 @@ def get_behavior_ophys_experiment(self, ophys_experiment_id: int f" there are {row.shape[0]} entries.") file_id = str(int(row[self.cache.file_id_column])) data_path = self._get_data_path(file_id=file_id) - return BehaviorOphysExperiment.from_nwb_path(str(data_path)) + return BehaviorOphysExperiment.from_nwb_path( + str(data_path)) def _get_ophys_session_table(self): session_table_path = self._get_metadata_path( diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py index fbbd5ba84..99b8b02e1 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py @@ -7,8 +7,6 @@ BehaviorSession) from allensdk.brain_observatory.behavior.behavior_ophys_experiment import ( BehaviorOphysExperiment) -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsApi, BehaviorOphysLimsApi) from allensdk.internal.api import db_connection_creator from allensdk.brain_observatory.ecephys.ecephys_project_api.http_engine \ import (HttpEngine) @@ -327,8 +325,8 @@ def get_behavior_ophys_experiment(self, ophys_experiment_id: int :type ophys_experiment_id: int :rtype: BehaviorOphysExperiment """ - return BehaviorOphysExperiment( - BehaviorOphysLimsApi(ophys_experiment_id)) + return BehaviorOphysExperiment.from_lims( + ophys_experiment_id=ophys_experiment_id) def _get_ophys_experiment_table(self) -> pd.DataFrame: """ @@ -484,7 +482,8 @@ def get_behavior_session( :type behavior_session_id: int :rtype: BehaviorSession """ - return BehaviorSession(BehaviorLimsApi(behavior_session_id)) + return BehaviorSession.from_lims( + behavior_session_id=behavior_session_id) def get_ophys_experiment_table( self, diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py b/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py index eaeedfed7..22d1c026c 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py @@ -13,10 +13,16 @@ from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ .project_table import \ ProjectTable -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import BehaviorProjectLimsApi # noqa: E501 +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.full_genotype import \ + FullGenotype + +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.reporter_line import \ + ReporterLine + class SessionsTable(ProjectTable): """Class for storing and manipulating project-level data @@ -45,11 +51,11 @@ def __init__( def postprocess_additional(self): self._df['reporter_line'] = self._df['reporter_line'].apply( - BehaviorMetadata.parse_reporter_line) + ReporterLine.parse) self._df['cre_line'] = self._df['full_genotype'].apply( - BehaviorMetadata.parse_cre_line) + lambda x: FullGenotype(x).parse_cre_line()) self._df['indicator'] = self._df['reporter_line'].apply( - BehaviorMetadata.parse_indicator) + lambda x: ReporterLine(x).parse_indicator()) self.__add_session_number() diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py index b6d3c867b..36f875eed 100644 --- a/allensdk/brain_observatory/behavior/behavior_session.py +++ b/allensdk/brain_observatory/behavior/behavior_session.py @@ -1,107 +1,320 @@ -from typing import Any, Optional, List, Dict, Type, Tuple -import logging +import datetime +from typing import Any, List, Dict, Optional +import pynwb import pandas as pd import numpy as np -import inspect - -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.core.lazy_property import LazyPropertyMixin -from allensdk.brain_observatory.session_api_utils import ParamsMixin -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsApi, BehaviorNwbApi) -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - session_base.behavior_base import BehaviorBase +import pytz + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.licks import Licks +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.behavior_metadata import \ + BehaviorMetadata, get_expt_description +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.date_of_acquisition import \ + DateOfAcquisition +from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards +from allensdk.brain_observatory.behavior.data_objects.stimuli.stimuli import \ + Stimuli +from allensdk.brain_observatory.behavior.data_objects.task_parameters import \ + TaskParameters +from allensdk.brain_observatory.behavior.data_objects.trials.trial_table \ + import \ + TrialTable from allensdk.brain_observatory.behavior.trials_processing import ( construct_rolling_performance_df, calculate_reward_rate_fix_nans) +from allensdk.brain_observatory.behavior.data_objects import ( + BehaviorSessionId, StimulusTimestamps, RunningSpeed, RunningAcquisition, + DataObject +) + +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator, PostgresQueryMixin + + +class BehaviorSession(DataObject, LimsReadableInterface, + NwbReadableInterface, + JsonReadableInterface, NwbWritableInterface): + """Represents data from a single Visual Behavior behavior session. + Initialize by using class methods `from_lims` or `from_nwb_path`. + """ + def __init__( + self, + behavior_session_id: BehaviorSessionId, + stimulus_timestamps: StimulusTimestamps, + running_acquisition: RunningAcquisition, + raw_running_speed: RunningSpeed, + running_speed: RunningSpeed, + licks: Licks, + rewards: Rewards, + stimuli: Stimuli, + task_parameters: TaskParameters, + trials: TrialTable, + metadata: BehaviorMetadata, + date_of_acquisition: DateOfAcquisition + ): + super().__init__(name='behavior_session', value=self) + + self._behavior_session_id = behavior_session_id + self._licks = licks + self._rewards = rewards + self._running_acquisition = running_acquisition + self._running_speed = running_speed + self._raw_running_speed = raw_running_speed + self._stimuli = stimuli + self._stimulus_timestamps = stimulus_timestamps + self._task_parameters = task_parameters + self._trials = trials + self._metadata = metadata + self._date_of_acquisition = date_of_acquisition + # ==================== class and utility methods ====================== -BehaviorDataApi = Type[BehaviorBase] - - -class BehaviorSession(LazyPropertyMixin): - def __init__(self, api: Optional[BehaviorDataApi] = None): - self.api = api - - # LazyProperty constructor provided by LazyPropertyMixin - LazyProperty = self.LazyProperty - - # Initialize attributes to be lazily evaluated - self._behavior_session_id = LazyProperty( - self.api.get_behavior_session_id) - self._licks = LazyProperty(self.api.get_licks, settable=True) - self._rewards = LazyProperty(self.api.get_rewards, settable=True) - self._running_speed = LazyProperty(self.api.get_running_speed, - settable=True, lowpass=True) - self._raw_running_speed = LazyProperty(self.api.get_running_speed, - settable=True, lowpass=False) - - def stimulus_getter(): - _df = self.api.get_stimulus_presentations() - _df.drop(['index'], axis=1, errors='ignore') - _df = _df[['start_time', 'stop_time', - 'duration', - 'image_name', 'image_index', - 'is_change', 'omitted', - 'start_frame', 'end_frame', - 'image_set']] - return _df - self._stimulus_presentations = LazyProperty( - stimulus_getter, settable=True) - - self._stimulus_templates = LazyProperty( - self.api.get_stimulus_templates, settable=True) - self._stimulus_timestamps = LazyProperty( - self.api.get_stimulus_timestamps, settable=True) - self._task_parameters = LazyProperty(self.api.get_task_parameters, - settable=True) - - def trials_getter(): - _df = self.api.get_trials() - _df = _df[['initial_image_name', 'change_image_name', - 'stimulus_change', 'change_time', - 'go', 'catch', 'lick_times', 'response_time', - 'response_latency', 'reward_time', 'reward_volume', - 'hit', 'false_alarm', 'miss', 'correct_reject', - 'aborted', 'auto_rewarded', 'change_frame', - 'start_time', 'stop_time', 'trial_length']] - return _df - self._trials = LazyProperty(trials_getter, settable=True) - - self._metadata = LazyProperty(self.api.get_metadata, settable=True) + @classmethod + def from_json(cls, + session_data: dict, + monitor_delay: Optional[float] = None) \ + -> "BehaviorSession": + """ - # ==================== class and utility methods ====================== + Parameters + ---------- + session_data + Dict of input data necessary to construct a session + monitor_delay + Monitor delay. If not provided, will use an estimate. + To provide this value, see for example + allensdk.brain_observatory.behavior.data_objects.stimuli.util. + calculate_monitor_delay + + Returns + ------- + `BehaviorSession` instance + + """ + behavior_session_id = BehaviorSessionId.from_json( + dict_repr=session_data) + stimulus_file = StimulusFile.from_json(dict_repr=session_data) + stimulus_timestamps = StimulusTimestamps.from_json( + dict_repr=session_data) + running_acquisition = RunningAcquisition.from_json( + dict_repr=session_data) + raw_running_speed = RunningSpeed.from_json( + dict_repr=session_data, filtered=False + ) + running_speed = RunningSpeed.from_json(dict_repr=session_data) + metadata = BehaviorMetadata.from_json(dict_repr=session_data) + + if monitor_delay is None: + monitor_delay = cls._get_monitor_delay() + + licks, rewards, stimuli, task_parameters, trials = \ + cls._read_data_from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + trial_monitor_delay=monitor_delay + ) + date_of_acquisition = DateOfAcquisition.from_json( + dict_repr=session_data)\ + .validate( + stimulus_file=stimulus_file, + behavior_session_id=behavior_session_id.value) + + return BehaviorSession( + behavior_session_id=behavior_session_id, + stimulus_timestamps=stimulus_timestamps, + running_acquisition=running_acquisition, + raw_running_speed=raw_running_speed, + running_speed=running_speed, + metadata=metadata, + licks=licks, + rewards=rewards, + stimuli=stimuli, + task_parameters=task_parameters, + trials=trials, + date_of_acquisition=date_of_acquisition + ) @classmethod - def from_lims(cls, behavior_session_id: int) -> "BehaviorSession": - return cls(api=BehaviorLimsApi(behavior_session_id)) + def from_lims(cls, behavior_session_id: int, + lims_db: Optional[PostgresQueryMixin] = None, + stimulus_timestamps: Optional[StimulusTimestamps] = None, + monitor_delay: Optional[float] = None, + date_of_acquisition: Optional[DateOfAcquisition] = None) \ + -> "BehaviorSession": + """ + + Parameters + ---------- + behavior_session_id + Behavior session id + lims_db + Database connection. If not provided will create a new one. + stimulus_timestamps + Stimulus timestamps. If not provided, will calculate stimulus + timestamps from stimulus file. + monitor_delay + Monitor delay. If not provided, will use an estimate. + To provide this value, see for example + allensdk.brain_observatory.behavior.data_objects.stimuli.util. + calculate_monitor_delay + date_of_acquisition + Date of acquisition. If not provided, will read from + behavior_sessions table. + Returns + ------- + `BehaviorSession` instance + """ + if lims_db is None: + lims_db = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP + ) + + behavior_session_id = BehaviorSessionId(behavior_session_id) + stimulus_file = StimulusFile.from_lims( + db=lims_db, behavior_session_id=behavior_session_id.value) + if stimulus_timestamps is None: + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + running_acquisition = RunningAcquisition.from_lims( + lims_db, behavior_session_id.value + ) + raw_running_speed = RunningSpeed.from_lims( + lims_db, behavior_session_id.value, filtered=False, + stimulus_timestamps=stimulus_timestamps + ) + running_speed = RunningSpeed.from_lims( + lims_db, behavior_session_id.value, + stimulus_timestamps=stimulus_timestamps + ) + behavior_metadata = BehaviorMetadata.from_lims( + behavior_session_id=behavior_session_id, lims_db=lims_db + ) + + if monitor_delay is None: + monitor_delay = cls._get_monitor_delay() + + licks, rewards, stimuli, task_parameters, trials = \ + cls._read_data_from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + trial_monitor_delay=monitor_delay + ) + if date_of_acquisition is None: + date_of_acquisition = DateOfAcquisition.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + date_of_acquisition = date_of_acquisition.validate( + stimulus_file=stimulus_file, + behavior_session_id=behavior_session_id.value) + + return BehaviorSession( + behavior_session_id=behavior_session_id, + stimulus_timestamps=stimulus_timestamps, + running_acquisition=running_acquisition, + raw_running_speed=raw_running_speed, + running_speed=running_speed, + metadata=behavior_metadata, + licks=licks, + rewards=rewards, + stimuli=stimuli, + task_parameters=task_parameters, + trials=trials, + date_of_acquisition=date_of_acquisition + ) @classmethod - def from_nwb_path( - cls, nwb_path: str, **api_kwargs: Any) -> "BehaviorSession": - return cls(api=BehaviorNwbApi.from_path(path=nwb_path, **api_kwargs)) - - def cache_clear(self) -> None: - """Convenience method to clear the api cache, if applicable.""" - try: - self.api.cache_clear() - except AttributeError: - logging.getLogger("BehaviorSession").warning( - "Attempted to clear API cache, but method `cache_clear`" - f" does not exist on {self.api.__class__.__name__}") - - def list_api_methods(self) -> List[Tuple[str, str]]: - """Convenience method to expose list of API `get` methods. These - methods can be accessed by referencing the API used to initialize this - BehaviorSession via its `api` instance attribute. - :rtype: list of tuples, where the first value in the tuple is the - method name, and the second value is the method docstring. + def from_nwb(cls, nwbfile: NWBFile, **kwargs) -> "BehaviorSession": + behavior_session_id = BehaviorSessionId.from_nwb(nwbfile) + stimulus_timestamps = StimulusTimestamps.from_nwb(nwbfile) + running_acquisition = RunningAcquisition.from_nwb(nwbfile) + raw_running_speed = RunningSpeed.from_nwb(nwbfile, filtered=False) + running_speed = RunningSpeed.from_nwb(nwbfile) + metadata = BehaviorMetadata.from_nwb(nwbfile) + licks = Licks.from_nwb(nwbfile=nwbfile) + rewards = Rewards.from_nwb(nwbfile=nwbfile) + stimuli = Stimuli.from_nwb(nwbfile=nwbfile) + task_parameters = TaskParameters.from_nwb(nwbfile=nwbfile) + trials = TrialTable.from_nwb(nwbfile=nwbfile) + date_of_acquisition = DateOfAcquisition.from_nwb(nwbfile=nwbfile) + + return BehaviorSession( + behavior_session_id=behavior_session_id, + stimulus_timestamps=stimulus_timestamps, + running_acquisition=running_acquisition, + raw_running_speed=raw_running_speed, + running_speed=running_speed, + metadata=metadata, + licks=licks, + rewards=rewards, + stimuli=stimuli, + task_parameters=task_parameters, + trials=trials, + date_of_acquisition=date_of_acquisition + ) + + @classmethod + def from_nwb_path(cls, nwb_path: str, **kwargs) -> "BehaviorSession": + """ + + Parameters + ---------- + nwb_path + Path to nwb file + kwargs + Kwargs to be passed to `from_nwb` + + Returns + ------- + An instantiation of a `BehaviorSession` """ - methods = [m for m in inspect.getmembers(self.api, inspect.ismethod) - if m[0].startswith("get_")] - docs = [inspect.getdoc(m[1]) or "" for m in methods] - method_names = [m[0] for m in methods] - return list(zip(method_names, docs)) + with pynwb.NWBHDF5IO(str(nwb_path), 'r') as read_io: + nwbfile = read_io.read() + return cls.from_nwb(nwbfile=nwbfile, **kwargs) + + def to_nwb(self, add_metadata=True) -> NWBFile: + """ + + Parameters + ---------- + add_metadata + Set this to False to prevent adding metadata to the nwb + instance. + """ + nwbfile = NWBFile( + session_description=self._get_session_type(), + identifier=self._get_identifier(), + session_start_time=self._date_of_acquisition.value, + file_create_date=pytz.utc.localize(datetime.datetime.now()), + institution="Allen Institute for Brain Science", + keywords=self._get_keywords(), + experiment_description=get_expt_description( + session_type=self._get_session_type()) + ) + + self._stimulus_timestamps.to_nwb(nwbfile=nwbfile) + self._running_acquisition.to_nwb(nwbfile=nwbfile) + self._raw_running_speed.to_nwb(nwbfile=nwbfile) + self._running_speed.to_nwb(nwbfile=nwbfile) + + if add_metadata: + self._metadata.to_nwb(nwbfile=nwbfile) + + self._licks.to_nwb(nwbfile=nwbfile) + self._rewards.to_nwb(nwbfile=nwbfile) + self._stimuli.to_nwb(nwbfile=nwbfile) + self._task_parameters.to_nwb(nwbfile=nwbfile) + self._trials.to_nwb(nwbfile=nwbfile) + + return nwbfile def list_data_attributes_and_methods(self) -> List[str]: """Convenience method for end-users to list attributes and methods @@ -117,16 +330,14 @@ def list_data_attributes_and_methods(self) -> List[str]: to get data. """ attrs_and_methods_to_ignore: set = { - "api", - "cache_clear", + "from_json", "from_lims", "from_nwb_path", - "LazyProperty", - "list_api_methods", "list_data_attributes_and_methods" } - attrs_and_methods_to_ignore.update(dir(ParamsMixin)) - attrs_and_methods_to_ignore.update(dir(LazyPropertyMixin)) + attrs_and_methods_to_ignore.update(dir(NwbReadableInterface)) + attrs_and_methods_to_ignore.update(dir(NwbWritableInterface)) + attrs_and_methods_to_ignore.update(dir(DataObject)) class_dir = dir(self) attrs_and_methods = [ r for r in class_dir @@ -326,14 +537,14 @@ def get_performance_metrics( return performance_metrics - # ====================== properties and setters ======================== + # ====================== properties ======================== @property def behavior_session_id(self) -> int: """Unique identifier for a behavioral session. :rtype: int """ - return self._behavior_session_id + return self._behavior_session_id.value @property def licks(self) -> pd.DataFrame: @@ -356,11 +567,7 @@ def licks(self) -> pd.DataFrame: frame of lick """ - return self._licks - - @licks.setter - def licks(self, value): - self._licks = value + return self._licks.value @property def rewards(self) -> pd.DataFrame: @@ -390,11 +597,7 @@ def rewards(self) -> pd.DataFrame: throughout as needed """ - return self._rewards - - @rewards.setter - def rewards(self, value): - self._rewards = value + return self._rewards.value @property def running_speed(self) -> pd.DataFrame: @@ -417,11 +620,7 @@ def running_speed(self) -> pd.DataFrame: speed: (float) speed in cm/sec """ - return self._running_speed - - @running_speed.setter - def running_speed(self, value): - self._running_speed = value + return self._running_speed.value @property def raw_running_speed(self) -> pd.DataFrame: @@ -442,11 +641,7 @@ def raw_running_speed(self) -> pd.DataFrame: speed: (float) speed in cm/sec """ - return self._raw_running_speed - - @raw_running_speed.setter - def raw_running_speed(self, value): - self._raw_running_speed = value + return self._raw_running_speed.value @property def stimulus_presentations(self) -> pd.DataFrame: @@ -465,35 +660,29 @@ def stimulus_presentations(self) -> pd.DataFrame: stimulus_presentations_id [index]: (int) identifier for a stimulus presentation (presentation of an image) - start_time: (float) - image presentation start time in seconds - stop_time: (float) - image presentation end time in seconds duration: (float) duration of an image presentation (flash) in seconds (stop_time - start_time). NaN if omitted - image_name: (str) + end_frame: (float) + image presentation end frame image_index: (int) image index (0-7) for a given session, corresponding to each image name - is_change: (bool) - True if the presentation represents a change - in image + image_set: (string) + image set for this behavior session + index: (int) + an index assigned to each stimulus presentation omitted: (bool) True if no image was shown for this stimulus presentation start_frame: (int) image presentation start frame - end_frame: (float) - image presentation end frame - image_set: (string) - image set for this behavior session + start_time: (float) + image presentation start time in seconds + stop_time: (float) + image presentation end time in seconds """ - return self._stimulus_presentations - - @stimulus_presentations.setter - def stimulus_presentations(self, value): - self._stimulus_presentations = value + return self._stimuli.presentations.value @property def stimulus_templates(self) -> pd.DataFrame: @@ -515,11 +704,7 @@ def stimulus_templates(self) -> pd.DataFrame: image array of warped stimulus image """ - return self._stimulus_templates.to_dataframe() - - @stimulus_templates.setter - def stimulus_templates(self, value): - self._stimulus_templates = value + return self._stimuli.templates.value.to_dataframe() @property def stimulus_timestamps(self) -> np.ndarray: @@ -537,11 +722,7 @@ def stimulus_timestamps(self) -> np.ndarray: np.ndarray Timestamps associated with stimulus presentations on the monitor """ - return self._stimulus_timestamps - - @stimulus_timestamps.setter - def stimulus_timestamps(self, value): - self._stimulus_timestamps = value + return self._stimulus_timestamps.value @property def task_parameters(self) -> dict: @@ -589,11 +770,7 @@ def task_parameters(self) -> dict: Stimulus type ('gratings' or 'images'). """ - return self._task_parameters - - @task_parameters.setter - def task_parameters(self, value): - self._task_parameters = value + return self._task_parameters.to_dict()['task_parameters'] @property def trials(self) -> pd.DataFrame: @@ -609,29 +786,9 @@ def trials(self) -> pd.DataFrame: dataframe columns: trials_id: (int) trial identifier - initial_image_name: (string) - name of image presented at start of trial - change_image_name: (string) - name of image that is changed to at the change time, - on go trials - stimulus_change: (bool) - True if an image change occurs during the trial - (if the trial was both a 'go' trial and the trial - was not aborted) - change_time: (float) - go: (bool) - Trial type. True if there was a change in stimulus - image identity on this trial - catch: (bool) - Trial type. True if there was not a change in stimulus - identity on this trial lick_times: (array of float) array of lick times in seconds during that trial. Empty array if no licks occured during the trial. - response_time: (float) - time of first lick in trial in seconds and NaN if - trial aborted - response_latency: (float) reward_time: (NaN or float) Time the reward is delivered following a correct response or on auto rewarded trials. @@ -647,34 +804,47 @@ def trials(self) -> pd.DataFrame: miss: (bool) Behavior response type. On a go trial, mouse either does not lick at all, or licks after reward window - correct_reject: (bool) - Behavior response type. On a catch trial, mouse - either does not lick at all or licks after reward - window + stimulus_change: (bool) + True if an image change occurs during the trial + (if the trial was both a 'go' trial and the trial + was not aborted) aborted: (bool) Behavior response type. True if the mouse licks before the scheduled change time. + go: (bool) + Trial type. True if there was a change in stimulus + image identity on this trial + catch: (bool) + Trial type. True if there was not a change in stimulus + identity on this trial auto_rewarded: (bool) True if free reward was delivered for that trial. Occurs during the first 5 trials of a session and throughout as needed. - change_frame: (float) + correct_reject: (bool) + Behavior response type. On a catch trial, mouse + either does not lick at all or licks after reward + window start_time: (float) start time of the trial in seconds stop_time: (float) end time of the trial in seconds trial_length: (float) duration of trial in seconds (stop_time -start_time) + response_time: (float) + time of first lick in trial in seconds and NaN if + trial aborted + initial_image_name: (string) + name of image presented at start of trial + change_image_name: (string) + name of image that is changed to at the change time, + on go trials """ - return self._trials - - @trials.setter - def trials(self, value): - self._trials = value + return self._trials.value @property def metadata(self) -> Dict[str, Any]: - """metadata for a give session + """metadata for a given session Returns ------- @@ -711,14 +881,66 @@ def metadata(self) -> Dict[str, Any]: frame rate (Hz) at which the visual stimulus is displayed """ - if isinstance(self._metadata, BehaviorMetadata): - metadata = self._metadata.to_dict() - else: - # NWB API returns as dict - metadata = self._metadata + return self._get_metadata(behavior_metadata=self._metadata) + + @classmethod + def _read_data_from_stimulus_file( + cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps, + trial_monitor_delay: float): + """Helper method to read data from stimulus file""" + licks = Licks.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps) + rewards = Rewards.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps) + stimuli = Stimuli.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps) + task_parameters = TaskParameters.from_stimulus_file( + stimulus_file=stimulus_file) + trials = TrialTable.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + licks=licks, + rewards=rewards, + monitor_delay=trial_monitor_delay + ) + return licks, rewards, stimuli, task_parameters, trials + + def _get_metadata(self, behavior_metadata: BehaviorMetadata) -> dict: + """Returns dict of metadata""" + return { + 'equipment_name': behavior_metadata.equipment.value, + 'sex': behavior_metadata.subject_metadata.sex, + 'age_in_days': behavior_metadata.subject_metadata.age_in_days, + 'stimulus_frame_rate': behavior_metadata.stimulus_frame_rate, + 'session_type': behavior_metadata.session_type, + 'date_of_acquisition': self._date_of_acquisition.value, + 'reporter_line': behavior_metadata.subject_metadata.reporter_line, + 'cre_line': behavior_metadata.subject_metadata.cre_line, + 'behavior_session_uuid': behavior_metadata.behavior_session_uuid, + 'driver_line': behavior_metadata.subject_metadata.driver_line, + 'mouse_id': behavior_metadata.subject_metadata.mouse_id, + 'full_genotype': behavior_metadata.subject_metadata.full_genotype, + 'behavior_session_id': behavior_metadata.behavior_session_id + } + + def _get_identifier(self) -> str: + return str(self._behavior_session_id) + + def _get_session_type(self) -> str: + return self._metadata.session_type - return metadata + @staticmethod + def _get_keywords(): + """Keywords for NWB file""" + return ["visual", "behavior", "task"] - @metadata.setter - def metadata(self, value): - self._metadata = value + @staticmethod + def _get_monitor_delay(): + # This is the median estimate across all rigs + # as discussed in + # https://github.com/AllenInstitute/AllenSDK/issues/1318 + return 0.02115 diff --git a/allensdk/brain_observatory/behavior/data_files/__init__.py b/allensdk/brain_observatory/behavior/data_files/__init__.py new file mode 100644 index 000000000..67cbc7175 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/__init__.py @@ -0,0 +1,3 @@ +from allensdk.brain_observatory.behavior.data_files._data_file_abc import DataFile # noqa E501, F401 +from allensdk.brain_observatory.behavior.data_files.stimulus_file import StimulusFile # noqa E501, F401 +from allensdk.brain_observatory.behavior.data_files.sync_file import SyncFile # noqa E501, F401 diff --git a/allensdk/brain_observatory/behavior/data_files/_data_file_abc.py b/allensdk/brain_observatory/behavior/data_files/_data_file_abc.py new file mode 100644 index 000000000..2461fec82 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/_data_file_abc.py @@ -0,0 +1,92 @@ +import abc +from typing import Any, Union +from pathlib import Path + + +class DataFile(abc.ABC): + """An abstract class that prototypes methods for accessing internal + data files. + + These data files contain information necessary to sucessfully instantiate + one or many `DataObject`(s). + + External users should ignore this class (and subclasses) as as they + will only ever be using `from_nwb()` and `to_nwb()` `DataObject` methods. + """ + + def __init__(self, filepath: Union[str, Path]): # pragma: no cover + self._filepath: str = str(filepath) + self._data = self.load_data(filepath=self._filepath) + + @property + def data(self) -> Any: # pragma: no cover + return self._data + + @property + def filepath(self) -> str: # pragma: no cover + return self._filepath + + @classmethod + @abc.abstractmethod + def from_json(cls, dict_repr: dict) -> "DataFile": # pragma: no cover + """Populates a DataFile from a JSON compatible dict (likely parsed by + argschema) + + Returns + ------- + DataFile: + An instantiated DataFile which has `data` and `filepath` properties + """ + # Example: + # filepath = dict_repr["my_data_file_path"] + # return cls.instantiate(filepath=filepath) + raise NotImplementedError() + + @abc.abstractmethod + def to_json(self) -> dict: # pragma: no cover + """Given an already populated DataFile, return the dict that + when used with the `from_json()` classmethod would produce the same + DataFile + + Returns + ------- + dict: + The JSON (in dict form) that would produce the DataFile. + """ + raise NotImplementedError() + + @classmethod + @abc.abstractmethod + def from_lims(cls) -> "DataFile": # pragma: no cover + """Populate a DataFile from an internal database (likely LIMS) + + Returns + ------- + DataFile: + An instantiated DataFile which has `data` and `filepath` properties + """ + # Example: + # query = """SELECT my_file FROM some_lims_table""" + # filepath = dbconn.fetchone(query, strict=True) + # return cls.instantiate(filepath=filepath) + raise NotImplementedError() + + @staticmethod + @abc.abstractmethod + def load_data(filepath: Union[str, Path]) -> Any: # pragma: no cover + """Given a filepath (that is meant to by read by the DataFile type), + load the contents of the file into a Python type. + (dict, DataFrame, list, etc...) + + Parameters + ---------- + filepath : Union[str, Path] + The filepath that the DataFile class should load. + + Returns + ------- + Any + A Python data type that has been parsed/loaded from the provided + filepath. + """ + raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/metadata/__init__.py b/allensdk/brain_observatory/behavior/data_files/avg_projection_file.py similarity index 100% rename from allensdk/brain_observatory/behavior/metadata/__init__.py rename to allensdk/brain_observatory/behavior/data_files/avg_projection_file.py diff --git a/allensdk/brain_observatory/behavior/data_files/demix_file.py b/allensdk/brain_observatory/behavior/data_files/demix_file.py new file mode 100644 index 000000000..69aa81958 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/demix_file.py @@ -0,0 +1,66 @@ +import json +from typing import Dict, Union +from pathlib import Path + +import h5py +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import DataFile + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class DemixFile(DataFile): + """A DataFile which contains methods for accessing and loading + demixed traces. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "DemixFile": + filepath = dict_repr["demix_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"demix_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "DemixFile": + query = """ + SELECT wkf.storage_directory || wkf.filename AS demix_file + FROM ophys_experiments oe + JOIN well_known_files wkf ON wkf.attachable_id = oe.id + JOIN well_known_file_types wkft + ON wkft.id = wkf.well_known_file_type_id + WHERE wkf.attachable_type = 'OphysExperiment' + AND wkft.name = 'DemixedTracesFile' + AND oe.id = {}; + """.format(ophys_experiment_id) + filepath = db.fetchone(query, strict=True) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> pd.DataFrame: + with h5py.File(filepath, 'r') as in_file: + traces = in_file['data'][()] + roi_id = in_file['roi_names'][()] + idx = pd.Index(roi_id, name='cell_roi_id', dtype=int) + return pd.DataFrame({'corrected_fluorescence': list(traces)}, + index=idx) diff --git a/allensdk/brain_observatory/behavior/data_files/dff_file.py b/allensdk/brain_observatory/behavior/data_files/dff_file.py new file mode 100644 index 000000000..f36f3962a --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/dff_file.py @@ -0,0 +1,65 @@ +import json +import numpy as np +from typing import Dict, Union +from pathlib import Path + +import h5py +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import DataFile + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class DFFFile(DataFile): + """A DataFile which contains methods for accessing and loading + DFF traces. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "DFFFile": + filepath = dict_repr["dff_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"dff_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "DFFFile": + query = """ + SELECT wkf.storage_directory || wkf.filename AS dff_file + FROM ophys_experiments oe + JOIN well_known_files wkf ON wkf.attachable_id = oe.id + JOIN well_known_file_types wkft + ON wkft.id = wkf.well_known_file_type_id + WHERE wkft.name = 'OphysDffTraceFile' + AND oe.id = {}; + """.format(ophys_experiment_id) + filepath = db.fetchone(query, strict=True) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> pd.DataFrame: + with h5py.File(filepath, 'r') as raw_file: + traces = np.asarray(raw_file['data'], dtype=np.float64) + roi_names = np.asarray(raw_file['roi_names']) + idx = pd.Index(roi_names, name='cell_roi_id', dtype=int) + return pd.DataFrame({'dff': [x for x in traces]}, index=idx) diff --git a/allensdk/brain_observatory/behavior/data_files/event_detection_file.py b/allensdk/brain_observatory/behavior/data_files/event_detection_file.py new file mode 100644 index 000000000..1956b045d --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/event_detection_file.py @@ -0,0 +1,73 @@ +import json +import numpy as np +from typing import Dict, Union, Tuple +from pathlib import Path + +import h5py +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import DataFile +from allensdk.internal.core.lims_utilities import safe_system_path + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class EventDetectionFile(DataFile): + """A DataFile which contains methods for accessing and loading + events. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "EventDetectionFile": + filepath = dict_repr["events_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"events_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "EventDetectionFile": + query = f''' + SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath + FROM ophys_experiments oe + LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id + JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id + WHERE wkft.name = 'OphysEventTraceFile' + AND oe.id = {ophys_experiment_id}; + ''' # noqa E501 + filepath = safe_system_path(db.fetchone(query, strict=True)) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> \ + Tuple[np.ndarray, pd.DataFrame]: + with h5py.File(filepath, 'r') as f: + events = f['events'][:] + lambdas = f['lambdas'][:] + noise_stds = f['noise_stds'][:] + roi_ids = f['roi_names'][:] + + df = pd.DataFrame({ + 'lambda': lambdas, + 'noise_std': noise_stds, + 'cell_roi_id': roi_ids + }) + return events, df diff --git a/allensdk/brain_observatory/behavior/data_files/eye_tracking_file.py b/allensdk/brain_observatory/behavior/data_files/eye_tracking_file.py new file mode 100644 index 000000000..e8a281b78 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/eye_tracking_file.py @@ -0,0 +1,50 @@ +from typing import Dict, Union +from pathlib import Path + +import pandas as pd + +from allensdk.brain_observatory.behavior.eye_tracking_processing import \ + load_eye_tracking_hdf +from allensdk.internal.api import PostgresQueryMixin +from allensdk.internal.core.lims_utilities import safe_system_path +from allensdk.brain_observatory.behavior.data_files import DataFile + + +class EyeTrackingFile(DataFile): + """A DataFile which contains methods for accessing and loading + eye tracking data. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + def from_json(cls, dict_repr: dict) -> "EyeTrackingFile": + filepath = dict_repr["eye_tracking_filepath"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"eye_tracking_filepath": str(self.filepath)} + + @classmethod + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "EyeTrackingFile": + query = f""" + SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file + FROM ophys_experiments oe + LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.ophys_session_id + JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id + WHERE wkf.attachable_type = 'OphysSession' + AND wkft.name = 'EyeTracking Ellipses' + AND oe.id = {ophys_experiment_id}; + """ # noqa E501 + filepath = db.fetchone(query, strict=True) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> pd.DataFrame: + filepath = safe_system_path(file_name=filepath) + # TODO move the contents of this function here + return load_eye_tracking_hdf(filepath) diff --git a/allensdk/brain_observatory/behavior/session_apis/__init__.py b/allensdk/brain_observatory/behavior/data_files/max_projection_file.py similarity index 100% rename from allensdk/brain_observatory/behavior/session_apis/__init__.py rename to allensdk/brain_observatory/behavior/data_files/max_projection_file.py diff --git a/allensdk/brain_observatory/behavior/data_files/rigid_motion_transform_file.py b/allensdk/brain_observatory/behavior/data_files/rigid_motion_transform_file.py new file mode 100644 index 000000000..4349bc27c --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/rigid_motion_transform_file.py @@ -0,0 +1,62 @@ +import json +from typing import Dict, Union +from pathlib import Path + +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import DataFile +from allensdk.internal.core.lims_utilities import safe_system_path + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class RigidMotionTransformFile(DataFile): + """A DataFile which contains methods for accessing and loading + rigid motion transform output. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "RigidMotionTransformFile": + filepath = dict_repr["rigid_motion_transform_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"rigid_motion_transform_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "RigidMotionTransformFile": + query = """ + SELECT wkf.storage_directory || wkf.filename AS transform_file + FROM ophys_experiments oe + JOIN well_known_files wkf ON wkf.attachable_id = oe.id + JOIN well_known_file_types wkft + ON wkft.id = wkf.well_known_file_type_id + WHERE wkf.attachable_type = 'OphysExperiment' + AND wkft.name = 'OphysMotionXyOffsetData' + AND oe.id = {}; + """.format(ophys_experiment_id) + filepath = safe_system_path(db.fetchone(query, strict=True)) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> pd.DataFrame: + motion_correction = pd.read_csv(filepath) + return motion_correction[['x', 'y']] diff --git a/allensdk/brain_observatory/behavior/data_files/stimulus_file.py b/allensdk/brain_observatory/behavior/data_files/stimulus_file.py new file mode 100644 index 000000000..3feeb0378 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/stimulus_file.py @@ -0,0 +1,74 @@ +import json +from typing import Dict, Union +from pathlib import Path + +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.internal.core.lims_utilities import safe_system_path +from allensdk.brain_observatory.behavior.data_files import DataFile + +# Query returns path to StimulusPickle file for given behavior session +STIMULUS_FILE_QUERY_TEMPLATE = """ + SELECT + wkf.storage_directory || wkf.filename AS stim_file + FROM + well_known_files wkf + WHERE + wkf.attachable_id = {behavior_session_id} + AND wkf.attachable_type = 'BehaviorSession' + AND wkf.well_known_file_type_id IN ( + SELECT id + FROM well_known_file_types + WHERE name = 'StimulusPickle'); +""" + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, behavior_session_id: int): + return hashkey(behavior_session_id) + + +class StimulusFile(DataFile): + """A DataFile which contains methods for accessing and loading visual + behavior stimulus *.pkl files. + + This file type contains a number of parameters collected during a behavior + session including information about stimulus presentations, rewards, + trials, and timing for all of the above. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "StimulusFile": + filepath = dict_repr["behavior_stimulus_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"behavior_stimulus_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + behavior_session_id: Union[int, str] + ) -> "StimulusFile": + query = STIMULUS_FILE_QUERY_TEMPLATE.format( + behavior_session_id=behavior_session_id + ) + filepath = db.fetchone(query, strict=True) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> dict: + filepath = safe_system_path(file_name=filepath) + return pd.read_pickle(filepath) diff --git a/allensdk/brain_observatory/behavior/data_files/sync_file.py b/allensdk/brain_observatory/behavior/data_files/sync_file.py new file mode 100644 index 000000000..dab04f9f7 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_files/sync_file.py @@ -0,0 +1,71 @@ +import json +from typing import Dict, Union +from pathlib import Path + +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.internal.core.lims_utilities import safe_system_path +from allensdk.brain_observatory.behavior.sync import get_sync_data +from allensdk.brain_observatory.behavior.data_files import DataFile + + +# Query returns path to sync timing file associated with ophys experiment +SYNC_FILE_QUERY_TEMPLATE = """ + SELECT wkf.storage_directory || wkf.filename AS sync_file + FROM ophys_experiments oe + JOIN ophys_sessions os ON oe.ophys_session_id = os.id + JOIN well_known_files wkf ON wkf.attachable_id = os.id + JOIN well_known_file_types wkft + ON wkft.id = wkf.well_known_file_type_id + WHERE wkf.attachable_type = 'OphysSession' + AND wkft.name = 'OphysRigSync' + AND oe.id = {ophys_experiment_id}; +""" + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class SyncFile(DataFile): + """A DataFile which contains methods for accessing and loading visual + behavior stimulus *.pkl files. + + This file type contains global timing information for different data + streams collected during a behavior + ophys session. + """ + + def __init__(self, filepath: Union[str, Path]): + super().__init__(filepath=filepath) + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json(cls, dict_repr: dict) -> "SyncFile": + filepath = dict_repr["sync_file"] + return cls(filepath=filepath) + + def to_json(self) -> Dict[str, str]: + return {"sync_file": str(self.filepath)} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: Union[int, str] + ) -> "SyncFile": + query = SYNC_FILE_QUERY_TEMPLATE.format( + ophys_experiment_id=ophys_experiment_id + ) + filepath = db.fetchone(query, strict=True) + return cls(filepath=filepath) + + @staticmethod + def load_data(filepath: Union[str, Path]) -> dict: + filepath = safe_system_path(file_name=filepath) + return get_sync_data(sync_path=filepath) diff --git a/allensdk/brain_observatory/behavior/data_objects/__init__.py b/allensdk/brain_observatory/behavior/data_objects/__init__.py new file mode 100644 index 000000000..9ea4e7d0c --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/__init__.py @@ -0,0 +1,7 @@ +from allensdk.brain_observatory.behavior.data_objects.base._data_object_abc import DataObject # noqa: E501, F401 +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_session_id import BehaviorSessionId # noqa: E501, F401 +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .stimulus_timestamps.stimulus_timestamps import StimulusTimestamps # noqa: E501, F401 +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_speed import RunningSpeed # noqa: E501, F401 +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_acquisition import RunningAcquisition # noqa: E501, F401 diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py b/allensdk/brain_observatory/behavior/data_objects/base/__init__.py similarity index 100% rename from allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py rename to allensdk/brain_observatory/behavior/data_objects/base/__init__.py diff --git a/allensdk/brain_observatory/behavior/data_objects/base/_data_object_abc.py b/allensdk/brain_observatory/behavior/data_objects/base/_data_object_abc.py new file mode 100644 index 000000000..3119ee77a --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/base/_data_object_abc.py @@ -0,0 +1,160 @@ +import abc +from collections import deque +from enum import Enum +from typing import Any, Optional, Set + +from allensdk.brain_observatory.comparison_utils import compare_fields + + +class DataObject(abc.ABC): + """An abstract class that prototypes properties that represent a + category of experimental data/metadata (e.g. running speed, + rewards, licks, etc.) and that prototypes methods to allow conversion of + the experimental data/metadata to and from various + data sources and sinks (e.g. LIMS, JSON, NWB). + """ + + def __init__(self, name: str, value: Any, + exclude_from_equals: Optional[Set[str]] = None): + """ + :param name + Name + :param value + Value + :param exclude_from_equals + Optional set which will exclude these properties from comparison + checks to another DataObject + """ + self._name = name + self._value = value + + efe = exclude_from_equals if exclude_from_equals else set() + self._exclude_from_equals = efe + + @property + def name(self) -> str: + return self._name + + @property + def value(self) -> Any: + return self._value + + def to_dict(self) -> dict: + """ + Serialize DataObject to dict + :return + A nested dict serializing the DataObject + + notes + If a DataObject contains properties, these properties will either: + 1) be serialized to nested dict with "name" attribute of + DataObject if the property is itself a DataObject + 2) Value for property will be added with name of property + :examples + >>> class Simple(DataObject): + ... def __init__(self): + ... super().__init__(name='simple', value=1) + >>> s = Simple() + >>> assert s.to_dict() == {'simple': 1} + + >>> class B(DataObject): + ... def __init__(self): + ... super().__init__(name='b', value='!') + + >>> class A(DataObject): + ... def __init__(self, b: B): + ... super().__init__(name='a', value=self) + ... self._b = b + ... @property + ... def prop1(self): + ... return self._b + ... @property + ... def prop2(self): + ... return '@' + >>> a = A(b=B()) + >>> assert a.to_dict() == {'a': {'b': '!', 'prop2': '@'}} + """ + res = dict() + q = deque([(self._name, self, [])]) + + while q: + name, value, path = q.popleft() + if isinstance(value, DataObject): + # The path stores the nested key structure + # Here, build onto the nested key structure + newpath = path + [name] + + def _get_keys_and_values(base_value: DataObject): + properties = [] + for name, value in base_value._get_properties().items(): + if value is base_value: + # skip properties that return self + # (leads to infinite recursion) + continue + if name == 'name': + # The name is the key + continue + + if isinstance(value, DataObject): + # The key will be the DataObject "name" field + name = value._name + else: + # The key will be the property name + pass + properties.append((name, value, newpath)) + return properties + properties = _get_keys_and_values(base_value=value) + + # Find the nested dict + cur = res + for p in path: + cur = cur[p] + + if isinstance(value._value, DataObject): + # it's nested + cur[value._name] = dict() + for p in properties: + q.append(p) + else: + # it's flat + cur[name] = value._value + + else: + cur = res + for p in path: + cur = cur[p] + + if isinstance(value, Enum): + # convert to string + value = value.value + cur[name] = value + + return res + + def _get_properties(self): + """Returns all property names and values""" + def is_prop(attr): + return isinstance(getattr(type(self), attr, None), property) + props = [attr for attr in dir(self) if is_prop(attr)] + return {name: getattr(self, name) for name in props} + + def __eq__(self, other: "DataObject"): + if type(self) != type(other): + msg = f'Do not know how to compare with type {type(other)}' + raise NotImplementedError(msg) + + d_self = self.to_dict() + d_other = other.to_dict() + + for p in d_self: + if p in self._exclude_from_equals: + continue + x1 = d_self[p] + x2 = d_other[p] + + try: + compare_fields(x1=x1, x2=x2, + ignore_keys=self._exclude_from_equals) + except AssertionError: + return False + return True diff --git a/allensdk/brain_observatory/behavior/data_objects/base/readable_interfaces.py b/allensdk/brain_observatory/behavior/data_objects/base/readable_interfaces.py new file mode 100644 index 000000000..f7c589802 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/base/readable_interfaces.py @@ -0,0 +1,105 @@ +import abc + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject + + +class JsonReadableInterface(abc.ABC): + """Marks a data object as readable from json""" + @classmethod + @abc.abstractmethod + def from_json(cls, dict_repr: dict) -> "DataObject": # pragma: no cover + """Populates a DataFile from a JSON compatible dict (likely parsed by + argschema) + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + raise NotImplementedError() + + +class LimsReadableInterface(abc.ABC): + """Marks a data object as readable from LIMS""" + @classmethod + @abc.abstractmethod + def from_lims(cls, *args) -> "DataObject": # pragma: no cover + """Populate a DataObject from an internal database (likely LIMS) + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + # Example: + # return cls(name="my_data_object", value=42) + raise NotImplementedError() + + +class NwbReadableInterface(abc.ABC): + """Marks a data object as readable from NWB""" + @classmethod + @abc.abstractmethod + def from_nwb(cls, nwbfile: NWBFile) -> "DataObject": # pragma: no cover + """Populate a DataObject from a pyNWB file object. + + Parameters + ---------- + nwbfile: + The file object (NWBFile) of a pynwb dataset file. + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + raise NotImplementedError() + + +class DataFileReadableInterface(abc.ABC): + """Marks a data object as readable from various data files, not covered by + existing interfaces""" + @classmethod + @abc.abstractmethod + def from_data_file(cls, *args) -> "DataObject": + """Populate a DataObject from the data file + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + raise NotImplementedError() + + +class StimulusFileReadableInterface(abc.ABC): + """Marks a data object as readable from stimulus file""" + @classmethod + @abc.abstractmethod + def from_stimulus_file(cls, stimulus_file: StimulusFile) -> "DataObject": + """Populate a DataObject from the stimulus file + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + raise NotImplementedError() + + +class SyncFileReadableInterface(abc.ABC): + """Marks a data object as readable from sync file""" + @classmethod + @abc.abstractmethod + def from_sync_file(cls, *args) -> "DataObject": + """Populate a DataObject from the sync file + + Returns + ------- + DataObject: + An instantiated DataObject which has `name` and `value` properties + """ + raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/data_objects/base/writable_interfaces.py b/allensdk/brain_observatory/behavior/data_objects/base/writable_interfaces.py new file mode 100644 index 000000000..04d04fef9 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/base/writable_interfaces.py @@ -0,0 +1,40 @@ +import abc + +from pynwb import NWBFile + + +class JsonWritableInterface(abc.ABC): + """Marks a data object as writable to NWB""" + @abc.abstractmethod + def to_json(self) -> dict: # pragma: no cover + """Given an already populated DataObject, return the dict that + when used with the `from_json()` classmethod would produce the same + DataObject + + Returns + ------- + dict: + The JSON (in dict form) that would produce the DataObject. + """ + raise NotImplementedError() + + +class NwbWritableInterface(abc.ABC): + """Marks a data object as writable to NWB""" + @abc.abstractmethod + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: # pragma: no cover + """Given an already populated DataObject, return an pyNWB file object + that had had DataObject data added. + + Parameters + ---------- + nwbfile : NWBFile + An NWB file object + + Returns + ------- + NWBFile + An NWB file object that has had data from the DataObject added + to it. + """ + raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/__init__.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/__init__.py similarity index 100% rename from allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/__init__.py rename to allensdk/brain_observatory/behavior/data_objects/cell_specimens/__init__.py diff --git a/allensdk/brain_observatory/behavior/data_objects/cell_specimens/cell_specimens.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/cell_specimens.py new file mode 100644 index 000000000..a626083a9 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/cell_specimens.py @@ -0,0 +1,606 @@ +from typing import Optional, Tuple + +import numpy as np +import pandas as pd +from pynwb import NWBFile, ProcessingModule +from pynwb.ophys import OpticalChannel, ImageSegmentation + +import allensdk.brain_observatory.roi_masks as roi +from allensdk.brain_observatory.behavior.data_files.demix_file import DemixFile +from allensdk.brain_observatory.behavior.data_files.dff_file import DFFFile +from allensdk.brain_observatory.behavior.data_files.event_detection_file \ + import \ + EventDetectionFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.cell_specimens.events \ + import \ + Events +from allensdk.brain_observatory.behavior.data_objects.cell_specimens.traces \ + .corrected_fluorescence_traces import \ + CorrectedFluorescenceTraces +from allensdk.brain_observatory.behavior.data_objects.cell_specimens.traces \ + .dff_traces import \ + DFFTraces +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.field_of_view_shape import \ + FieldOfViewShape +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.imaging_plane import \ + ImagingPlane +from allensdk.brain_observatory.behavior.data_objects.timestamps \ + .ophys_timestamps import \ + OphysTimestamps +from allensdk.brain_observatory.behavior.image_api import Image +from allensdk.brain_observatory.nwb import CELL_SPECIMEN_COL_DESCRIPTIONS +from allensdk.brain_observatory.nwb.nwb_utils import add_image_to_nwb +from allensdk.internal.api import PostgresQueryMixin + + +class EventsParams: + """Container for arguments to event detection""" + + def __init__(self, + filter_scale: float = 2, + filter_n_time_steps: int = 20): + """ + :param filter_scale + See Events.filter_scale + :param filter_n_time_steps + See Events.filter_n_time_steps + """ + self._filter_scale = filter_scale + self._filter_n_time_steps = filter_n_time_steps + + @property + def filter_scale(self): + return self._filter_scale + + @property + def filter_n_time_steps(self): + return self._filter_n_time_steps + + +class CellSpecimenMeta(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + """Cell specimen metadata""" + def __init__(self, imaging_plane: ImagingPlane, emission_lambda=520.0): + super().__init__(name='cell_spcimen_meta', value=self) + self._emission_lambda = emission_lambda + self._imaging_plane = imaging_plane + + @property + def emission_lambda(self): + return self._emission_lambda + + @property + def imaging_plane(self): + return self._imaging_plane + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin, + ophys_timestamps: OphysTimestamps) -> "CellSpecimenMeta": + imaging_plane_meta = ImagingPlane.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db, + ophys_timestamps=ophys_timestamps) + return cls(imaging_plane=imaging_plane_meta) + + @classmethod + def from_json(cls, dict_repr: dict, + ophys_timestamps: OphysTimestamps) -> "CellSpecimenMeta": + imaging_plane_meta = ImagingPlane.from_json( + dict_repr=dict_repr, ophys_timestamps=ophys_timestamps) + return cls(imaging_plane=imaging_plane_meta) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "CellSpecimenMeta": + ophys_module = nwbfile.processing['ophys'] + image_seg = ophys_module.data_interfaces['image_segmentation'] + plane_segmentations = image_seg.plane_segmentations + cell_specimen_table = plane_segmentations['cell_specimen_table'] + + imaging_plane = cell_specimen_table.imaging_plane + optical_channel = imaging_plane.optical_channel[0] + emission_lambda = optical_channel.emission_lambda + + imaging_plane = ImagingPlane.from_nwb(nwbfile=nwbfile) + return CellSpecimenMeta(emission_lambda=emission_lambda, + imaging_plane=imaging_plane) + + +class CellSpecimens(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface, + NwbWritableInterface): + def __init__(self, + cell_specimen_table: pd.DataFrame, + meta: CellSpecimenMeta, + dff_traces: DFFTraces, + corrected_fluorescence_traces: CorrectedFluorescenceTraces, + events: Events, + ophys_timestamps: OphysTimestamps, + segmentation_mask_image_spacing: Tuple, + exclude_invalid_rois=True): + """ + A container for cell specimens including traces, events, metadata, etc. + + Parameters + ---------- + cell_specimen_table + index cell_specimen_id + columns: + - cell_roi_id + - height + - mask_image_plane + - max_correction_down + - max_correction_left + - max_correction_right + - max_correction_up + - roi_mask + - valid_roi + - width + - x + - y + meta + dff_traces + corrected_fluorescence_traces + events + ophys_timestamps + segmentation_mask_image_spacing + Spacing to pass to sitk when constructing segmentation mask image + exclude_invalid_rois + Whether to exclude invalid rois + + """ + super().__init__(name='cell_specimen_table', value=self) + + # Validate ophys timestamps, traces + ophys_timestamps = ophys_timestamps.validate( + number_of_frames=dff_traces.get_number_of_frames()) + self._validate_traces( + ophys_timestamps=ophys_timestamps, dff_traces=dff_traces, + corrected_fluorescence_traces=corrected_fluorescence_traces, + cell_roi_ids=cell_specimen_table['cell_roi_id'].values) + + if exclude_invalid_rois: + cell_specimen_table = cell_specimen_table[ + cell_specimen_table['valid_roi']] + + # Filter/reorder rois according to cell_specimen_table + dff_traces.filter_and_reorder( + roi_ids=cell_specimen_table['cell_roi_id'].values) + corrected_fluorescence_traces.filter_and_reorder( + roi_ids=cell_specimen_table['cell_roi_id'].values) + + # Note: setting raise_if_rois_missing to False for events, since + # there seem to be cases where cell_specimen_table contains rois not in + # events + # See ie https://app.zenhub.com/workspaces/allensdk-10-5c17f74db59cfb36f158db8c/issues/alleninstitute/allensdk/2139 # noqa + events.filter_and_reorder( + roi_ids=cell_specimen_table['cell_roi_id'].values, + raise_if_rois_missing=False) + + self._meta = meta + self._cell_specimen_table = cell_specimen_table + self._dff_traces = dff_traces + self._corrected_fluorescence_traces = corrected_fluorescence_traces + self._events = events + self._segmentation_mask_image = self._get_segmentation_mask_image( + spacing=segmentation_mask_image_spacing) + + @property + def table(self) -> pd.DataFrame: + return self._cell_specimen_table + + @property + def roi_masks(self) -> pd.DataFrame: + return self._cell_specimen_table[['cell_roi_id', 'roi_mask']] + + @property + def meta(self) -> CellSpecimenMeta: + return self._meta + + @property + def dff_traces(self) -> pd.DataFrame: + df = self.table[['cell_roi_id']].join(self._dff_traces.value, + on='cell_roi_id') + return df + + @property + def corrected_fluorescence_traces(self) -> pd.DataFrame: + df = self.table[['cell_roi_id']].join( + self._corrected_fluorescence_traces.value, on='cell_roi_id') + return df + + @property + def events(self) -> pd.DataFrame: + df = self.table.reset_index() + df = df[['cell_roi_id', 'cell_specimen_id']] \ + .merge(self._events.value, on='cell_roi_id') + df = df.set_index('cell_specimen_id') + return df + + @property + def segmentation_mask_image(self) -> Image: + return self._segmentation_mask_image + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin, + ophys_timestamps: OphysTimestamps, + segmentation_mask_image_spacing: Tuple, + exclude_invalid_rois=True, + events_params: Optional[EventsParams] = None) \ + -> "CellSpecimens": + def _get_ophys_cell_segmentation_run_id() -> int: + """Get the ophys cell segmentation run id associated with an + ophys experiment id""" + query = """ + SELECT oseg.id + FROM ophys_experiments oe + JOIN ophys_cell_segmentation_runs oseg + ON oe.id = oseg.ophys_experiment_id + WHERE oseg.current = 't' + AND oe.id = {}; + """.format(ophys_experiment_id) + return lims_db.fetchone(query, strict=True) + + def _get_cell_specimen_table(): + ophys_cell_seg_run_id = _get_ophys_cell_segmentation_run_id() + query = """ + SELECT * + FROM cell_rois cr + WHERE cr.ophys_cell_segmentation_run_id = {}; + """.format(ophys_cell_seg_run_id) + initial_cs_table = pd.read_sql(query, lims_db.get_connection()) + cst = initial_cs_table.rename( + columns={'id': 'cell_roi_id', 'mask_matrix': 'roi_mask'}) + cst.drop(['ophys_experiment_id', + 'ophys_cell_segmentation_run_id'], + inplace=True, axis=1) + cst = cst.to_dict() + fov_shape = FieldOfViewShape.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + cst = cls._postprocess( + cell_specimen_table=cst, fov_shape=fov_shape) + return cst + + def _get_dff_traces(): + dff_file = DFFFile.from_lims( + ophys_experiment_id=ophys_experiment_id, + db=lims_db) + return DFFTraces.from_data_file( + dff_file=dff_file) + + def _get_corrected_fluorescence_traces(): + demix_file = DemixFile.from_lims( + ophys_experiment_id=ophys_experiment_id, + db=lims_db) + return CorrectedFluorescenceTraces.from_data_file( + demix_file=demix_file) + + def _get_events(): + events_file = EventDetectionFile.from_lims( + ophys_experiment_id=ophys_experiment_id, + db=lims_db) + return cls._get_events(events_file=events_file, + events_params=events_params) + + cell_specimen_table = _get_cell_specimen_table() + meta = CellSpecimenMeta.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db, + ophys_timestamps=ophys_timestamps) + dff_traces = _get_dff_traces() + corrected_fluorescence_traces = _get_corrected_fluorescence_traces() + events = _get_events() + + return CellSpecimens( + cell_specimen_table=cell_specimen_table, meta=meta, + dff_traces=dff_traces, + corrected_fluorescence_traces=corrected_fluorescence_traces, + events=events, + ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=segmentation_mask_image_spacing, + exclude_invalid_rois=exclude_invalid_rois + ) + + @classmethod + def from_json(cls, dict_repr: dict, + ophys_timestamps: OphysTimestamps, + segmentation_mask_image_spacing: Tuple, + exclude_invalid_rois=True, + events_params: Optional[EventsParams] = None) \ + -> "CellSpecimens": + cell_specimen_table = dict_repr['cell_specimen_table_dict'] + fov_shape = FieldOfViewShape.from_json(dict_repr=dict_repr) + cell_specimen_table = cls._postprocess( + cell_specimen_table=cell_specimen_table, fov_shape=fov_shape) + + def _get_dff_traces(): + dff_file = DFFFile.from_json(dict_repr=dict_repr) + return DFFTraces.from_data_file( + dff_file=dff_file) + + def _get_corrected_fluorescence_traces(): + demix_file = DemixFile.from_json(dict_repr=dict_repr) + return CorrectedFluorescenceTraces.from_data_file( + demix_file=demix_file) + + def _get_events(): + events_file = EventDetectionFile.from_json(dict_repr=dict_repr) + return cls._get_events(events_file=events_file, + events_params=events_params) + + meta = CellSpecimenMeta.from_json(dict_repr=dict_repr, + ophys_timestamps=ophys_timestamps) + dff_traces = _get_dff_traces() + corrected_fluorescence_traces = _get_corrected_fluorescence_traces() + events = _get_events() + return CellSpecimens( + cell_specimen_table=cell_specimen_table, meta=meta, + dff_traces=dff_traces, + corrected_fluorescence_traces=corrected_fluorescence_traces, + events=events, + ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=segmentation_mask_image_spacing, + exclude_invalid_rois=exclude_invalid_rois) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile, + segmentation_mask_image_spacing: Tuple, + exclude_invalid_rois=True, + events_params: Optional[EventsParams] = None) \ + -> "CellSpecimens": + # NOTE: ROI masks are stored in full frame width and height arrays + ophys_module = nwbfile.processing['ophys'] + image_seg = ophys_module.data_interfaces['image_segmentation'] + plane_segmentations = image_seg.plane_segmentations + cell_specimen_table = plane_segmentations['cell_specimen_table'] + + def _read_table(cell_specimen_table): + df = cell_specimen_table.to_dataframe() + + # Ensure int64 used instead of int32 + df = df.astype( + {col: 'int64' for col in df.select_dtypes('int32').columns}) + + # Because pynwb stores this field as "image_mask", it is renamed + # here + df = df.rename(columns={'image_mask': 'roi_mask'}) + + df.index.rename('cell_roi_id', inplace=True) + df['cell_specimen_id'] = [None if id_ == -1 else id_ + for id_ in df['cell_specimen_id'].values] + + df.reset_index(inplace=True) + df.set_index('cell_specimen_id', inplace=True) + return df + + df = _read_table(cell_specimen_table=cell_specimen_table) + meta = CellSpecimenMeta.from_nwb(nwbfile=nwbfile) + dff_traces = DFFTraces.from_nwb(nwbfile=nwbfile) + corrected_fluorescence_traces = CorrectedFluorescenceTraces.from_nwb( + nwbfile=nwbfile) + + def _get_events(): + ep = EventsParams() if events_params is None else events_params + return Events.from_nwb( + nwbfile=nwbfile, filter_scale=ep.filter_scale, + filter_n_time_steps=ep.filter_n_time_steps) + + events = _get_events() + ophys_timestamps = OphysTimestamps.from_nwb(nwbfile=nwbfile) + + return CellSpecimens( + cell_specimen_table=df, meta=meta, dff_traces=dff_traces, + corrected_fluorescence_traces=corrected_fluorescence_traces, + events=events, + ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=segmentation_mask_image_spacing, + exclude_invalid_rois=exclude_invalid_rois) + + def to_nwb(self, nwbfile: NWBFile, + ophys_timestamps: OphysTimestamps) -> NWBFile: + """ + :param nwbfile + In-memory nwb file object + :param ophys_timestamps + ophys timestamps + """ + # 1. Add cell specimen table + cell_roi_table = self.table.reset_index().set_index( + 'cell_roi_id') + metadata = nwbfile.lab_meta_data['metadata'] + + device = nwbfile.get_device() + + # FOV: + fov_width = metadata.field_of_view_width + fov_height = metadata.field_of_view_height + imaging_plane_description = \ + "{} field of view in {} at depth {} " \ + "um".format( + (fov_width, fov_height), + self._meta.imaging_plane.targeted_structure, + metadata.imaging_depth) + + # Optical Channel: + optical_channel = OpticalChannel( + name='channel_1', + description='2P Optical Channel', + emission_lambda=self._meta.emission_lambda) + + # Imaging Plane: + imaging_plane = nwbfile.create_imaging_plane( + name='imaging_plane_1', + optical_channel=optical_channel, + description=imaging_plane_description, + device=device, + excitation_lambda=self._meta.imaging_plane.excitation_lambda, + imaging_rate=self._meta.imaging_plane.ophys_frame_rate, + indicator=self._meta.imaging_plane.indicator, + location=self._meta.imaging_plane.targeted_structure) + + # Image Segmentation: + image_segmentation = ImageSegmentation(name="image_segmentation") + + if 'ophys' not in nwbfile.processing: + ophys_module = ProcessingModule('ophys', 'Ophys processing module') + nwbfile.add_processing_module(ophys_module) + else: + ophys_module = nwbfile.processing['ophys'] + + ophys_module.add_data_interface(image_segmentation) + + # Plane Segmentation: + plane_segmentation = image_segmentation.create_plane_segmentation( + name='cell_specimen_table', + description="Segmented rois", + imaging_plane=imaging_plane) + + for col_name in cell_roi_table.columns: + # the columns 'roi_mask', 'pixel_mask', and 'voxel_mask' are + # already defined in the nwb.ophys::PlaneSegmentation Object + if col_name not in ['id', 'mask_matrix', 'roi_mask', + 'pixel_mask', 'voxel_mask']: + # This builds the columns with name of column and description + # of column both equal to the column name in the cell_roi_table + plane_segmentation.add_column( + col_name, + CELL_SPECIMEN_COL_DESCRIPTIONS.get( + col_name, + "No Description Available")) + + # go through each roi and add it to the plan segmentation object + for cell_roi_id, table_row in cell_roi_table.iterrows(): + # NOTE: The 'roi_mask' in this cell_roi_table has already been + # processing by the function from + # allensdk.brain_observatory.behavior.session_apis.data_io + # .ophys_lims_api + # get_cell_specimen_table() method. As a result, the ROI is + # stored in + # an array that is the same shape as the FULL field of view of the + # experiment (e.g. 512 x 512). + mask = table_row.pop('roi_mask') + + csid = table_row.pop('cell_specimen_id') + table_row['cell_specimen_id'] = -1 if csid is None else csid + table_row['id'] = cell_roi_id + plane_segmentation.add_roi(image_mask=mask, **table_row.to_dict()) + + # 2. Add DFF traces + self._dff_traces.to_nwb(nwbfile=nwbfile, + ophys_timestamps=ophys_timestamps) + + # 3. Add Corrected fluorescence traces + self._corrected_fluorescence_traces.to_nwb(nwbfile=nwbfile) + + # 4. Add events + self._events.to_nwb(nwbfile=nwbfile) + + # 5. Add segmentation mask image + add_image_to_nwb(nwbfile=nwbfile, + image_data=self._segmentation_mask_image, + image_name='segmentation_mask_image') + + return nwbfile + + def _get_segmentation_mask_image(self, spacing: tuple) -> Image: + """a 2D binary image of all cell masks + + Parameters + ---------- + spacing + See image_api.Image for details + + Returns + ---------- + allensdk.brain_observatory.behavior.image_api.Image: + array-like interface to segmentation_mask image data and + metadata + """ + mask_data = np.sum(self.roi_masks['roi_mask']).astype(int) + + mask_image = Image( + data=mask_data, + spacing=spacing, + unit='mm' + ) + return mask_image + + @staticmethod + def _postprocess(cell_specimen_table: dict, + fov_shape: FieldOfViewShape) -> pd.DataFrame: + """Converts raw cell_specimen_table dict to dataframe""" + cell_specimen_table = pd.DataFrame.from_dict( + cell_specimen_table).set_index( + 'cell_roi_id').sort_index() + fov_width = fov_shape.width + fov_height = fov_shape.height + + # Convert cropped ROI masks to uncropped versions + roi_mask_list = [] + for cell_roi_id, table_row in cell_specimen_table.iterrows(): + # Deserialize roi data into AllenSDK RoiMask object + curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height, + label=None, mask_group=-1) + curr_roi.x = table_row['x'] + curr_roi.y = table_row['y'] + curr_roi.width = table_row['width'] + curr_roi.height = table_row['height'] + curr_roi.mask = np.array(table_row['roi_mask']) + roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool)) + + cell_specimen_table['roi_mask'] = roi_mask_list + cell_specimen_table = cell_specimen_table[ + sorted(cell_specimen_table.columns)] + + cell_specimen_table.index.rename('cell_roi_id', inplace=True) + cell_specimen_table.reset_index(inplace=True) + cell_specimen_table.set_index('cell_specimen_id', inplace=True) + return cell_specimen_table + + def _validate_traces( + self, ophys_timestamps: OphysTimestamps, + dff_traces: DFFTraces, + corrected_fluorescence_traces: CorrectedFluorescenceTraces, + cell_roi_ids: np.ndarray): + """validates traces""" + trace_col_map = { + 'dff_traces': 'dff', + 'corrected_fluorescence_traces': 'corrected_fluorescence' + } + for traces in (dff_traces, corrected_fluorescence_traces): + # validate traces contain expected roi ids + if not np.in1d(traces.value.index, cell_roi_ids).all(): + raise RuntimeError(f"{traces.name} contains ROI IDs that " + f"are not in " + f"cell_specimen_table.cell_roi_id") + if not np.in1d(cell_roi_ids, traces.value.index).all(): + raise RuntimeError(f"cell_specimen_table contains ROI IDs " + f"that are not in {traces.name}") + + # validate traces contain expected timepoints + num_trace_timepoints = len(traces.value.iloc[0] + [trace_col_map[traces.name]]) + num_ophys_timestamps = ophys_timestamps.value.shape[0] + if num_trace_timepoints != num_ophys_timestamps: + raise RuntimeError(f'{traces.name} contains ' + f'{num_trace_timepoints} ' + f'but there are {num_ophys_timestamps} ' + f'ophys timestamps') + + @staticmethod + def _get_events(events_file: EventDetectionFile, + events_params: Optional[EventsParams] = None): + if events_params is None: + events_params = EventsParams() + return Events.from_data_file( + events_file=events_file, + filter_scale=events_params.filter_scale, + filter_n_time_steps=events_params.filter_n_time_steps) diff --git a/allensdk/brain_observatory/behavior/data_objects/cell_specimens/events.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/events.py new file mode 100644 index 000000000..ac45f255a --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/events.py @@ -0,0 +1,137 @@ +import numpy as np +import pandas as pd +from hdmf.backends.hdf5 import H5DataIO +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files.event_detection_file \ + import \ + EventDetectionFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + DataFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .rois_mixin import \ + RoisMixin +from allensdk.brain_observatory.behavior.event_detection import \ + filter_events_array +from allensdk.brain_observatory.behavior.write_nwb.extensions\ + .event_detection.ndx_ophys_events import \ + OphysEventDetection + + +class Events(DataObject, RoisMixin, DataFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + """Events + columns: + events: np.array + lambda: float + noise_std: float + cell_roi_id: int + """ + def __init__(self, + events: np.ndarray, + events_meta: pd.DataFrame, + filter_scale: float = 2, + filter_n_time_steps: int = 20): + """ + Parameters + ---------- + events + events + events_meta + lambda, noise_std, cell_roi_id for each roi + filter_scale + See filter_events_array for description + filter_n_time_steps + See filter_events_array for description + """ + + filtered_events = filter_events_array( + arr=events, scale=filter_scale, n_time_steps=filter_n_time_steps) + + # Convert matrix to list of 1d arrays so that it can be stored + # in a single column of the dataframe + events = [x for x in events] + filtered_events = [x for x in filtered_events] + + df = pd.DataFrame({ + 'events': events, + 'filtered_events': filtered_events, + 'lambda': events_meta['lambda'], + 'noise_std': events_meta['noise_std'], + 'cell_roi_id': events_meta['cell_roi_id'] + }) + super().__init__(name='events', value=df) + + @classmethod + def from_data_file(cls, + events_file: EventDetectionFile, + filter_scale: float = 2, + filter_n_time_steps: int = 20) -> "Events": + events, events_meta = events_file.data + return cls(events=events, events_meta=events_meta, + filter_scale=filter_scale, + filter_n_time_steps=filter_n_time_steps) + + @classmethod + def from_nwb(cls, + nwbfile: NWBFile, + filter_scale: float = 2, + filter_n_time_steps: int = 20) -> "Events": + event_detection = nwbfile.processing['ophys']['event_detection'] + # NOTE: The rois with events are stored in event detection + partial_cell_specimen_table = event_detection.rois.to_dataframe() + + events = event_detection.data[:] + + # events stored time x roi. Change back to roi x time + events = events.T + + events_meta = pd.DataFrame({ + 'cell_roi_id': partial_cell_specimen_table.index, + 'lambda': event_detection.lambdas[:], + 'noise_std': event_detection.noise_stds[:] + }) + return cls(events=events, events_meta=events_meta, + filter_scale=filter_scale, + filter_n_time_steps=filter_n_time_steps) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + events = self.value.set_index('cell_roi_id') + + ophys_module = nwbfile.processing['ophys'] + dff_interface = ophys_module.data_interfaces['dff'] + traces = dff_interface.roi_response_series['traces'] + seg_interface = ophys_module.data_interfaces['image_segmentation'] + + cell_specimen_table = ( + seg_interface.plane_segmentations['cell_specimen_table']) + cell_specimen_df = cell_specimen_table.to_dataframe() + + # We only want to store the subset of rois that have events data + rois_with_events_indices = [cell_specimen_df.index.get_loc(label) + for label in events.index] + roi_table_region = cell_specimen_table.create_roi_table_region( + description="Cells with detected events", + region=rois_with_events_indices) + + events_data = np.vstack(events['events']) + events = OphysEventDetection( + # time x rois instead of rois x time + # store using compression since sparse + data=H5DataIO(events_data.T, compression=True), + + lambdas=events['lambda'].values, + noise_stds=events['noise_std'].values, + unit='N/A', + rois=roi_table_region, + timestamps=traces.timestamps + ) + + ophys_module.add_data_interface(events) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/cell_specimens/rois_mixin.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/rois_mixin.py new file mode 100644 index 000000000..dda58a7a6 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/rois_mixin.py @@ -0,0 +1,89 @@ +import warnings + +import numpy as np +import pandas as pd + + +class RoisMixin: + """A mixin for a collection of rois stored as a dataframe + (._value is a dataframe)""" + _value: pd.DataFrame + + def filter_and_reorder(self, roi_ids: np.ndarray, + raise_if_rois_missing=True): + """Orders dataframe according to input roi_ids. + Will also filter dataframe to contain only rois given by roi_ids. + Use for, ie excluding invalid rois + + Parameters + ---------- + roi_ids + Filter/reorder _value to these roi_ids + raise_if_rois_missing + Whether to raise exception if there are rois in the input roi_ids + not in the dataframe + + Notes + ---------- + Will both filter and reorder dataframe to have same order as the + input roi_ids. + + If there are values in the input roi_ids that are not in the dataframe, + then these roi ids will be ignored and a warning will be logged. + + Raises + ---------- + RuntimeError if raise_if_rois_missing and there are input roi_ids not + in dataframe + """ + def handle_rois_in_input_not_in_dataframe(): + msg = f'Input contains roi ids not in ' \ + f'{type(self).__name__}.' + if raise_if_rois_missing: + raise RuntimeError(msg) + warnings.warn(msg) + + # Drop rows where NaN + self._value = self._value.dropna(axis=0) + + # Make sure dtypes same after dropping NaN rows + # (adding NaN records coerces int to float) + for c in self._value: + # Skipping column added due to reset_index + if c == 'index': + continue + + if self._value[c].dtype != original_dtypes[c]: + self._value[c] = self._value[c].astype(original_dtypes[c]) + + original_index_name = self._value.index.name + original_index_type = self._value.index.dtype + original_dtypes = self._value.dtypes + if original_index_name is None: + original_index_name = 'index' + + if original_index_name != 'cell_roi_id': + self._value = (self._value + .reset_index() + .set_index('cell_roi_id')) + + # Reorders dataframe according to roi_ids + self._value = self._value.reindex(roi_ids) + + is_na = self._value.isna().any(axis=0) + + if is_na.any(): + # There are some roi ids in input not in index. + handle_rois_in_input_not_in_dataframe() + + if original_index_name != 'cell_roi_id': + # Set it back to the original index + self._value = (self._value + .reset_index() + .set_index(original_index_name)) + # Set index back to original dtype + # (can get coerced from int to float) + self._value.index = self._value.index.astype(original_index_type) + if original_index_name == 'index': + # Set it back to None + self._value.index.name = None diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/__init__.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/__init__.py similarity index 100% rename from allensdk/brain_observatory/behavior/session_apis/abcs/session_base/__init__.py rename to allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/__init__.py diff --git a/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/corrected_fluorescence_traces.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/corrected_fluorescence_traces.py new file mode 100644 index 000000000..36f9a2cd9 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/corrected_fluorescence_traces.py @@ -0,0 +1,83 @@ +import numpy as np +import pandas as pd +from pynwb import NWBFile +from pynwb.ophys import Fluorescence + +from allensdk.brain_observatory.behavior.data_files.demix_file import DemixFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + DataFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .rois_mixin import \ + RoisMixin + + +class CorrectedFluorescenceTraces(DataObject, RoisMixin, + DataFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, traces: pd.DataFrame): + """ + + Parameters + ---------- + traces + index cell_roi_id + columns: + - corrected_fluorescence + list of float + """ + super().__init__(name='corrected_fluorescence_traces', value=traces) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) \ + -> "CorrectedFluorescenceTraces": + corr_fluorescence_nwb = nwbfile.processing[ + 'ophys'].data_interfaces[ + 'corrected_fluorescence'].roi_response_series['traces'] + # f traces stored as timepoints x rois in NWB + # We want rois x timepoints, hence the transpose + f_traces = corr_fluorescence_nwb.data[:].T + df = pd.DataFrame({'corrected_fluorescence': f_traces.tolist()}, + index=pd.Index( + data=corr_fluorescence_nwb.rois.table.id[:], + name='cell_roi_id')) + return cls(traces=df) + + @classmethod + def from_data_file(cls, + demix_file: DemixFile) \ + -> "CorrectedFluorescenceTraces": + corrected_fluorescence_traces = demix_file.data + return cls(traces=corrected_fluorescence_traces) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + corrected_fluorescence_traces = self.value['corrected_fluorescence'] + + # Convert from Series of lists to numpy array + # of shape ROIs x timepoints + traces = np.stack( + [x for x in corrected_fluorescence_traces]) + + # Create/Add corrected_fluorescence_traces modules and interfaces: + ophys_module = nwbfile.processing['ophys'] + + roi_table_region = \ + nwbfile.processing['ophys'].data_interfaces[ + 'dff'].roi_response_series[ + 'traces'].rois # noqa: E501 + ophys_timestamps = ophys_module.get_data_interface( + 'dff').roi_response_series['traces'].timestamps + f_interface = Fluorescence(name='corrected_fluorescence') + ophys_module.add_data_interface(f_interface) + + f_interface.create_roi_response_series( + name='traces', + data=traces.T, # Should be stored as timepoints x rois + unit='NA', + rois=roi_table_region, + timestamps=ophys_timestamps) + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/dff_traces.py b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/dff_traces.py new file mode 100644 index 000000000..8b8f5af6a --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/cell_specimens/traces/dff_traces.py @@ -0,0 +1,88 @@ +import pandas as pd +import numpy as np +from pynwb import NWBFile +from pynwb.ophys import DfOverF + +from allensdk.brain_observatory.behavior.data_files.dff_file import DFFFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + DataFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .rois_mixin import \ + RoisMixin +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .ophys_timestamps import \ + OphysTimestamps + + +class DFFTraces(DataObject, RoisMixin, + DataFileReadableInterface, NwbReadableInterface, + NwbWritableInterface): + def __init__(self, traces: pd.DataFrame): + """ + Parameters + ---------- + traces + index cell_roi_id + columns: + dff: List of float + """ + super().__init__(name='dff_traces', value=traces) + + def to_nwb(self, nwbfile: NWBFile, + ophys_timestamps: OphysTimestamps) -> NWBFile: + dff_traces = self.value[['dff']] + + ophys_module = nwbfile.processing['ophys'] + # trace data in the form of rois x timepoints + trace_data = np.array([dff_traces.loc[cell_roi_id].dff + for cell_roi_id in dff_traces.index.values]) + + cell_specimen_table = nwbfile.processing['ophys'].data_interfaces[ + 'image_segmentation'].plane_segmentations[ + 'cell_specimen_table'] # noqa: E501 + roi_table_region = cell_specimen_table.create_roi_table_region( + description="segmented cells labeled by cell_specimen_id", + region=slice(len(dff_traces))) + + # Create/Add dff modules and interfaces: + assert dff_traces.index.name == 'cell_roi_id' + dff_interface = DfOverF(name='dff') + ophys_module.add_data_interface(dff_interface) + + dff_interface.create_roi_response_series( + name='traces', + data=trace_data.T, # Should be stored as timepoints x rois + unit='NA', + rois=roi_table_region, + timestamps=ophys_timestamps.value) + + return nwbfile + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "DFFTraces": + dff_nwb = nwbfile.processing[ + 'ophys'].data_interfaces['dff'].roi_response_series['traces'] + # dff traces stored as timepoints x rois in NWB + # We want rois x timepoints, hence the transpose + dff_traces = dff_nwb.data[:].T + + df = pd.DataFrame({'dff': dff_traces.tolist()}, + index=pd.Index(data=dff_nwb.rois.table.id[:], + name='cell_roi_id')) + return DFFTraces(traces=df) + + @classmethod + def from_data_file(cls, dff_file: DFFFile) -> "DFFTraces": + dff_traces = dff_file.data + return DFFTraces(traces=dff_traces) + + def get_number_of_frames(self) -> int: + """Returns the number of frames in the movie""" + if self.value.empty: + raise RuntimeError('Cannot determine number of frames') + return len(self.value.iloc[0]['dff']) diff --git a/allensdk/brain_observatory/behavior/data_objects/eye_tracking/__init__.py b/allensdk/brain_observatory/behavior/data_objects/eye_tracking/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/eye_tracking/eye_tracking_table.py b/allensdk/brain_observatory/behavior/data_objects/eye_tracking/eye_tracking_table.py new file mode 100644 index 000000000..6a765afe2 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/eye_tracking/eye_tracking_table.py @@ -0,0 +1,198 @@ +import logging +import warnings +from pathlib import Path +from typing import Optional + +import numpy as np +import pandas as pd +from pynwb import NWBFile, TimeSeries + +from allensdk.brain_observatory import sync_utilities +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_files.eye_tracking_file import \ + EyeTrackingFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + NwbReadableInterface, DataFileReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.eye_tracking_processing import \ + process_eye_tracking_data, determine_outliers, determine_likely_blinks +from allensdk.brain_observatory.nwb.eye_tracking.ndx_ellipse_eye_tracking \ + import \ + EllipseSeries, EllipseEyeTracking +from allensdk.brain_observatory.sync_dataset import Dataset + + +class EyeTrackingTable(DataObject, DataFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + """corneal, eye, and pupil ellipse fit data""" + _logger = logging.getLogger(__name__) + + def __init__(self, eye_tracking: pd.DataFrame): + super().__init__(name='eye_tracking', value=eye_tracking) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + eye_tracking_df = self.value + + eye_tracking = EllipseSeries( + name='eye_tracking', + reference_frame='nose', + data=eye_tracking_df[['eye_center_x', 'eye_center_y']].values, + area=eye_tracking_df['eye_area'].values, + area_raw=eye_tracking_df['eye_area_raw'].values, + width=eye_tracking_df['eye_width'].values, + height=eye_tracking_df['eye_height'].values, + angle=eye_tracking_df['eye_phi'].values, + timestamps=eye_tracking_df['timestamps'].values + ) + + pupil_tracking = EllipseSeries( + name='pupil_tracking', + reference_frame='nose', + data=eye_tracking_df[['pupil_center_x', 'pupil_center_y']].values, + area=eye_tracking_df['pupil_area'].values, + area_raw=eye_tracking_df['pupil_area_raw'].values, + width=eye_tracking_df['pupil_width'].values, + height=eye_tracking_df['pupil_height'].values, + angle=eye_tracking_df['pupil_phi'].values, + timestamps=eye_tracking + ) + + corneal_reflection_tracking = EllipseSeries( + name='corneal_reflection_tracking', + reference_frame='nose', + data=eye_tracking_df[['cr_center_x', 'cr_center_y']].values, + area=eye_tracking_df['cr_area'].values, + area_raw=eye_tracking_df['cr_area_raw'].values, + width=eye_tracking_df['cr_width'].values, + height=eye_tracking_df['cr_height'].values, + angle=eye_tracking_df['cr_phi'].values, + timestamps=eye_tracking + ) + + likely_blink = TimeSeries(timestamps=eye_tracking, + data=eye_tracking_df['likely_blink'].values, + name='likely_blink', + description='blinks', + unit='N/A') + + ellipse_eye_tracking = EllipseEyeTracking( + eye_tracking=eye_tracking, + pupil_tracking=pupil_tracking, + corneal_reflection_tracking=corneal_reflection_tracking, + likely_blink=likely_blink + ) + + nwbfile.add_acquisition(ellipse_eye_tracking) + return nwbfile + + @classmethod + def from_nwb(cls, nwbfile: NWBFile, + z_threshold: float = 3.0, + dilation_frames: int = 2) -> Optional["EyeTrackingTable"]: + """ + Parameters + ----------- + nwbfile + z_threshold + See from_lims for description + dilation_frames + See from_lims for description + """ + try: + eye_tracking_acquisition = nwbfile.acquisition['EyeTracking'] + except KeyError as e: + warnings.warn("This ophys session " + f"'{int(nwbfile.identifier)}' has no eye " + f"tracking data. (NWB error: {e})") + return None + + eye_tracking = eye_tracking_acquisition.eye_tracking + pupil_tracking = eye_tracking_acquisition.pupil_tracking + corneal_reflection_tracking = \ + eye_tracking_acquisition.corneal_reflection_tracking + + eye_tracking_dict = { + "timestamps": eye_tracking.timestamps[:], + "cr_area": corneal_reflection_tracking.area_raw[:], + "eye_area": eye_tracking.area_raw[:], + "pupil_area": pupil_tracking.area_raw[:], + "likely_blink": eye_tracking_acquisition.likely_blink.data[:], + + "pupil_area_raw": pupil_tracking.area_raw[:], + "cr_area_raw": corneal_reflection_tracking.area_raw[:], + "eye_area_raw": eye_tracking.area_raw[:], + + "cr_center_x": corneal_reflection_tracking.data[:, 0], + "cr_center_y": corneal_reflection_tracking.data[:, 1], + "cr_width": corneal_reflection_tracking.width[:], + "cr_height": corneal_reflection_tracking.height[:], + "cr_phi": corneal_reflection_tracking.angle[:], + + "eye_center_x": eye_tracking.data[:, 0], + "eye_center_y": eye_tracking.data[:, 1], + "eye_width": eye_tracking.width[:], + "eye_height": eye_tracking.height[:], + "eye_phi": eye_tracking.angle[:], + + "pupil_center_x": pupil_tracking.data[:, 0], + "pupil_center_y": pupil_tracking.data[:, 1], + "pupil_width": pupil_tracking.width[:], + "pupil_height": pupil_tracking.height[:], + "pupil_phi": pupil_tracking.angle[:], + + } + + eye_tracking_data = pd.DataFrame(eye_tracking_dict) + eye_tracking_data.index = eye_tracking_data.index.rename('frame') + + # re-calculate likely blinks for new z_threshold and dilate_frames + area_df = eye_tracking_data[['eye_area_raw', 'pupil_area_raw']] + outliers = determine_outliers(area_df, z_threshold=z_threshold) + likely_blinks = determine_likely_blinks( + eye_tracking_data['eye_area_raw'], + eye_tracking_data['pupil_area_raw'], + outliers, + dilation_frames=dilation_frames) + + eye_tracking_data["likely_blink"] = likely_blinks + eye_tracking_data.at[likely_blinks, "eye_area"] = np.nan + eye_tracking_data.at[likely_blinks, "pupil_area"] = np.nan + eye_tracking_data.at[likely_blinks, "cr_area"] = np.nan + + return EyeTrackingTable(eye_tracking=eye_tracking_data) + + @classmethod + def from_data_file(cls, data_file: EyeTrackingFile, + sync_file: SyncFile, + z_threshold: float = 3.0, dilation_frames: int = 2 + ) -> "EyeTrackingTable": + """ + Parameters + ---------- + data_file + sync_file + z_threshold : float, optional + See EyeTracking.from_lims + dilation_frames : int, optional + See EyeTracking.from_lims + """ + cls._logger.info(f"Getting eye_tracking_data with " + f"'z_threshold={z_threshold}', " + f"'dilation_frames={dilation_frames}'") + + sync_path = Path(sync_file.filepath) + + frame_times = sync_utilities.get_synchronized_frame_times( + session_sync_file=sync_path, + sync_line_label_keys=Dataset.EYE_TRACKING_KEYS, + trim_after_spike=False) + + eye_tracking_data = process_eye_tracking_data(data_file.data, + frame_times, + z_threshold, + dilation_frames) + return EyeTrackingTable(eye_tracking=eye_tracking_data) diff --git a/allensdk/brain_observatory/behavior/data_objects/eye_tracking/rig_geometry.py b/allensdk/brain_observatory/behavior/data_objects/eye_tracking/rig_geometry.py new file mode 100644 index 000000000..692e818eb --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/eye_tracking/rig_geometry.py @@ -0,0 +1,284 @@ +import warnings + +import numpy as np +import pandas as pd +from typing import Optional + +import pynwb +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + LimsReadableInterface, JsonReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.schemas import \ + OphysEyeTrackingRigMetadataSchema +from allensdk.brain_observatory.nwb import load_pynwb_extension +from allensdk.internal.api import PostgresQueryMixin + + +class Coordinates: + """Represents coordinates in 3d space""" + def __init__(self, x: float, y: float, z: float): + self._x = x + self._y = y + self._z = z + + @property + def x(self): + return self._x + + @property + def y(self): + return self._y + + @property + def z(self): + return self._z + + def __iter__(self): + yield self._x + yield self._y + yield self._z + + def __eq__(self, other): + if type(other) not in (type(self), list): + raise ValueError(f'Do not know how to compare with type ' + f'{type(other)}') + if isinstance(other, list): + return self._x == other[0] and \ + self._y == other[1] and \ + self._z == other[2] + else: + return self._x == other.x and \ + self._y == other.y and \ + self._z == other.z + + def __str__(self): + return f'[{self._x}, {self._y}, {self._z}]' + + +class RigGeometry(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, equipment: str, + monitor_position_mm: Coordinates, + monitor_rotation_deg: Coordinates, + camera_position_mm: Coordinates, + camera_rotation_deg: Coordinates, + led_position: Coordinates): + super().__init__(name='rig_geometry', value=self) + self._monitor_position_mm = monitor_position_mm + self._monitor_rotation_deg = monitor_rotation_deg + self._camera_position_mm = camera_position_mm + self._camera_rotation_deg = camera_rotation_deg + self._led_position = led_position + self._equipment = equipment + + @property + def monitor_position_mm(self): + return self._monitor_position_mm + + @property + def monitor_rotation_deg(self): + return self._monitor_rotation_deg + + @property + def camera_position_mm(self): + return self._camera_position_mm + + @property + def camera_rotation_deg(self): + return self._camera_rotation_deg + + @property + def led_position(self): + return self._led_position + + @property + def equipment(self): + return self._equipment + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + eye_tracking_rig_mod = pynwb.ProcessingModule( + name='eye_tracking_rig_metadata', + description='Eye tracking rig metadata module') + + nwb_extension = load_pynwb_extension( + OphysEyeTrackingRigMetadataSchema, 'ndx-aibs-behavior-ophys') + + rig_metadata = nwb_extension( + name="eye_tracking_rig_metadata", + equipment=self._equipment, + monitor_position=list(self._monitor_position_mm), + monitor_position__unit_of_measurement="mm", + camera_position=list(self._camera_position_mm), + camera_position__unit_of_measurement="mm", + led_position=list(self._led_position), + led_position__unit_of_measurement="mm", + monitor_rotation=list(self._monitor_rotation_deg), + monitor_rotation__unit_of_measurement="deg", + camera_rotation=list(self._camera_rotation_deg), + camera_rotation__unit_of_measurement="deg" + ) + + eye_tracking_rig_mod.add_data_interface(rig_metadata) + nwbfile.add_processing_module(eye_tracking_rig_mod) + return nwbfile + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> Optional["RigGeometry"]: + try: + et_mod = \ + nwbfile.get_processing_module("eye_tracking_rig_metadata") + except KeyError as e: + warnings.warn("This ophys session " + f"'{int(nwbfile.identifier)}' has no eye " + f"tracking rig metadata. (NWB error: {e})") + return None + + meta = et_mod.get_data_interface("eye_tracking_rig_metadata") + + monitor_position = meta.monitor_position[:] + monitor_position = (monitor_position.tolist() + if isinstance(monitor_position, np.ndarray) + else monitor_position) + + monitor_rotation = meta.monitor_rotation[:] + monitor_rotation = (monitor_rotation.tolist() + if isinstance(monitor_rotation, np.ndarray) + else monitor_rotation) + + camera_position = meta.camera_position[:] + camera_position = (camera_position.tolist() + if isinstance(camera_position, np.ndarray) + else camera_position) + + camera_rotation = meta.camera_rotation[:] + camera_rotation = (camera_rotation.tolist() + if isinstance(camera_rotation, np.ndarray) + else camera_rotation) + + led_position = meta.led_position[:] + led_position = (led_position.tolist() + if isinstance(led_position, np.ndarray) + else led_position) + + return RigGeometry( + equipment=meta.equipment, + monitor_position_mm=Coordinates(*monitor_position), + camera_position_mm=Coordinates(*camera_position), + led_position=Coordinates(*led_position), + monitor_rotation_deg=Coordinates(*monitor_rotation), + camera_rotation_deg=Coordinates(*camera_rotation) + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "RigGeometry": + rg = dict_repr['eye_tracking_rig_geometry'] + return RigGeometry( + equipment=rg['equipment'], + monitor_position_mm=Coordinates(*rg['monitor_position_mm']), + monitor_rotation_deg=Coordinates(*rg['monitor_rotation_deg']), + camera_position_mm=Coordinates(*rg['camera_position_mm']), + camera_rotation_deg=Coordinates(*rg['camera_rotation_deg']), + led_position=Coordinates(*rg['led_position']) + ) + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> Optional["RigGeometry"]: + query = f''' + SELECT oec.*, oect.name as config_type, equipment.name as + equipment_name + FROM ophys_sessions os + JOIN observatory_experiment_configs oec ON oec.equipment_id = + os.equipment_id + JOIN observatory_experiment_config_types oect ON oect.id = + oec.observatory_experiment_config_type_id + JOIN ophys_experiments oe ON oe.ophys_session_id = os.id + JOIN equipment ON equipment.id = oec.equipment_id + WHERE oe.id = {ophys_experiment_id} AND + oec.active_date <= os.date_of_acquisition AND + oect.name IN ('eye camera position', 'led position', 'screen position') + ''' # noqa E501 + # Get the raw data + rig_geometry = pd.read_sql(query, lims_db.get_connection()) + + if rig_geometry.empty: + # There is no rig geometry for this experiment + return None + + # Map the config types to new names + rig_geometry_config_type_map = { + 'eye camera position': 'camera', + 'screen position': 'monitor', + 'led position': 'led' + } + rig_geometry['config_type'] = rig_geometry['config_type'] \ + .map(rig_geometry_config_type_map) + + rig_geometry = cls._select_most_recent_geometry( + rig_geometry=rig_geometry) + + # Construct dictionary for positions + position = rig_geometry[['center_x_mm', 'center_y_mm', 'center_z_mm']] + position.index = [ + f'{v}_position_mm' if v != 'led' + else f'{v}_position' for v in position.index] + position = position.to_dict(orient='index') + position = { + config_type: + Coordinates( + values['center_x_mm'], + values['center_y_mm'], + values['center_z_mm']) + for config_type, values in position.items() + } + + # Construct dictionary for rotations + rotation = rig_geometry[['rotation_x_deg', 'rotation_y_deg', + 'rotation_z_deg']] + rotation = rotation[rotation.index != 'led'] + rotation.index = [f'{v}_rotation_deg' for v in rotation.index] + rotation = rotation.to_dict(orient='index') + rotation = { + config_type: + Coordinates( + values['rotation_x_deg'], + values['rotation_y_deg'], + values['rotation_z_deg'] + ) + for config_type, values in rotation.items() + } + + # Combine the dictionaries + rig_geometry = { + **position, + **rotation, + 'equipment': rig_geometry['equipment_name'].iloc[0] + } + return RigGeometry(**rig_geometry) + + @staticmethod + def _select_most_recent_geometry(rig_geometry: pd.DataFrame): + """There can be multiple geometry entries in LIMS for a rig. + Select most recent one. + + Parameters + ---------- + rig_geometry + Table of geometries for rig as returned by LIMS + + Notes + ---------- + The geometries in rig_geometry are assumed to precede the + date_of_acquisition of the session + (only relevant for retrieving from LIMS) + """ + rig_geometry = rig_geometry.sort_values('active_date', ascending=False) + rig_geometry = rig_geometry.groupby('config_type') \ + .apply(lambda x: x.iloc[0]) + return rig_geometry diff --git a/allensdk/brain_observatory/behavior/data_objects/licks.py b/allensdk/brain_observatory/behavior/data_objects/licks.py new file mode 100644 index 000000000..2dc4af332 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/licks.py @@ -0,0 +1,116 @@ +import logging +import warnings +from typing import Optional + +import pandas as pd +from pynwb import NWBFile, TimeSeries, ProcessingModule + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface + + +class Licks(DataObject, StimulusFileReadableInterface, NwbReadableInterface, + NwbWritableInterface): + _logger = logging.getLogger(__name__) + + def __init__(self, licks: pd.DataFrame): + """ + :param licks + dataframe containing the following columns: + - timestamps: float + stimulus timestamps in which there was a lick + - frame: int + frame number in which there was a lick + """ + super().__init__(name='licks', value=licks) + + @classmethod + def from_stimulus_file(cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps) -> "Licks": + """Get lick data from pkl file. + This function assumes that the first sensor in the list of + lick_sensors is the desired lick sensor. + + Since licks can occur outside of a trial context, the lick times + are extracted from the vsyncs and the frame number in `lick_events`. + Since we don't have a timestamp for when in "experiment time" the + vsync stream starts (from self.get_stimulus_timestamps), we compute + it by fitting a linear regression (frame number x time) for the + `start_trial` and `end_trial` events in the `trial_log`, to true + up these time streams. + + :returns: pd.DataFrame + Two columns: "time", which contains the sync time + of the licks that occurred in this session and "frame", + the frame numbers of licks that occurred in this session + """ + data = stimulus_file.data + + lick_frames = (data["items"]["behavior"]["lick_sensors"][0] + ["lick_events"]) + + # there's an occasional bug where the number of logged + # frames is one greater than the number of vsync intervals. + # If the animal licked on this last frame it will cause an + # error here. This fixes the problem. + # see: https://github.com/AllenInstitute/visual_behavior_analysis + # /issues/572 # noqa: E501 + # & https://github.com/AllenInstitute/visual_behavior_analysis + # /issues/379 # noqa:E501 + # + # This bugfix copied from + # https://github.com/AllenInstitute/visual_behavior_analysis/blob + # /master/visual_behavior/translator/foraging2/extract.py#L640-L647 + + if len(lick_frames) > 0: + if lick_frames[-1] == len(stimulus_timestamps.value): + lick_frames = lick_frames[:-1] + cls._logger.error('removed last lick - ' + 'it fell outside of stimulus_timestamps ' + 'range') + + lick_times = \ + [stimulus_timestamps.value[frame] for frame in lick_frames] + df = pd.DataFrame({"timestamps": lick_times, "frame": lick_frames}) + return cls(licks=df) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> Optional["Licks"]: + if 'licking' in nwbfile.processing: + lick_module = nwbfile.processing['licking'] + licks = lick_module.get_data_interface('licks') + + df = pd.DataFrame({ + 'timestamps': licks.timestamps[:], + 'frame': licks.data[:] + }) + else: + warnings.warn("This session " + f"'{int(nwbfile.identifier)}' has no rewards data.") + return None + return cls(licks=df) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + lick_timeseries = TimeSeries( + name='licks', + data=self.value['frame'].values, + timestamps=self.value['timestamps'].values, + description=('Timestamps and stimulus presentation ' + 'frame indices for lick events'), + unit='N/A' + ) + + # Add lick interface to nwb file, by way of a processing module: + licks_mod = ProcessingModule('licking', + 'Licking behavior processing module') + licks_mod.add_data_interface(lick_timeseries) + nwbfile.add_processing_module(licks_mod) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/__init__.py b/allensdk/brain_observatory/behavior/data_objects/metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/__init__.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/metadata/behavior_metadata.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_metadata.py similarity index 50% rename from allensdk/brain_observatory/behavior/metadata/behavior_metadata.py rename to allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_metadata.py index 493bbfda9..d7a77c923 100644 --- a/allensdk/brain_observatory/behavior/metadata/behavior_metadata.py +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_metadata.py @@ -1,16 +1,40 @@ -import abc import uuid -import warnings -from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, Optional import re import numpy as np -import pytz - -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_data_extractor_base import \ - BehaviorDataExtractorBase -from allensdk.brain_observatory.session_api_utils import compare_session_fields +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_session_uuid import \ + BehaviorSessionUUID +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.equipment import \ + Equipment +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.foraging_id import \ + ForagingId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.session_type import \ + SessionType +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.stimulus_frame_rate import \ + StimulusFrameRate +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.subject_metadata import \ + SubjectMetadata +from allensdk.brain_observatory.behavior.schemas import BehaviorMetadataSchema +from allensdk.brain_observatory.nwb import load_pynwb_extension +from allensdk.internal.api import PostgresQueryMixin description_dict = { # key is a regex and value is returned on match @@ -151,297 +175,146 @@ def get_task_parameters(data: Dict) -> Dict: return task_parameters -class BehaviorMetadata: +class BehaviorMetadata(DataObject, LimsReadableInterface, + JsonReadableInterface, + NwbReadableInterface, + JsonWritableInterface, + NwbWritableInterface): """Container class for behavior metadata""" - def __init__(self, extractor: BehaviorDataExtractorBase, - stimulus_timestamps: np.ndarray, - behavior_stimulus_file: dict): + def __init__(self, + subject_metadata: SubjectMetadata, + behavior_session_id: BehaviorSessionId, + equipment: Equipment, + stimulus_frame_rate: StimulusFrameRate, + session_type: SessionType, + behavior_session_uuid: BehaviorSessionUUID): + super().__init__(name='behavior_metadata', value=self) + self._subject_metadata = subject_metadata + self._behavior_session_id = behavior_session_id + self._equipment = equipment + self._stimulus_frame_rate = stimulus_frame_rate + self._session_type = session_type + self._behavior_session_uuid = behavior_session_uuid - self._extractor = extractor - self._stimulus_timestamps = stimulus_timestamps - self._behavior_stimulus_file = behavior_stimulus_file self._exclude_from_equals = set() - @property - def equipment_name(self) -> str: - return self._extractor.get_equipment_name() + @classmethod + def from_lims( + cls, + behavior_session_id: BehaviorSessionId, + lims_db: PostgresQueryMixin + ) -> "BehaviorMetadata": + subject_metadata = SubjectMetadata.from_lims( + behavior_session_id=behavior_session_id, lims_db=lims_db) + equipment = Equipment.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + + stimulus_file = StimulusFile.from_lims( + db=lims_db, behavior_session_id=behavior_session_id.value) + stimulus_frame_rate = StimulusFrameRate.from_stimulus_file( + stimulus_file=stimulus_file) + session_type = SessionType.from_stimulus_file( + stimulus_file=stimulus_file) + + foraging_id = ForagingId.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + behavior_session_uuid = BehaviorSessionUUID.from_stimulus_file( + stimulus_file=stimulus_file)\ + .validate(behavior_session_id=behavior_session_id.value, + foraging_id=foraging_id.value, + stimulus_file=stimulus_file) + + return cls( + subject_metadata=subject_metadata, + behavior_session_id=behavior_session_id, + equipment=equipment, + stimulus_frame_rate=stimulus_frame_rate, + session_type=session_type, + behavior_session_uuid=behavior_session_uuid, + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "BehaviorMetadata": + subject_metadata = SubjectMetadata.from_json(dict_repr=dict_repr) + behavior_session_id = BehaviorSessionId.from_json(dict_repr=dict_repr) + equipment = Equipment.from_json(dict_repr=dict_repr) + + stimulus_file = StimulusFile.from_json(dict_repr=dict_repr) + stimulus_frame_rate = StimulusFrameRate.from_stimulus_file( + stimulus_file=stimulus_file) + session_type = SessionType.from_stimulus_file( + stimulus_file=stimulus_file) + session_uuid = BehaviorSessionUUID.from_stimulus_file( + stimulus_file=stimulus_file) + + return cls( + subject_metadata=subject_metadata, + behavior_session_id=behavior_session_id, + equipment=equipment, + stimulus_frame_rate=stimulus_frame_rate, + session_type=session_type, + behavior_session_uuid=session_uuid, + ) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "BehaviorMetadata": + subject_metadata = SubjectMetadata.from_nwb(nwbfile=nwbfile) + + behavior_session_id = BehaviorSessionId.from_nwb(nwbfile=nwbfile) + equipment = Equipment.from_nwb(nwbfile=nwbfile) + stimulus_frame_rate = StimulusFrameRate.from_nwb(nwbfile=nwbfile) + session_type = SessionType.from_nwb(nwbfile=nwbfile) + session_uuid = BehaviorSessionUUID.from_nwb(nwbfile=nwbfile) + + return cls( + subject_metadata=subject_metadata, + behavior_session_id=behavior_session_id, + equipment=equipment, + stimulus_frame_rate=stimulus_frame_rate, + session_type=session_type, + behavior_session_uuid=session_uuid + ) @property - def sex(self) -> str: - return self._extractor.get_sex() - - @property - def age_in_days(self) -> Optional[int]: - """Converts the age cod into a numeric days representation""" - - age = self._extractor.get_age() - return self.parse_age_in_days(age=age, warn=True) + def equipment(self) -> Equipment: + return self._equipment @property def stimulus_frame_rate(self) -> float: - return self._get_frame_rate(timestamps=self._stimulus_timestamps) + return self._stimulus_frame_rate.value @property def session_type(self) -> str: - return self._extractor.get_stimulus_name() - - @property - def date_of_acquisition(self) -> datetime: - """Return the timestamp for when experiment was started in UTC - - NOTE: This method will only get acquisition datetime from - extractor (data from LIMS) methods. As a sanity check, - it will also read the acquisition datetime from the behavior stimulus - (*.pkl) file and raise a warning if the date differs too much from the - datetime obtained from the behavior stimulus (*.pkl) file. - - :rtype: datetime - """ - extractor_acq_date = self._extractor.get_date_of_acquisition() - - pkl_data = self._behavior_stimulus_file - pkl_raw_acq_date = pkl_data["start_time"] - if isinstance(pkl_raw_acq_date, datetime): - pkl_acq_date = pytz.utc.localize(pkl_raw_acq_date) - - elif isinstance(pkl_raw_acq_date, (int, float)): - # We are dealing with an older pkl file where the acq time is - # stored as a Unix style timestamp string - parsed_pkl_acq_date = datetime.fromtimestamp(pkl_raw_acq_date) - pkl_acq_date = pytz.utc.localize(parsed_pkl_acq_date) - else: - pkl_acq_date = None - warnings.warn( - "Could not parse the acquisition datetime " - f"({pkl_raw_acq_date}) found in the following stimulus *.pkl: " - f"{self._extractor.get_behavior_stimulus_file()}" - ) - - if pkl_acq_date: - acq_start_diff = ( - extractor_acq_date - pkl_acq_date).total_seconds() - # If acquisition dates differ by more than an hour - if abs(acq_start_diff) > 3600: - session_id = self._extractor.get_behavior_session_id() - warnings.warn( - "The `date_of_acquisition` field in LIMS " - f"({extractor_acq_date}) for behavior session " - f"({session_id}) deviates by more " - f"than an hour from the `start_time` ({pkl_acq_date}) " - "specified in the associated stimulus *.pkl file: " - f"{self._extractor.get_behavior_stimulus_file()}" - ) - return extractor_acq_date - - @property - def reporter_line(self) -> Optional[str]: - reporter_line = self._extractor.get_reporter_line() - return self.parse_reporter_line(reporter_line=reporter_line, warn=True) - - @property - def cre_line(self) -> Optional[str]: - """Parses cre_line from full_genotype""" - cre_line = self.parse_cre_line(full_genotype=self.full_genotype, - warn=True) - return cre_line + return self._session_type.value @property def behavior_session_uuid(self) -> Optional[uuid.UUID]: - """Get the universally unique identifier (UUID) - """ - data = self._behavior_stimulus_file - behavior_pkl_uuid = data.get("session_uuid") - - behavior_session_id = self._extractor.get_behavior_session_id() - foraging_id = self._extractor.get_foraging_id() - - # Sanity check to ensure that pkl file data matches up with - # the behavior session that the pkl file has been associated with. - assert_err_msg = ( - f"The behavior session UUID ({behavior_pkl_uuid}) in the " - f"behavior stimulus *.pkl file " - f"({self._extractor.get_behavior_stimulus_file()}) does " - f"does not match the foraging UUID ({foraging_id}) for " - f"behavior session: {behavior_session_id}") - assert behavior_pkl_uuid == foraging_id, assert_err_msg - - if behavior_pkl_uuid is None: - bs_uuid = None - else: - bs_uuid = uuid.UUID(behavior_pkl_uuid) - return bs_uuid - - @property - def driver_line(self) -> List[str]: - return sorted(self._extractor.get_driver_line()) + return self._behavior_session_uuid.value @property - def mouse_id(self) -> int: - return self._extractor.get_mouse_id() - - @property - def full_genotype(self) -> str: - return self._extractor.get_full_genotype() + def behavior_session_id(self) -> int: + return self._behavior_session_id.value @property - def behavior_session_id(self) -> int: - return self._extractor.get_behavior_session_id() - - def get_extractor(self): - return self._extractor - - @abc.abstractmethod - def to_dict(self) -> dict: - """Returns dict representation of all properties in class""" - vars_ = vars(BehaviorMetadata) - return self._get_properties(vars_=vars_) - - @staticmethod - def _get_frame_rate(timestamps: np.ndarray): - return np.round(1 / np.mean(np.diff(timestamps)), 0) - - @staticmethod - def parse_cre_line(full_genotype: str, warn=False) -> Optional[str]: - """ - Parameters - ---------- - full_genotype - formatted from LIMS, e.g. - Vip-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt - warn - Whether to output warning if parsing fails - - Returns - ---------- - cre_line - just the Cre line, e.g. Vip-IRES-Cre, or None if not possible to - parse - """ - if ';' not in full_genotype: - if warn: - warnings.warn('Unable to parse cre_line from full_genotype') - return None - return full_genotype.split(';')[0].replace('/wt', '') - - @staticmethod - def parse_age_in_days(age: str, warn=False) -> Optional[int]: - """Converts the age code into a numeric days representation - - Parameters - ---------- - age - age code, ie P123 - warn - Whether to output warning if parsing fails - """ - if not age.startswith('P'): - if warn: - warnings.warn('Could not parse numeric age from age code ' - '(age code does not start with "P")') - return None - - match = re.search(r'\d+', age) - - if match is None: - if warn: - warnings.warn('Could not parse numeric age from age code ' - '(no numeric values found in age code)') - return None - - start, end = match.span() - return int(age[start:end]) - - @staticmethod - def parse_reporter_line(reporter_line: Optional[List[str]], - warn=False) -> Optional[str]: - """There can be multiple reporter lines, so it is returned from LIMS - as a list. But there shouldn't be more than 1 for behavior. This - tries to convert to str - - Parameters - ---------- - reporter_line - List of reporter line - warn - Whether to output warnings if parsing fails - - Returns - --------- - single reporter line, or None if not possible - """ - if reporter_line is None: - if warn: - warnings.warn('Error parsing reporter line. It is null.') - return None - - if len(reporter_line) == 0: - if warn: - warnings.warn('Error parsing reporter line. ' - 'The array is empty') - return None - - if isinstance(reporter_line, str): - return reporter_line - - if len(reporter_line) > 1: - if warn: - warnings.warn('More than 1 reporter line. Returning the first ' - 'one') - - return reporter_line[0] - - def _get_properties(self, vars_: dict): - """Returns all property names and values""" - return {name: getattr(self, name) for name, value in vars_.items() - if isinstance(value, property)} - - def __eq__(self, other): - if not isinstance(other, (BehaviorMetadata, dict)): - msg = f'Do not know how to compare with type {type(other)}' - raise NotImplementedError(msg) - - properties_self = self.to_dict() - - if isinstance(other, dict): - properties_other = other - else: - properties_other = other.to_dict() - - for p in properties_self: - if p in self._exclude_from_equals: - continue - - x1 = properties_self[p] - x2 = properties_other[p] - - try: - compare_session_fields(x1=x1, x2=x2) - except AssertionError: - return False - return True - - @staticmethod - def parse_indicator(reporter_line: Optional[str], warn=False) -> Optional[ - str]: - """Parses indicator from reporter""" - reporter_substring_indicator_map = { - 'GCaMP6f': 'GCaMP6f', - 'GC6f': 'GCaMP6f', - 'GCaMP6s': 'GCaMP6s' - } - if reporter_line is None: - if warn: - warnings.warn( - 'Could not parse indicator from reporter because ' - 'there is no reporter') - return None - - for substr, indicator in reporter_substring_indicator_map.items(): - if substr in reporter_line: - return indicator - - if warn: - warnings.warn( - 'Could not parse indicator from reporter because none' - 'of the expected substrings were found in the reporter') - return None + def subject_metadata(self): + return self._subject_metadata + + def to_json(self) -> dict: + pass + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + self._subject_metadata.to_nwb(nwbfile=nwbfile) + self._equipment.to_nwb(nwbfile=nwbfile) + extension = load_pynwb_extension(BehaviorMetadataSchema, + 'ndx-aibs-behavior-ophys') + nwb_metadata = extension( + name='metadata', + behavior_session_id=self.behavior_session_id, + behavior_session_uuid=str(self.behavior_session_uuid), + stimulus_frame_rate=self.stimulus_frame_rate, + session_type=self.session_type, + equipment_name=self.equipment.value + ) + nwbfile.add_lab_meta_data(nwb_metadata) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_id.py new file mode 100644 index 000000000..058462494 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_id.py @@ -0,0 +1,54 @@ +from pynwb import NWBFile + +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + JsonWritableInterface +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_objects import DataObject + + +def from_lims_cache_key(cls, db, ophys_experiment_id: int): + return hashkey(ophys_experiment_id) + + +class BehaviorSessionId(DataObject, LimsReadableInterface, + JsonReadableInterface, + NwbReadableInterface, + JsonWritableInterface): + def __init__(self, behavior_session_id: int): + super().__init__(name="behavior_session_id", value=behavior_session_id) + + @classmethod + def from_json(cls, dict_repr: dict) -> "BehaviorSessionId": + return cls(behavior_session_id=dict_repr["behavior_session_id"]) + + def to_json(self) -> dict: + return {"behavior_session_id": self.value} + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, db: PostgresQueryMixin, + ophys_experiment_id: int + ) -> "BehaviorSessionId": + query = f""" + SELECT bs.id + FROM ophys_experiments oe + -- every ophys_experiment should have an ophys_session + JOIN ophys_sessions os ON oe.ophys_session_id = os.id + JOIN behavior_sessions bs ON os.id = bs.ophys_session_id + WHERE oe.id = {ophys_experiment_id}; + """ + behavior_session_id = db.fetchone(query, strict=True) + return cls(behavior_session_id=behavior_session_id) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "BehaviorSessionId": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(behavior_session_id=metadata.behavior_session_id) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_uuid.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_uuid.py new file mode 100644 index 000000000..0129e8d10 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_session_uuid.py @@ -0,0 +1,49 @@ +import uuid +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + NwbReadableInterface, StimulusFileReadableInterface + + +class BehaviorSessionUUID(DataObject, StimulusFileReadableInterface, + NwbReadableInterface): + """the universally unique identifier (UUID)""" + def __init__(self, behavior_session_uuid: Optional[uuid.UUID]): + super().__init__(name="behavior_session_uuid", + value=behavior_session_uuid) + + @classmethod + def from_stimulus_file( + cls, stimulus_file: StimulusFile) -> "BehaviorSessionUUID": + id = stimulus_file.data.get('session_uuid') + if id: + id = uuid.UUID(id) + return cls(behavior_session_uuid=id) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "BehaviorSessionUUID": + metadata = nwbfile.lab_meta_data['metadata'] + id = uuid.UUID(metadata.behavior_session_uuid) + return cls(behavior_session_uuid=id) + + def validate(self, behavior_session_id: int, + foraging_id: int, + stimulus_file: StimulusFile) -> "BehaviorSessionUUID": + """ + Sanity check to ensure that pkl file data matches up with + the behavior session that the pkl file has been associated with. + """ + assert_err_msg = ( + f"The behavior session UUID ({self.value}) in the " + f"behavior stimulus *.pkl file " + f"({stimulus_file.filepath}) does " + f"does not match the foraging UUID ({foraging_id}) for " + f"behavior session: {behavior_session_id}") + assert self.value == foraging_id, assert_err_msg + + return self diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/date_of_acquisition.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/date_of_acquisition.py new file mode 100644 index 000000000..7f2789c9a --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/date_of_acquisition.py @@ -0,0 +1,116 @@ +import warnings +from datetime import datetime + +import pytz +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class DateOfAcquisition(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + """timestamp for when experiment was started in UTC""" + def __init__(self, date_of_acquisition: datetime): + super().__init__(name="date_of_acquisition", value=date_of_acquisition) + + @classmethod + def from_json(cls, dict_repr: dict) -> "DateOfAcquisition": + doa = dict_repr['date_of_acquisition'] + doa = datetime.strptime(doa, "%Y-%m-%d %H:%M:%S") + tz = pytz.timezone("America/Los_Angeles") + doa = tz.localize(doa) + + # NOTE: LIMS writes to JSON in local time. Needs to be converted to UTC + doa = doa.astimezone(pytz.utc) + + return cls(date_of_acquisition=doa) + + @classmethod + def from_lims( + cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "DateOfAcquisition": + query = """ + SELECT bs.date_of_acquisition + FROM behavior_sessions bs + WHERE bs.id = {}; + """.format(behavior_session_id) + + experiment_date = lims_db.fetchone(query, strict=True) + experiment_date = cls._postprocess_lims_datetime( + datetime=experiment_date) + return cls(date_of_acquisition=experiment_date) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "DateOfAcquisition": + return cls(date_of_acquisition=nwbfile.session_start_time) + + def validate(self, stimulus_file: StimulusFile, + behavior_session_id: int) -> "DateOfAcquisition": + """raise a warning if the date differs too much from the + datetime obtained from the behavior stimulus (*.pkl) file.""" + pkl_data = stimulus_file.data + pkl_raw_acq_date = pkl_data["start_time"] + if isinstance(pkl_raw_acq_date, datetime): + pkl_acq_date = pytz.utc.localize(pkl_raw_acq_date) + + elif isinstance(pkl_raw_acq_date, (int, float)): + # We are dealing with an older pkl file where the acq time is + # stored as a Unix style timestamp string + parsed_pkl_acq_date = datetime.fromtimestamp(pkl_raw_acq_date) + pkl_acq_date = pytz.utc.localize(parsed_pkl_acq_date) + else: + pkl_acq_date = None + warnings.warn( + "Could not parse the acquisition datetime " + f"({pkl_raw_acq_date}) found in the following stimulus *.pkl: " + f"{stimulus_file.filepath}" + ) + + if pkl_acq_date: + acq_start_diff = ( + self.value - pkl_acq_date).total_seconds() + # If acquisition dates differ by more than an hour + if abs(acq_start_diff) > 3600: + session_id = behavior_session_id + warnings.warn( + "The `date_of_acquisition` field in LIMS " + f"({self.value}) for behavior session " + f"({session_id}) deviates by more " + f"than an hour from the `start_time` ({pkl_acq_date}) " + "specified in the associated stimulus *.pkl file: " + f"{stimulus_file.filepath}" + ) + return self + + @staticmethod + def _postprocess_lims_datetime(datetime: datetime): + """Applies postprocessing to datetime read from LIMS""" + # add utc tz + datetime = pytz.utc.localize(datetime) + + return datetime + + +class DateOfAcquisitionOphys(DateOfAcquisition): + """Ophys experiments read date of acquisition from the ophys_sessions + table in LIMS instead of the behavior_sessions table""" + + @classmethod + def from_lims( + cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "DateOfAcquisitionOphys": + query = f""" + SELECT os.date_of_acquisition + FROM ophys_experiments oe + JOIN ophys_sessions os ON oe.ophys_session_id = os.id + WHERE oe.id = {ophys_experiment_id}; + """ + doa = lims_db.fetchone(query=query) + doa = cls._postprocess_lims_datetime( + datetime=doa) + return DateOfAcquisitionOphys(date_of_acquisition=doa) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/equipment.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/equipment.py new file mode 100644 index 000000000..4b8020fc6 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/equipment.py @@ -0,0 +1,73 @@ +from enum import Enum + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class EquipmentType(Enum): + MESOSCOPE = 'MESOSCOPE' + OTHER = 'OTHER' + + +class Equipment(DataObject, JsonReadableInterface, LimsReadableInterface, + NwbReadableInterface, JsonWritableInterface, + NwbWritableInterface): + """the name of the experimental rig.""" + def __init__(self, equipment_name: str): + super().__init__(name="equipment_name", value=equipment_name) + + @classmethod + def from_json(cls, dict_repr: dict) -> "Equipment": + return cls(equipment_name=dict_repr["rig_name"]) + + def to_json(self) -> dict: + return {"eqipment_name": self.value} + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "Equipment": + query = f""" + SELECT e.name AS device_name + FROM behavior_sessions bs + JOIN equipment e ON e.id = bs.equipment_id + WHERE bs.id = {behavior_session_id}; + """ + equipment_name = lims_db.fetchone(query, strict=True) + return cls(equipment_name=equipment_name) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Equipment": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(equipment_name=metadata.equipment_name) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + if self.type == EquipmentType.MESOSCOPE: + device_config = { + "name": self.value, + "description": "Allen Brain Observatory - Mesoscope 2P Rig" + } + else: + device_config = { + "name": self.value, + "description": "Allen Brain Observatory - Scientifica 2P " + "Rig", + "manufacturer": "Scientifica" + } + nwbfile.create_device(**device_config) + return nwbfile + + @property + def type(self): + if self.value.startswith('MESO'): + et = EquipmentType.MESOSCOPE + else: + et = EquipmentType.OTHER + return et diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/foraging_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/foraging_id.py new file mode 100644 index 000000000..3b2bae420 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/foraging_id.py @@ -0,0 +1,32 @@ +import uuid + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class ForagingId(DataObject, LimsReadableInterface, JsonReadableInterface): + """Foraging id""" + def __init__(self, foraging_id: uuid.UUID): + super().__init__(name="foraging_id", value=foraging_id) + + @classmethod + def from_json(cls, dict_repr: dict) -> "ForagingId": + pass + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "ForagingId": + query = f""" + SELECT + foraging_id + FROM + behavior_sessions + WHERE + behavior_sessions.id = {behavior_session_id}; + """ + foraging_id = lims_db.fetchone(query, strict=True) + foraging_id = uuid.UUID(foraging_id) + return cls(foraging_id=foraging_id) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/session_type.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/session_type.py new file mode 100644 index 000000000..67c43d230 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/session_type.py @@ -0,0 +1,36 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + NwbReadableInterface, StimulusFileReadableInterface + + +class SessionType(DataObject, StimulusFileReadableInterface, + NwbReadableInterface): + """the stimulus set used""" + def __init__(self, session_type: str): + super().__init__(name="session_type", value=session_type) + + @classmethod + def from_stimulus_file( + cls, + stimulus_file: StimulusFile) -> "SessionType": + try: + stimulus_name = \ + stimulus_file.data["items"]["behavior"]["cl_params"]["stage"] + except KeyError: + raise RuntimeError( + f"Could not obtain stimulus_name/stage information from " + f"the *.pkl file ({stimulus_file.filepath}) " + f"for the behavior session to save as NWB! The " + f"following series of nested keys did not work: " + f"['items']['behavior']['cl_params']['stage']" + ) + return cls(session_type=stimulus_name) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "SessionType": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(session_type=metadata.session_type) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/stimulus_frame_rate.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/stimulus_frame_rate.py new file mode 100644 index 000000000..a9e75b514 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/stimulus_frame_rate.py @@ -0,0 +1,33 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + NwbReadableInterface, StimulusFileReadableInterface +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .stimulus_timestamps.stimulus_timestamps import \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.timestamps.util import \ + calc_frame_rate + + +class StimulusFrameRate(DataObject, StimulusFileReadableInterface, + NwbReadableInterface): + """Stimulus frame rate""" + def __init__(self, stimulus_frame_rate: float): + super().__init__(name="stimulus_frame_rate", value=stimulus_frame_rate) + + @classmethod + def from_stimulus_file( + cls, + stimulus_file: StimulusFile) -> "StimulusFrameRate": + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + frame_rate = calc_frame_rate(timestamps=stimulus_timestamps.value) + return cls(stimulus_frame_rate=frame_rate) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "StimulusFrameRate": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(stimulus_frame_rate=metadata.stimulus_frame_rate) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_ophys_metadata.py b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_ophys_metadata.py new file mode 100644 index 000000000..1a28168e0 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/behavior_ophys_metadata.py @@ -0,0 +1,167 @@ +from typing import Union + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_metadata import \ + BehaviorMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata\ + .multi_plane_metadata import \ + MultiplaneMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_experiment_metadata import \ + OphysExperimentMetadata +from allensdk.brain_observatory.behavior.schemas import \ + OphysBehaviorMetadataSchema +from allensdk.brain_observatory.nwb import load_pynwb_extension +from allensdk.internal.api import PostgresQueryMixin + + +class BehaviorOphysMetadata(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface, + NwbWritableInterface): + def __init__(self, behavior_metadata: BehaviorMetadata, + ophys_metadata: Union[OphysExperimentMetadata, + MultiplaneMetadata]): + super().__init__(name='behavior_ophys_metadata', value=self) + + self._behavior_metadata = behavior_metadata + self._ophys_metadata = ophys_metadata + + @property + def behavior_metadata(self) -> BehaviorMetadata: + return self._behavior_metadata + + @property + def ophys_metadata(self) -> Union["OphysExperimentMetadata", + "MultiplaneMetadata"]: + return self._ophys_metadata + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin, + is_multiplane=False) -> "BehaviorOphysMetadata": + """ + + Parameters + ---------- + ophys_experiment_id + lims_db + is_multiplane + Whether to fetch metadata for an experiment that is part of a + container containing multiple imaging planes + """ + behavior_session_id = BehaviorSessionId.from_lims( + ophys_experiment_id=ophys_experiment_id, db=lims_db) + + behavior_metadata = BehaviorMetadata.from_lims( + behavior_session_id=behavior_session_id, lims_db=lims_db) + + if is_multiplane: + ophys_metadata = MultiplaneMetadata.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + else: + ophys_metadata = OphysExperimentMetadata.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + + return cls(behavior_metadata=behavior_metadata, + ophys_metadata=ophys_metadata) + + @classmethod + def from_json(cls, dict_repr: dict, + is_multiplane=False) -> "BehaviorOphysMetadata": + """ + + Parameters + ---------- + dict_repr + is_multiplane + Whether to fetch metadata for an experiment that is part of a + container containing multiple imaging planes + + Returns + ------- + + """ + behavior_metadata = BehaviorMetadata.from_json(dict_repr=dict_repr) + + if is_multiplane: + ophys_metadata = MultiplaneMetadata.from_json( + dict_repr=dict_repr) + else: + ophys_metadata = OphysExperimentMetadata.from_json( + dict_repr=dict_repr) + + return cls(behavior_metadata=behavior_metadata, + ophys_metadata=ophys_metadata) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile, + is_multiplane=False) -> "BehaviorOphysMetadata": + """ + + Parameters + ---------- + nwbfile + is_multiplane + Whether to fetch metadata for an experiment that is part of a + container containing multiple imaging planes + """ + behavior_metadata = BehaviorMetadata.from_nwb(nwbfile=nwbfile) + + if is_multiplane: + ophys_metadata = MultiplaneMetadata.from_nwb( + nwbfile=nwbfile) + else: + ophys_metadata = OphysExperimentMetadata.from_nwb( + nwbfile=nwbfile) + + return cls(behavior_metadata=behavior_metadata, + ophys_metadata=ophys_metadata) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + self._behavior_metadata.subject_metadata.to_nwb(nwbfile=nwbfile) + self._behavior_metadata.equipment.to_nwb(nwbfile=nwbfile) + + nwb_extension = load_pynwb_extension( + OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys') + + behavior_meta = self._behavior_metadata + ophys_meta = self._ophys_metadata + + if isinstance(ophys_meta, MultiplaneMetadata): + imaging_plane_group = ophys_meta.imaging_plane_group + imaging_plane_group_count = ophys_meta.imaging_plane_group_count + else: + imaging_plane_group_count = 0 + imaging_plane_group = -1 + + nwb_metadata = nwb_extension( + name='metadata', + ophys_session_id=ophys_meta.ophys_session_id, + field_of_view_width=ophys_meta.field_of_view_shape.width, + field_of_view_height=ophys_meta.field_of_view_shape.height, + imaging_plane_group=imaging_plane_group, + imaging_plane_group_count=imaging_plane_group_count, + stimulus_frame_rate=behavior_meta.stimulus_frame_rate, + experiment_container_id=ophys_meta.experiment_container_id, + ophys_experiment_id=ophys_meta.ophys_experiment_id, + session_type=behavior_meta.session_type, + equipment_name=behavior_meta.equipment.value, + imaging_depth=ophys_meta.imaging_depth, + behavior_session_uuid=str(behavior_meta.behavior_session_uuid), + behavior_session_id=behavior_meta.behavior_session_id + ) + nwbfile.add_lab_meta_data(nwb_metadata) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/__init__.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/experiment_container_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/experiment_container_id.py new file mode 100644 index 000000000..dc358de3e --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/experiment_container_id.py @@ -0,0 +1,35 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class ExperimentContainerId(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + """"experiment container id""" + def __init__(self, experiment_container_id: int): + super().__init__(name='experiment_container_id', + value=experiment_container_id) + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "ExperimentContainerId": + query = """ + SELECT visual_behavior_experiment_container_id + FROM ophys_experiments_visual_behavior_experiment_containers + WHERE ophys_experiment_id = {}; + """.format(ophys_experiment_id) + container_id = lims_db.fetchone(query, strict=False) + return cls(experiment_container_id=container_id) + + @classmethod + def from_json(cls, dict_repr: dict) -> "ExperimentContainerId": + return cls(experiment_container_id=dict_repr['container_id']) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "ExperimentContainerId": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(experiment_container_id=metadata.experiment_container_id) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/field_of_view_shape.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/field_of_view_shape.py new file mode 100644 index 000000000..f085e4aca --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/field_of_view_shape.py @@ -0,0 +1,48 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class FieldOfViewShape(DataObject, LimsReadableInterface, + NwbReadableInterface, JsonReadableInterface): + def __init__(self, height: int, width: int): + super().__init__(name='field_of_view_shape', value=self) + + self._height = height + self._width = width + + @property + def height(self): + return self._height + + @property + def width(self): + return self._width + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "FieldOfViewShape": + query = f""" + SELECT oe.movie_width as width, oe.movie_height as height + FROM ophys_experiments oe + WHERE oe.id = {ophys_experiment_id}; + """ + df = lims_db.select(query=query) + height = df.iloc[0]['height'] + width = df.iloc[0]['width'] + return cls(height=height, width=width) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "FieldOfViewShape": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(height=metadata.field_of_view_height, + width=metadata.field_of_view_width) + + @classmethod + def from_json(cls, dict_repr: dict) -> "FieldOfViewShape": + return cls(height=dict_repr['movie_height'], + width=dict_repr['movie_width']) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_depth.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_depth.py new file mode 100644 index 000000000..f5b265a2d --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_depth.py @@ -0,0 +1,35 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class ImagingDepth(DataObject, LimsReadableInterface, NwbReadableInterface, + JsonReadableInterface): + def __init__(self, imaging_depth: int): + super().__init__(name='imaging_depth', value=imaging_depth) + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "ImagingDepth": + query = """ + SELECT id.depth + FROM ophys_experiments oe + JOIN ophys_sessions os ON oe.ophys_session_id = os.id + LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id + WHERE oe.id = {}; + """.format(ophys_experiment_id) + imaging_depth = lims_db.fetchone(query, strict=True) + return cls(imaging_depth=imaging_depth) + + @classmethod + def from_json(cls, dict_repr: dict) -> "ImagingDepth": + return cls(imaging_depth=dict_repr['targeted_depth']) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "ImagingDepth": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(imaging_depth=metadata.imaging_depth) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_plane.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_plane.py new file mode 100644 index 000000000..7c48b6df3 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/imaging_plane.py @@ -0,0 +1,107 @@ +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.reporter_line import \ + ReporterLine +from allensdk.brain_observatory.behavior.data_objects.timestamps \ + .ophys_timestamps import OphysTimestamps +from allensdk.brain_observatory.behavior.data_objects.timestamps.util import \ + calc_frame_rate +from allensdk.internal.api import PostgresQueryMixin + + +class ImagingPlane(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + def __init__(self, ophys_frame_rate: float, + targeted_structure: str, + excitation_lambda: float, + indicator: Optional[str]): + super().__init__(name='imaging_plane', value=self) + self._ophys_frame_rate = ophys_frame_rate + self._targeted_structure = targeted_structure + self._excitation_lambda = excitation_lambda + self._indicator = indicator + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin, + ophys_timestamps: OphysTimestamps, + excitation_lambda=910.0) -> "ImagingPlane": + behavior_session_id = BehaviorSessionId.from_lims( + db=lims_db, ophys_experiment_id=ophys_experiment_id) + ophys_frame_rate = calc_frame_rate(timestamps=ophys_timestamps.value) + targeted_structure = cls._get_targeted_structure_from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + reporter_line = ReporterLine.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + indicator = reporter_line.parse_indicator(warn=True) + return cls(ophys_frame_rate=ophys_frame_rate, + targeted_structure=targeted_structure, + excitation_lambda=excitation_lambda, + indicator=indicator) + + @classmethod + def from_json(cls, dict_repr: dict, + ophys_timestamps: OphysTimestamps, + excitation_lambda=910.0) -> "ImagingPlane": + targeted_structure = dict_repr['targeted_structure'] + ophys_fame_rate = calc_frame_rate(timestamps=ophys_timestamps.value) + reporter_line = ReporterLine.from_json(dict_repr=dict_repr) + indicator = reporter_line.parse_indicator(warn=True) + return cls(targeted_structure=targeted_structure, + ophys_frame_rate=ophys_fame_rate, + excitation_lambda=excitation_lambda, + indicator=indicator) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "ImagingPlane": + ophys_module = nwbfile.processing['ophys'] + image_seg = ophys_module.data_interfaces['image_segmentation'] + imaging_plane = image_seg.plane_segmentations[ + 'cell_specimen_table'].imaging_plane + ophys_frame_rate = imaging_plane.imaging_rate + targeted_structure = imaging_plane.location + excitation_lambda = imaging_plane.excitation_lambda + + reporter_line = ReporterLine.from_nwb(nwbfile=nwbfile) + indicator = reporter_line.parse_indicator(warn=True) + return cls(ophys_frame_rate=ophys_frame_rate, + targeted_structure=targeted_structure, + excitation_lambda=excitation_lambda, + indicator=indicator) + + @property + def ophys_frame_rate(self) -> float: + return self._ophys_frame_rate + + @property + def targeted_structure(self) -> str: + return self._targeted_structure + + @property + def excitation_lambda(self) -> float: + return self._excitation_lambda + + @property + def indicator(self) -> Optional[str]: + return self._indicator + + @staticmethod + def _get_targeted_structure_from_lims(ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> str: + query = """ + SELECT st.acronym + FROM ophys_experiments oe + LEFT JOIN structures st ON st.id = oe.targeted_structure_id + WHERE oe.id = {}; + """.format(ophys_experiment_id) + targeted_structure = lims_db.fetchone(query, strict=True) + return targeted_structure diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/__init__.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/imaging_plane_group.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/imaging_plane_group.py new file mode 100644 index 000000000..176325c58 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/imaging_plane_group.py @@ -0,0 +1,77 @@ +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class ImagingPlaneGroup(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + def __init__(self, plane_group: int, plane_group_count: int): + super().__init__(name='plane_group', value=self) + self._plane_group = plane_group + self._plane_group_count = plane_group_count + + @property + def plane_group(self): + return self._plane_group + + @property + def plane_group_count(self): + return self._plane_group_count + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> \ + Optional["ImagingPlaneGroup"]: + """ + + Parameters + ---------- + ophys_experiment_id + lims_db + + Returns + ------- + ImagingPlaneGroup instance if ophys_experiment given by + ophys_experiment_id is part of a plane group + else None + + """ + query = f''' + SELECT oe.id as ophys_experiment_id, pg.group_order AS plane_group + FROM ophys_experiments oe + JOIN ophys_sessions os ON oe.ophys_session_id = os.id + JOIN ophys_imaging_plane_groups pg + ON pg.id = oe.ophys_imaging_plane_group_id + WHERE os.id = ( + SELECT oe.ophys_session_id + FROM ophys_experiments oe + WHERE oe.id = {ophys_experiment_id} + ) + ''' + df = lims_db.select(query=query) + if df.empty: + return None + df = df.set_index('ophys_experiment_id') + plane_group = df.loc[ophys_experiment_id, 'plane_group'] + plane_group_count = df['plane_group'].nunique() + return cls(plane_group=plane_group, + plane_group_count=plane_group_count) + + @classmethod + def from_json(cls, dict_repr: dict) -> "ImagingPlaneGroup": + plane_group = dict_repr['imaging_plane_group'] + plane_group_count = dict_repr['plane_group_count'] + return cls(plane_group=plane_group, + plane_group_count=plane_group_count) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "ImagingPlaneGroup": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(plane_group=metadata.imaging_plane_group, + plane_group_count=metadata.imaging_plane_group_count) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/multi_plane_metadata.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/multi_plane_metadata.py new file mode 100644 index 000000000..e1f5317ac --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/multi_plane_metadata.py @@ -0,0 +1,102 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.experiment_container_id import \ + ExperimentContainerId +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.field_of_view_shape import \ + FieldOfViewShape +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.imaging_depth import \ + ImagingDepth +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.multi_plane_metadata \ + .imaging_plane_group import \ + ImagingPlaneGroup +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.ophys_experiment_metadata import \ + OphysExperimentMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.ophys_session_id import \ + OphysSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .ophys_experiment_metadata.project_code import \ + ProjectCode +from allensdk.internal.api import PostgresQueryMixin + + +class MultiplaneMetadata(OphysExperimentMetadata): + def __init__(self, + ophys_experiment_id: int, + ophys_session_id: OphysSessionId, + experiment_container_id: ExperimentContainerId, + field_of_view_shape: FieldOfViewShape, + imaging_depth: ImagingDepth, + imaging_plane_group: ImagingPlaneGroup, + project_code: ProjectCode): + super().__init__( + ophys_experiment_id=ophys_experiment_id, + ophys_session_id=ophys_session_id, + experiment_container_id=experiment_container_id, + field_of_view_shape=field_of_view_shape, + imaging_depth=imaging_depth, + project_code=project_code + ) + self._imaging_plane_group = imaging_plane_group + + @classmethod + def from_lims( + cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "MultiplaneMetadata": + ophys_experiment_metadata = OphysExperimentMetadata.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + imaging_plane_group = ImagingPlaneGroup.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + return cls( + ophys_experiment_id=ophys_experiment_metadata.ophys_experiment_id, + ophys_session_id=ophys_experiment_metadata._ophys_session_id, + experiment_container_id=ophys_experiment_metadata._experiment_container_id, # noqa E501 + field_of_view_shape=ophys_experiment_metadata._field_of_view_shape, + imaging_depth=ophys_experiment_metadata._imaging_depth, + project_code=ophys_experiment_metadata._project_code, + imaging_plane_group=imaging_plane_group + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "MultiplaneMetadata": + ophys_experiment_metadata = super().from_json(dict_repr=dict_repr) + imaging_plane_group = ImagingPlaneGroup.from_json(dict_repr=dict_repr) + return cls( + ophys_experiment_id=ophys_experiment_metadata.ophys_experiment_id, + ophys_session_id=ophys_experiment_metadata._ophys_session_id, + experiment_container_id=ophys_experiment_metadata._experiment_container_id, # noqa E501 + field_of_view_shape=ophys_experiment_metadata._field_of_view_shape, + imaging_depth=ophys_experiment_metadata._imaging_depth, + project_code=ophys_experiment_metadata._project_code, + imaging_plane_group=imaging_plane_group + ) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "MultiplaneMetadata": + ophys_experiment_metadata = super().from_nwb(nwbfile=nwbfile) + imaging_plane_group = ImagingPlaneGroup.from_nwb(nwbfile=nwbfile) + return cls( + ophys_experiment_id=ophys_experiment_metadata.ophys_experiment_id, + ophys_session_id=ophys_experiment_metadata._ophys_session_id, + experiment_container_id=ophys_experiment_metadata._experiment_container_id, # noqa E501 + field_of_view_shape=ophys_experiment_metadata._field_of_view_shape, + imaging_depth=ophys_experiment_metadata._imaging_depth, + project_code=ophys_experiment_metadata._project_code, + imaging_plane_group=imaging_plane_group + ) + + @property + def imaging_plane_group(self) -> int: + return self._imaging_plane_group.plane_group + + @property + def imaging_plane_group_count(self) -> int: + # TODO this is at the wrong level of abstraction. + # It is an attribute of the session, not the experiment. + # Currently, an Ophys Session metadata abstraction doesn't exist + return self._imaging_plane_group.plane_group_count diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_experiment_metadata.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_experiment_metadata.py new file mode 100644 index 000000000..247df5c87 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_experiment_metadata.py @@ -0,0 +1,138 @@ +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.experiment_container_id import \ + ExperimentContainerId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.field_of_view_shape import \ + FieldOfViewShape +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.imaging_depth import \ + ImagingDepth +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_session_id import \ + OphysSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.project_code import \ + ProjectCode +from allensdk.internal.api import PostgresQueryMixin + + +class OphysExperimentMetadata(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + """Container class for ophys experiment metadata""" + def __init__(self, + ophys_experiment_id: int, + ophys_session_id: OphysSessionId, + experiment_container_id: ExperimentContainerId, + field_of_view_shape: FieldOfViewShape, + imaging_depth: ImagingDepth, + project_code: Optional[ProjectCode] = None): + super().__init__(name='ophys_experiment_metadata', value=self) + self._ophys_experiment_id = ophys_experiment_id + self._ophys_session_id = ophys_session_id + self._experiment_container_id = experiment_container_id + self._field_of_view_shape = field_of_view_shape + self._imaging_depth = imaging_depth + self._project_code = project_code + + # project_code needs to be excluded from comparison + # since it's only exposed internally + self._exclude_from_equals = {'project_code'} + + @classmethod + def from_lims( + cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "OphysExperimentMetadata": + ophys_session_id = OphysSessionId.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + experiment_container_id = ExperimentContainerId.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + field_of_view_shape = FieldOfViewShape.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + imaging_depth = ImagingDepth.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + project_code = ProjectCode.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=lims_db) + + return cls( + ophys_experiment_id=ophys_experiment_id, + ophys_session_id=ophys_session_id, + experiment_container_id=experiment_container_id, + field_of_view_shape=field_of_view_shape, + imaging_depth=imaging_depth, + project_code=project_code + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "OphysExperimentMetadata": + ophys_session_id = OphysSessionId.from_json(dict_repr=dict_repr) + experiment_container_id = ExperimentContainerId.from_json( + dict_repr=dict_repr) + ophys_experiment_id = dict_repr['ophys_experiment_id'] + field_of_view_shape = FieldOfViewShape.from_json(dict_repr=dict_repr) + imaging_depth = ImagingDepth.from_json(dict_repr=dict_repr) + + return OphysExperimentMetadata( + ophys_experiment_id=ophys_experiment_id, + ophys_session_id=ophys_session_id, + experiment_container_id=experiment_container_id, + field_of_view_shape=field_of_view_shape, + imaging_depth=imaging_depth + ) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "OphysExperimentMetadata": + ophys_experiment_id = int(nwbfile.identifier) + ophys_session_id = OphysSessionId.from_nwb(nwbfile=nwbfile) + experiment_container_id = ExperimentContainerId.from_nwb( + nwbfile=nwbfile) + field_of_view_shape = FieldOfViewShape.from_nwb(nwbfile=nwbfile) + imaging_depth = ImagingDepth.from_nwb(nwbfile=nwbfile) + + return OphysExperimentMetadata( + ophys_experiment_id=ophys_experiment_id, + ophys_session_id=ophys_session_id, + experiment_container_id=experiment_container_id, + field_of_view_shape=field_of_view_shape, + imaging_depth=imaging_depth + ) + + # TODO rename to ophys_container_id + @property + def experiment_container_id(self) -> int: + return self._experiment_container_id.value + + @property + def field_of_view_shape(self) -> FieldOfViewShape: + return self._field_of_view_shape + + @property + def imaging_depth(self) -> int: + return self._imaging_depth.value + + @property + def ophys_experiment_id(self) -> int: + return self._ophys_experiment_id + + @property + def ophys_session_id(self) -> int: + # TODO this is at the wrong layer of abstraction. + # Should be at ophys session level + # (need to create ophys session class) + return self._ophys_session_id.value + + @property + def project_code(self) -> Optional[str]: + if self._project_code is None: + pc = self._project_code + else: + pc = self._project_code.value + return pc diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_session_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_session_id.py new file mode 100644 index 000000000..f70ea8574 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/ophys_session_id.py @@ -0,0 +1,35 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class OphysSessionId(DataObject, LimsReadableInterface, + JsonReadableInterface, NwbReadableInterface): + """"Ophys session id""" + def __init__(self, session_id: int): + super().__init__(name='session_id', + value=session_id) + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "OphysSessionId": + query = """ + SELECT oe.ophys_session_id + FROM ophys_experiments oe + WHERE id = {}; + """.format(ophys_experiment_id) + session_id = lims_db.fetchone(query, strict=False) + return cls(session_id=session_id) + + @classmethod + def from_json(cls, dict_repr: dict) -> "OphysSessionId": + return cls(session_id=dict_repr['ophys_session_id']) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "OphysSessionId": + metadata = nwbfile.lab_meta_data['metadata'] + return cls(session_id=metadata.ophys_session_id) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/project_code.py b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/project_code.py new file mode 100644 index 000000000..60216326e --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/project_code.py @@ -0,0 +1,26 @@ +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base\ + .readable_interfaces import \ + LimsReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class ProjectCode(DataObject, LimsReadableInterface): + def __init__(self, project_code: str): + super().__init__(name='project_code', value=project_code) + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "ProjectCode": + query = f""" + SELECT projects.code AS project_code + FROM ophys_sessions + JOIN projects ON projects.id = ophys_sessions.project_id + WHERE ophys_sessions.id = ( + SELECT oe.ophys_session_id + FROM ophys_experiments oe + WHERE oe.id = {ophys_experiment_id} + ) + """ + project_code = lims_db.fetchone(query, strict=True) + return cls(project_code=project_code) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/__init__.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/age.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/age.py new file mode 100644 index 000000000..111ac436f --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/age.py @@ -0,0 +1,77 @@ +import re +import warnings +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class Age(DataObject, JsonReadableInterface, LimsReadableInterface, + NwbReadableInterface): + """Age of animal (in days)""" + def __init__(self, age: int): + super().__init__(name="age_in_days", value=age) + + @classmethod + def from_json(cls, dict_repr: dict) -> "Age": + age = dict_repr["age"] + age = cls._age_code_to_days(age=age) + return cls(age=age) + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "Age": + query = f""" + SELECT a.name AS age + FROM behavior_sessions bs + JOIN donors d ON d.id = bs.donor_id + JOIN ages a ON a.id = d.age_id + WHERE bs.id = {behavior_session_id}; + """ + age = lims_db.fetchone(query, strict=True) + age = cls._age_code_to_days(age=age) + return cls(age=age) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Age": + age = cls._age_code_to_days(age=nwbfile.subject.age) + return cls(age=age) + + @staticmethod + def to_iso8601(age: int): + if age is None: + return 'null' + return f'P{age}D' + + @staticmethod + def _age_code_to_days(age: str, warn=False) -> Optional[int]: + """Converts the age code into a numeric days representation + + Parameters + ---------- + age + age code, ie P123 + warn + Whether to output warning if parsing fails + """ + if not age.startswith('P'): + if warn: + warnings.warn('Could not parse numeric age from age code ' + '(age code does not start with "P")') + return None + + match = re.search(r'\d+', age) + + if match is None: + if warn: + warnings.warn('Could not parse numeric age from age code ' + '(no numeric values found in age code)') + return None + + start, end = match.span() + return int(age[start:end]) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/driver_line.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/driver_line.py new file mode 100644 index 000000000..1d0a8a34c --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/driver_line.py @@ -0,0 +1,47 @@ +from typing import List + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin, \ + OneOrMoreResultExpectedError + + +class DriverLine(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface): + """the genotype name(s) of the driver line(s)""" + def __init__(self, driver_line: List[str]): + super().__init__(name="driver_line", value=driver_line) + + @classmethod + def from_json(cls, dict_repr: dict) -> "DriverLine": + return cls(driver_line=dict_repr['driver_line']) + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "DriverLine": + query = f""" + SELECT g.name AS driver_line + FROM behavior_sessions bs + JOIN donors d ON bs.donor_id=d.id + JOIN donors_genotypes dg ON dg.donor_id=d.id + JOIN genotypes g ON g.id=dg.genotype_id + JOIN genotype_types gt + ON gt.id=g.genotype_type_id AND gt.name = 'driver' + WHERE bs.id={behavior_session_id}; + """ + result = lims_db.fetchall(query) + if result is None or len(result) < 1: + raise OneOrMoreResultExpectedError( + f"Expected one or more, but received: '{result}' " + f"from query:\n'{query}'") + driver_line = sorted(result) + return cls(driver_line=driver_line) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "DriverLine": + driver_line = sorted(list(nwbfile.subject.driver_line)) + return cls(driver_line=driver_line) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/full_genotype.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/full_genotype.py new file mode 100644 index 000000000..be9977fc2 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/full_genotype.py @@ -0,0 +1,57 @@ +import warnings +from typing import Optional + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class FullGenotype(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface): + """the name of the subject's genotype""" + def __init__(self, full_genotype: str): + super().__init__(name="full_genotype", value=full_genotype) + + @classmethod + def from_json(cls, dict_repr: dict) -> "FullGenotype": + return cls(full_genotype=dict_repr['full_genotype']) + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "FullGenotype": + query = f""" + SELECT d.full_genotype + FROM behavior_sessions bs + JOIN donors d ON d.id=bs.donor_id + WHERE bs.id= {behavior_session_id}; + """ + genotype = lims_db.fetchone(query, strict=True) + return cls(full_genotype=genotype) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "FullGenotype": + return cls(full_genotype=nwbfile.subject.genotype) + + def parse_cre_line(self, warn=False) -> Optional[str]: + """ + Parameters + ---------- + warn + Whether to output warning if parsing fails + + Returns + ---------- + cre_line + just the Cre line, e.g. Vip-IRES-Cre, or None if not possible to + parse + """ + full_genotype = self.value + if ';' not in full_genotype: + if warn: + warnings.warn('Unable to parse cre_line from full_genotype') + return None + return full_genotype.split(';')[0].replace('/wt', '') diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py new file mode 100644 index 000000000..d29b9069d --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py @@ -0,0 +1,42 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class MouseId(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface): + """the LabTracks ID""" + def __init__(self, mouse_id: int): + super().__init__(name="mouse_id", value=mouse_id) + + @classmethod + def from_json(cls, dict_repr: dict) -> "MouseId": + mouse_id = dict_repr['external_specimen_name'] + mouse_id = int(mouse_id) + return cls(mouse_id=mouse_id) + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "MouseId": + # TODO: Should this even be included? + # Found sometimes there were entries with NONE which is + # why they are filtered out; also many entries in the table + # match the donor_id, which is why used DISTINCT + query = f""" + SELECT DISTINCT(sp.external_specimen_name) + FROM behavior_sessions bs + JOIN donors d ON bs.donor_id=d.id + JOIN specimens sp ON sp.donor_id=d.id + WHERE bs.id={behavior_session_id} + AND sp.external_specimen_name IS NOT NULL; + """ + mouse_id = int(lims_db.fetchone(query, strict=True)) + return cls(mouse_id=mouse_id) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "MouseId": + return cls(mouse_id=int(nwbfile.subject.subject_id)) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/reporter_line.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/reporter_line.py new file mode 100644 index 000000000..56fcfb838 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/reporter_line.py @@ -0,0 +1,113 @@ +import warnings +from typing import Optional, List, Union + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.internal.api import PostgresQueryMixin, \ + OneOrMoreResultExpectedError + + +class ReporterLine(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface): + """the genotype name(s) of the reporter line(s)""" + def __init__(self, reporter_line: Optional[str]): + super().__init__(name="reporter_line", value=reporter_line) + + @classmethod + def from_json(cls, dict_repr: dict) -> "ReporterLine": + reporter_line = dict_repr['reporter_line'] + reporter_line = cls.parse(reporter_line=reporter_line, warn=True) + return cls(reporter_line=reporter_line) + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "ReporterLine": + query = f""" + SELECT g.name AS reporter_line + FROM behavior_sessions bs + JOIN donors d ON bs.donor_id=d.id + JOIN donors_genotypes dg ON dg.donor_id=d.id + JOIN genotypes g ON g.id=dg.genotype_id + JOIN genotype_types gt + ON gt.id=g.genotype_type_id AND gt.name = 'reporter' + WHERE bs.id={behavior_session_id}; + """ + result = lims_db.fetchall(query) + if result is None or len(result) < 1: + raise OneOrMoreResultExpectedError( + f"Expected one or more, but received: '{result}' " + f"from query:\n'{query}'") + reporter_line = cls.parse(reporter_line=result, warn=True) + return cls(reporter_line=reporter_line) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "ReporterLine": + return cls(reporter_line=nwbfile.subject.reporter_line) + + @staticmethod + def parse(reporter_line: Union[Optional[List[str]], str], + warn=False) -> Optional[str]: + """There can be multiple reporter lines, so it is returned from LIMS + as a list. But there shouldn't be more than 1 for behavior. This + tries to convert to str + + Parameters + ---------- + reporter_line + List of reporter line + warn + Whether to output warnings if parsing fails + + Returns + --------- + single reporter line, or None if not possible + """ + if reporter_line is None: + if warn: + warnings.warn('Error parsing reporter line. It is null.') + return None + + if len(reporter_line) == 0: + if warn: + warnings.warn('Error parsing reporter line. ' + 'The array is empty') + return None + + if isinstance(reporter_line, str): + return reporter_line + + if len(reporter_line) > 1: + if warn: + warnings.warn('More than 1 reporter line. Returning the first ' + 'one') + + return reporter_line[0] + + def parse_indicator(self, warn=False) -> Optional[str]: + """Parses indicator from reporter""" + reporter_line = self.value + reporter_substring_indicator_map = { + 'GCaMP6f': 'GCaMP6f', + 'GC6f': 'GCaMP6f', + 'GCaMP6s': 'GCaMP6s' + } + if reporter_line is None: + if warn: + warnings.warn( + 'Could not parse indicator from reporter because ' + 'there is no reporter') + return None + + for substr, indicator in reporter_substring_indicator_map.items(): + if substr in reporter_line: + return indicator + + if warn: + warnings.warn( + 'Could not parse indicator from reporter because none' + 'of the expected substrings were found in the reporter') + return None diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/sex.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/sex.py new file mode 100644 index 000000000..aa81242fb --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/sex.py @@ -0,0 +1,41 @@ +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + JsonWritableInterface +from allensdk.internal.api import PostgresQueryMixin + + +class Sex(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface, JsonWritableInterface): + """sex of the animal (M/F)""" + def __init__(self, sex: str): + super().__init__(name="sex", value=sex) + + @classmethod + def from_json(cls, dict_repr: dict) -> "Sex": + return cls(sex=dict_repr["sex"]) + + def to_json(self) -> dict: + return {"sex": self.value} + + @classmethod + def from_lims(cls, behavior_session_id: int, + lims_db: PostgresQueryMixin) -> "Sex": + query = f""" + SELECT g.name AS sex + FROM behavior_sessions bs + JOIN donors d ON bs.donor_id = d.id + JOIN genders g ON g.id = d.gender_id + WHERE bs.id = {behavior_session_id}; + """ + sex = lims_db.fetchone(query, strict=True) + return cls(sex=sex) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Sex": + return cls(sex=nwbfile.subject.sex) diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/subject_metadata.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/subject_metadata.py new file mode 100644 index 000000000..710a07ed4 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/subject_metadata.py @@ -0,0 +1,162 @@ +from typing import Optional, List + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.age import \ + Age +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.driver_line import \ + DriverLine +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.full_genotype import \ + FullGenotype +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.mouse_id import \ + MouseId +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.reporter_line import \ + ReporterLine +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.sex import \ + Sex +from allensdk.brain_observatory.behavior.schemas import SubjectMetadataSchema +from allensdk.brain_observatory.nwb import load_pynwb_extension +from allensdk.internal.api import PostgresQueryMixin + + +class SubjectMetadata(DataObject, LimsReadableInterface, NwbReadableInterface, + NwbWritableInterface, JsonReadableInterface, + JsonWritableInterface): + """Subject metadata""" + + def __init__(self, + sex: Sex, + age: Age, + reporter_line: ReporterLine, + full_genotype: FullGenotype, + driver_line: DriverLine, + mouse_id: MouseId): + super().__init__(name='subject_metadata', value=self) + self._sex = sex + self._age = age + self._reporter_line = reporter_line + self._full_genotype = full_genotype + self._driver_line = driver_line + self._mouse_id = mouse_id + + @classmethod + def from_lims(cls, + behavior_session_id: BehaviorSessionId, + lims_db: PostgresQueryMixin) -> "SubjectMetadata": + sex = Sex.from_lims(behavior_session_id=behavior_session_id.value, + lims_db=lims_db) + age = Age.from_lims(behavior_session_id=behavior_session_id.value, + lims_db=lims_db) + reporter_line = ReporterLine.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + full_genotype = FullGenotype.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + driver_line = DriverLine.from_lims( + behavior_session_id=behavior_session_id.value, lims_db=lims_db) + mouse_id = MouseId.from_lims( + behavior_session_id=behavior_session_id.value, + lims_db=lims_db) + return cls( + sex=sex, + age=age, + full_genotype=full_genotype, + driver_line=driver_line, + mouse_id=mouse_id, + reporter_line=reporter_line + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "SubjectMetadata": + sex = Sex.from_json(dict_repr=dict_repr) + age = Age.from_json(dict_repr=dict_repr) + reporter_line = ReporterLine.from_json(dict_repr=dict_repr) + full_genotype = FullGenotype.from_json(dict_repr=dict_repr) + driver_line = DriverLine.from_json(dict_repr=dict_repr) + mouse_id = MouseId.from_json(dict_repr=dict_repr) + + return cls( + sex=sex, + age=age, + full_genotype=full_genotype, + driver_line=driver_line, + mouse_id=mouse_id, + reporter_line=reporter_line + ) + + def to_json(self) -> dict: + pass + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "SubjectMetadata": + mouse_id = MouseId.from_nwb(nwbfile=nwbfile) + sex = Sex.from_nwb(nwbfile=nwbfile) + age = Age.from_nwb(nwbfile=nwbfile) + reporter_line = ReporterLine.from_nwb(nwbfile=nwbfile) + driver_line = DriverLine.from_nwb(nwbfile=nwbfile) + genotype = FullGenotype.from_nwb(nwbfile=nwbfile) + + return cls( + mouse_id=mouse_id, + sex=sex, + age=age, + reporter_line=reporter_line, + driver_line=driver_line, + full_genotype=genotype + ) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + BehaviorSubject = load_pynwb_extension(SubjectMetadataSchema, + 'ndx-aibs-behavior-ophys') + nwb_subject = BehaviorSubject( + description="A visual behavior subject with a LabTracks ID", + age=Age.to_iso8601(age=self.age_in_days), + driver_line=self.driver_line, + genotype=self.full_genotype, + subject_id=str(self.mouse_id), + reporter_line=self.reporter_line, + sex=self.sex, + species='Mus musculus') + nwbfile.subject = nwb_subject + return nwbfile + + @property + def sex(self) -> str: + return self._sex.value + + @property + def age_in_days(self) -> Optional[int]: + return self._age.value + + @property + def reporter_line(self) -> Optional[str]: + return self._reporter_line.value + + @property + def full_genotype(self) -> str: + return self._full_genotype.value + + @property + def cre_line(self) -> Optional[str]: + return self._full_genotype.parse_cre_line(warn=True) + + @property + def driver_line(self) -> List[str]: + return self._driver_line.value + + @property + def mouse_id(self) -> int: + return self._mouse_id.value diff --git a/allensdk/brain_observatory/behavior/data_objects/motion_correction.py b/allensdk/brain_observatory/behavior/data_objects/motion_correction.py new file mode 100644 index 000000000..a6121aa49 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/motion_correction.py @@ -0,0 +1,71 @@ +import pandas as pd +from pynwb import NWBFile, TimeSeries + +from allensdk.brain_observatory.behavior.data_files\ + .rigid_motion_transform_file import \ + RigidMotionTransformFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + DataFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface + + +class MotionCorrection(DataObject, DataFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + """motion correction output""" + def __init__(self, motion_correction: pd.DataFrame): + """ + :param motion_correction + Columns: + x: float + y: float + """ + super().__init__(name='motion_correction', value=motion_correction) + + @classmethod + def from_data_file( + cls, rigid_motion_transform_file: RigidMotionTransformFile) \ + -> "MotionCorrection": + df = rigid_motion_transform_file.data + return cls(motion_correction=df) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "MotionCorrection": + ophys_module = nwbfile.processing['ophys'] + + motion_correction_data = { + 'x': ophys_module.get_data_interface( + 'ophys_motion_correction_x').data[:], + 'y': ophys_module.get_data_interface( + 'ophys_motion_correction_y').data[:] + } + + df = pd.DataFrame(motion_correction_data) + return cls(motion_correction=df) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + ophys_module = nwbfile.processing['ophys'] + ophys_timestamps = ophys_module.get_data_interface( + 'dff').roi_response_series['traces'].timestamps + + t1 = TimeSeries( + name='ophys_motion_correction_x', + data=self.value['x'].values, + timestamps=ophys_timestamps, + unit='pixels' + ) + + t2 = TimeSeries( + name='ophys_motion_correction_y', + data=self.value['y'].values, + timestamps=ophys_timestamps, + unit='pixels' + ) + + ophys_module.add_data_interface(t1) + ophys_module.add_data_interface(t2) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/projections.py b/allensdk/brain_observatory/behavior/data_objects/projections.py new file mode 100644 index 000000000..97ae9536c --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/projections.py @@ -0,0 +1,128 @@ +from matplotlib import image as mpimg +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, NwbReadableInterface, \ + LimsReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.image_api import ImageApi, Image +from allensdk.brain_observatory.nwb.nwb_utils import get_image, \ + add_image_to_nwb +from allensdk.internal.api import PostgresQueryMixin +from allensdk.internal.core.lims_utilities import safe_system_path + + +class Projections(DataObject, LimsReadableInterface, JsonReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, max_projection: Image, avg_projection: Image): + super().__init__(name='projections', value=self) + self._max_projection = max_projection + self._avg_projection = avg_projection + + @property + def max_projection(self) -> Image: + return self._max_projection + + @property + def avg_projection(self) -> Image: + return self._avg_projection + + @classmethod + def from_lims(cls, ophys_experiment_id: int, + lims_db: PostgresQueryMixin) -> "Projections": + def _get_filepaths(): + query = """ + SELECT + wkf.storage_directory || wkf.filename AS filepath, + wkft.name as wkfn + FROM ophys_experiments oe + JOIN ophys_cell_segmentation_runs ocsr + ON ocsr.ophys_experiment_id = oe.id + JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id + JOIN well_known_file_types wkft + ON wkft.id = wkf.well_known_file_type_id + WHERE ocsr.current = 't' + AND wkf.attachable_type = 'OphysCellSegmentationRun' + AND wkft.name IN ('OphysMaxIntImage', + 'OphysAverageIntensityProjectionImage') + AND oe.id = {}; + """.format(ophys_experiment_id) + res = lims_db.select(query=query) + res['filepath'] = res['filepath'].apply(safe_system_path) + return res + + def _get_pixel_size(): + query = """ + SELECT sc.resolution + FROM ophys_experiments oe + JOIN scans sc ON sc.image_id=oe.ophys_primary_image_id + WHERE oe.id = {}; + """.format(ophys_experiment_id) + return lims_db.fetchone(query, strict=True) + + res = _get_filepaths() + pixel_size = _get_pixel_size() + + max_projection_filepath = \ + res[res['wkfn'] == 'OphysMaxIntImage'].iloc[0]['filepath'] + max_projection = cls._from_filepath(filepath=max_projection_filepath, + pixel_size=pixel_size) + + avg_projection_filepath = \ + (res[res['wkfn'] == 'OphysAverageIntensityProjectionImage'].iloc[0] + ['filepath']) + avg_projection = cls._from_filepath(filepath=avg_projection_filepath, + pixel_size=pixel_size) + return Projections(max_projection=max_projection, + avg_projection=avg_projection) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Projections": + max_projection = get_image(nwbfile=nwbfile, name='max_projection', + module='ophys') + avg_projection = get_image(nwbfile=nwbfile, name='average_image', + module='ophys') + return Projections(max_projection=max_projection, + avg_projection=avg_projection) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + add_image_to_nwb(nwbfile=nwbfile, + image_data=self._max_projection, + image_name='max_projection') + add_image_to_nwb(nwbfile=nwbfile, + image_data=self._avg_projection, + image_name='average_image') + + return nwbfile + + @classmethod + def from_json(cls, dict_repr: dict) -> "Projections": + max_projection_filepath = dict_repr['max_projection_file'] + avg_projection_filepath = \ + dict_repr['average_intensity_projection_image_file'] + pixel_size = dict_repr['surface_2p_pixel_size_um'] + + max_projection = cls._from_filepath(filepath=max_projection_filepath, + pixel_size=pixel_size) + avg_projection = cls._from_filepath(filepath=avg_projection_filepath, + pixel_size=pixel_size) + return Projections(max_projection=max_projection, + avg_projection=avg_projection) + + @staticmethod + def _from_filepath(filepath: str, pixel_size: float) -> Image: + """ + :param filepath + path to image + :param pixel_size + pixel size in um + """ + img = mpimg.imread(filepath) + img = ImageApi.serialize(img, [pixel_size / 1000., + pixel_size / 1000.], 'mm') + img = ImageApi.deserialize(img=img) + return img diff --git a/allensdk/brain_observatory/behavior/data_objects/rewards.py b/allensdk/brain_observatory/behavior/data_objects/rewards.py new file mode 100644 index 000000000..7080fa719 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/rewards.py @@ -0,0 +1,84 @@ +import warnings +from typing import Optional + +import pandas as pd +from pynwb import NWBFile, TimeSeries, ProcessingModule + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface + + +class Rewards(DataObject, StimulusFileReadableInterface, NwbReadableInterface, + NwbWritableInterface): + def __init__(self, rewards: pd.DataFrame): + super().__init__(name='rewards', value=rewards) + + @classmethod + def from_stimulus_file( + cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps) -> "Rewards": + """Get reward data from pkl file, based on timestamps + (not sync file). + """ + data = stimulus_file.data + + trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"]) + rewards_dict = {"volume": [], "timestamps": [], "autorewarded": []} + for idx, trial in trial_df.iterrows(): + rewards = trial["rewards"] + # as i write this there can only ever be one reward per trial + if rewards: + rewards_dict["volume"].append(rewards[0][0]) + rewards_dict["timestamps"].append( + stimulus_timestamps.value[rewards[0][2]]) + auto_rwrd = trial["trial_params"]["auto_reward"] + rewards_dict["autorewarded"].append(auto_rwrd) + + df = pd.DataFrame(rewards_dict) + return cls(rewards=df) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> Optional["Rewards"]: + if 'rewards' in nwbfile.processing: + rewards = nwbfile.processing['rewards'] + time = rewards.get_data_interface('autorewarded').timestamps[:] + autorewarded = rewards.get_data_interface('autorewarded').data[:] + volume = rewards.get_data_interface('volume').data[:] + df = pd.DataFrame({ + 'volume': volume, 'timestamps': time, + 'autorewarded': autorewarded}) + else: + warnings.warn("This session " + f"'{int(nwbfile.identifier)}' has no rewards data.") + return None + return cls(rewards=df) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + reward_volume_ts = TimeSeries( + name='volume', + data=self.value['volume'].values, + timestamps=self.value['timestamps'].values, + unit='mL' + ) + + autorewarded_ts = TimeSeries( + name='autorewarded', + data=self.value['autorewarded'].values, + timestamps=reward_volume_ts.timestamps, + unit='mL' + ) + + rewards_mod = ProcessingModule('rewards', + 'Licking behavior processing module') + rewards_mod.add_data_interface(reward_volume_ts) + rewards_mod.add_data_interface(autorewarded_ts) + nwbfile.add_processing_module(rewards_mod) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/running_speed/__init__.py b/allensdk/brain_observatory/behavior/data_objects/running_speed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/running_speed/running_acquisition.py b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_acquisition.py new file mode 100644 index 000000000..2f640e158 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_acquisition.py @@ -0,0 +1,209 @@ + +import json +from typing import Optional + +from cachetools import cached, LRUCache +from cachetools.keys import hashkey + +import pandas as pd + +from pynwb import NWBFile, ProcessingModule +from pynwb.base import TimeSeries + +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_objects import ( + DataObject, StimulusTimestamps +) +from allensdk.brain_observatory.behavior.data_files import ( + StimulusFile +) +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing import ( # noqa: E501 + get_running_df +) + + +def from_json_cache_key( + cls, dict_repr: dict +): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key( + cls, db, + behavior_session_id: int, ophys_experiment_id: Optional[int] = None +): + return hashkey( + behavior_session_id, ophys_experiment_id + ) + + +class RunningAcquisition(DataObject, LimsReadableInterface, + NwbReadableInterface, NwbWritableInterface, + JsonWritableInterface): + """A DataObject which contains properties and methods to load, process, + and represent running acquisition data. + + Running aquisition data is represented as: + + Pandas Dataframe with an index of timestamps and the following columns: + "dx": Angular change, computed during data collection + "v_sig": Voltage signal from the encoder + "v_in": The theoretical maximum voltage that the encoder + will reach prior to "wrapping". This should + theoretically be 5V (after crossing 5V goes to 0V, or + vice versa). In practice the encoder does not always + reach this value before wrapping, which can cause + transient spikes in speed at the voltage "wraps". + """ + + def __init__( + self, + running_acquisition: pd.DataFrame, + stimulus_file: Optional[StimulusFile] = None, + stimulus_timestamps: Optional[StimulusTimestamps] = None, + ): + super().__init__(name="running_acquisition", value=running_acquisition) + self._stimulus_file = stimulus_file + self._stimulus_timestamps = stimulus_timestamps + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) + def from_json( + cls, + dict_repr: dict, + ) -> "RunningAcquisition": + stimulus_file = StimulusFile.from_json(dict_repr) + stimulus_timestamps = StimulusTimestamps.from_json(dict_repr) + running_acq_df = get_running_df( + data=stimulus_file.data, time=stimulus_timestamps.value, + ) + running_acq_df.drop("speed", axis=1, inplace=True) + + return cls( + running_acquisition=running_acq_df, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + ) + + def to_json(self) -> dict: + """[summary] + + Returns + ------- + dict + [description] + + Raises + ------ + RuntimeError + [description] + """ + if self._stimulus_file is None or self._stimulus_timestamps is None: + raise RuntimeError( + "RunningAcquisition DataObject lacks information about the " + "StimulusFile or StimulusTimestamps. This is likely due to " + "instantiating from NWB which prevents to_json() functionality" + ) + output_dict = dict() + output_dict.update(self._stimulus_file.to_json()) + output_dict.update(self._stimulus_timestamps.to_json()) + return output_dict + + @classmethod + @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) + def from_lims( + cls, + db: PostgresQueryMixin, + behavior_session_id: int, + ophys_experiment_id: Optional[int] = None, + ) -> "RunningAcquisition": + + stimulus_file = StimulusFile.from_lims(db, behavior_session_id) + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file + ) + running_acq_df = get_running_df( + data=stimulus_file.data, time=stimulus_timestamps.value, + ) + running_acq_df.drop("speed", axis=1, inplace=True) + + return cls( + running_acquisition=running_acq_df, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + ) + + @classmethod + def from_nwb( + cls, + nwbfile: NWBFile + ) -> "RunningAcquisition": + running_module = nwbfile.modules['running'] + dx_interface = running_module.get_data_interface('dx') + + dx = dx_interface.data + v_in = nwbfile.get_acquisition('v_in').data + v_sig = nwbfile.get_acquisition('v_sig').data + timestamps = dx_interface.timestamps[:] + + running_acq_df = pd.DataFrame( + { + 'dx': dx, + 'v_in': v_in, + 'v_sig': v_sig + }, + index=pd.Index(timestamps, name='timestamps') + ) + return cls(running_acquisition=running_acq_df) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + running_acquisition_df: pd.DataFrame = self.value + + running_dx_series = TimeSeries( + name='dx', + data=running_acquisition_df['dx'].values, + timestamps=running_acquisition_df.index.values, + unit='cm', + description=( + 'Running wheel angular change, computed during data collection' + ) + ) + v_sig = TimeSeries( + name='v_sig', + data=running_acquisition_df['v_sig'].values, + timestamps=running_acquisition_df.index.values, + unit='V', + description='Voltage signal from the running wheel encoder' + ) + v_in = TimeSeries( + name='v_in', + data=running_acquisition_df['v_in'].values, + timestamps=running_acquisition_df.index.values, + unit='V', + description=( + 'The theoretical maximum voltage that the running wheel ' + 'encoder will reach prior to "wrapping". This should ' + 'theoretically be 5V (after crossing 5V goes to 0V, or ' + 'vice versa). In practice the encoder does not always ' + 'reach this value before wrapping, which can cause ' + 'transient spikes in speed at the voltage "wraps".') + ) + + if 'running' in nwbfile.processing: + running_mod = nwbfile.processing['running'] + else: + running_mod = ProcessingModule('running', + 'Running speed processing module') + nwbfile.add_processing_module(running_mod) + + running_mod.add_data_interface(running_dx_series) + nwbfile.add_acquisition(v_sig) + nwbfile.add_acquisition(v_in) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/running_processing.py b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_processing.py similarity index 98% rename from allensdk/brain_observatory/behavior/running_processing.py rename to allensdk/brain_observatory/behavior/data_objects/running_speed/running_processing.py index 186c8b2ab..7222cb7d1 100644 --- a/allensdk/brain_observatory/behavior/running_processing.py +++ b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_processing.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd import warnings -from typing import Iterable, Union, Any, Optional +from typing import Iterable, Union, Optional def calc_deriv(x, time): @@ -299,7 +299,9 @@ def _zscore_threshold_1d(data: np.ndarray, return corrected_data -def get_running_df(data, time: np.ndarray, lowpass: bool = True, zscore_threshold=10.0): +def get_running_df( + data, time: np.ndarray, lowpass: bool = True, zscore_threshold=10.0 +): """ Given the data from the behavior 'pkl' file object and a 1d array of timestamps, compute the running speed. Returns a @@ -318,7 +320,8 @@ def get_running_df(data, time: np.ndarray, lowpass: bool = True, zscore_threshol Whether to apply a 10Hz low-pass filter to the running speed data. zscore_threshold: float - The threshold to use for removing outlier running speeds which might be noise and not true signal + The threshold to use for removing outlier running speeds which might + be noise and not true signal. Returns ------- diff --git a/allensdk/brain_observatory/behavior/data_objects/running_speed/running_speed.py b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_speed.py new file mode 100644 index 000000000..694d5f49f --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/running_speed/running_speed.py @@ -0,0 +1,176 @@ +from typing import Optional + +import pandas as pd + +from pynwb import NWBFile, ProcessingModule +from pynwb.base import TimeSeries + +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.core.exceptions import DataFrameIndexError +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_objects import ( + DataObject, StimulusTimestamps +) +from allensdk.brain_observatory.behavior.data_files import ( + StimulusFile +) +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing import ( # noqa: E501 + get_running_df +) + + +class RunningSpeed(DataObject, LimsReadableInterface, NwbReadableInterface, + NwbWritableInterface, JsonReadableInterface, + JsonWritableInterface): + """A DataObject which contains properties and methods to load, process, + and represent running speed data. + + Running speed data is represented as: + + Pandas Dataframe with the following columns: + "timestamps": Timestamps (in s) for calculated speed values + "speed": Computed running speed in cm/s + """ + + def __init__( + self, + running_speed: pd.DataFrame, + stimulus_file: Optional[StimulusFile] = None, + stimulus_timestamps: Optional[StimulusTimestamps] = None, + filtered: bool = True + ): + super().__init__(name='running_speed', value=running_speed) + self._stimulus_file = stimulus_file + self._stimulus_timestamps = stimulus_timestamps + self._filtered = filtered + + @staticmethod + def _get_running_speed_df( + stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps, + filtered: bool = True, + zscore_threshold: float = 1.0 + ) -> pd.DataFrame: + running_data_df = get_running_df( + data=stimulus_file.data, time=stimulus_timestamps.value, + lowpass=filtered, zscore_threshold=zscore_threshold + ) + if running_data_df.index.name != "timestamps": + raise DataFrameIndexError( + f"Expected running_data_df index to be named 'timestamps' " + f"But instead got: '{running_data_df.index.name}'" + ) + running_speed = pd.DataFrame({ + "timestamps": running_data_df.index.values, + "speed": running_data_df.speed.values + }) + return running_speed + + @classmethod + def from_json( + cls, + dict_repr: dict, + filtered: bool = True, + zscore_threshold: float = 10.0 + ) -> "RunningSpeed": + stimulus_file = StimulusFile.from_json(dict_repr) + stimulus_timestamps = StimulusTimestamps.from_json(dict_repr) + + running_speed = cls._get_running_speed_df( + stimulus_file, stimulus_timestamps, filtered, zscore_threshold + ) + return cls( + running_speed=running_speed, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + filtered=filtered) + + def to_json(self) -> dict: + if self._stimulus_file is None or self._stimulus_timestamps is None: + raise RuntimeError( + "RunningSpeed DataObject lacks information about the " + "StimulusFile or StimulusTimestamps. This is likely due to " + "instantiating from NWB which prevents to_json() functionality" + ) + output_dict = dict() + output_dict.update(self._stimulus_file.to_json()) + output_dict.update(self._stimulus_timestamps.to_json()) + return output_dict + + @classmethod + def from_lims( + cls, + db: PostgresQueryMixin, + behavior_session_id: int, + filtered: bool = True, + zscore_threshold: float = 10.0, + stimulus_timestamps: Optional[StimulusTimestamps] = None + ) -> "RunningSpeed": + stimulus_file = StimulusFile.from_lims(db, behavior_session_id) + if stimulus_timestamps is None: + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file + ) + + running_speed = cls._get_running_speed_df( + stimulus_file, stimulus_timestamps, filtered, zscore_threshold + ) + return cls( + running_speed=running_speed, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + filtered=filtered + ) + + @classmethod + def from_nwb( + cls, + nwbfile: NWBFile, + filtered=True + ) -> "RunningSpeed": + running_module = nwbfile.modules['running'] + interface_name = 'speed' if filtered else 'speed_unfiltered' + running_interface = running_module.get_data_interface(interface_name) + + timestamps = running_interface.timestamps[:] + values = running_interface.data[:] + + running_speed = pd.DataFrame( + { + "timestamps": timestamps, + "speed": values + } + ) + return cls(running_speed=running_speed, filtered=filtered) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + running_speed: pd.DataFrame = self.value + data = running_speed['speed'].values + timestamps = running_speed['timestamps'].values + + if self._filtered: + data_interface_name = "speed" + else: + data_interface_name = "speed_unfiltered" + + running_speed_series = TimeSeries( + name=data_interface_name, + data=data, + timestamps=timestamps, + unit='cm/s') + + if 'running' in nwbfile.processing: + running_mod = nwbfile.processing['running'] + else: + running_mod = ProcessingModule('running', + 'Running speed processing module') + nwbfile.add_processing_module(running_mod) + + running_mod.add_data_interface(running_speed_series) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/__init__.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py new file mode 100644 index 000000000..7797bf763 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py @@ -0,0 +1,215 @@ +from typing import Optional, List + +import pandas as pd +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.stimulus_processing import \ + get_stimulus_presentations, get_stimulus_metadata, is_change_event +from allensdk.brain_observatory.nwb import \ + create_stimulus_presentation_time_interval, get_column_name +from allensdk.brain_observatory.nwb.nwb_api import NwbApi + + +class Presentations(DataObject, StimulusFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + """Stimulus presentations""" + def __init__(self, presentations: pd.DataFrame): + super().__init__(name='presentations', value=presentations) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + """Adds a stimulus table (defining stimulus characteristics for each + time point in a session) to an nwbfile as TimeIntervals. + """ + stimulus_table = self.value.copy() + + ts = nwbfile.processing['stimulus'].get_data_interface('timestamps') + possible_names = {'stimulus_name', 'image_name'} + stimulus_name_column = get_column_name(stimulus_table.columns, + possible_names) + stimulus_names = stimulus_table[stimulus_name_column].unique() + + for stim_name in sorted(stimulus_names): + specific_stimulus_table = stimulus_table[stimulus_table[ + stimulus_name_column] == stim_name] # noqa: E501 + # Drop columns where all values in column are NaN + cleaned_table = specific_stimulus_table.dropna(axis=1, how='all') + # For columns with mixed strings and NaNs, fill NaNs with 'N/A' + for colname, series in cleaned_table.items(): + types = set(series.map(type)) + if len(types) > 1 and str in types: + series.fillna('N/A', inplace=True) + cleaned_table[colname] = series.transform(str) + + interval_description = (f"Presentation times and stimuli details " + f"for '{stim_name}' stimuli. " + f"\n" + f"Note: image_name references " + f"control_description in " + f"stimulus/templates") + presentation_interval = create_stimulus_presentation_time_interval( + name=f"{stim_name}_presentations", + description=interval_description, + columns_to_add=cleaned_table.columns + ) + + for row in cleaned_table.itertuples(index=False): + row = row._asdict() + + presentation_interval.add_interval( + **row, tags='stimulus_time_interval', timeseries=ts) + + nwbfile.add_time_intervals(presentation_interval) + + return nwbfile + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Presentations": + # Note: using NwbApi class because ecephys uses this method + # TODO figure out how behavior and ecephys can share this method + nwbapi = NwbApi.from_nwbfile(nwbfile=nwbfile) + df = nwbapi.get_stimulus_presentations() + + df['is_change'] = is_change_event(stimulus_presentations=df) + df = cls._postprocess(presentations=df, fill_omitted_values=False) + return Presentations(presentations=df) + + @classmethod + def from_stimulus_file( + cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps, + limit_to_images: Optional[List] = None) -> "Presentations": + """Get stimulus presentation data. + + :param stimulus_file + :param limit_to_images + Only return images given by these image names + :param stimulus_timestamps + + + :returns: pd.DataFrame -- + Table whose rows are stimulus presentations + (i.e. a given image, for a given duration, typically 250 ms) + and whose columns are presentation characteristics. + """ + stimulus_timestamps = stimulus_timestamps.value + data = stimulus_file.data + raw_stim_pres_df = get_stimulus_presentations( + data, stimulus_timestamps) + + # Fill in nulls for image_name + # This makes two assumptions: + # 1. Nulls in `image_name` should be "gratings_" + # 2. Gratings are only present (or need to be fixed) when all + # values for `image_name` are null. + if pd.isnull(raw_stim_pres_df["image_name"]).all(): + if ~pd.isnull(raw_stim_pres_df["orientation"]).all(): + raw_stim_pres_df["image_name"] = ( + raw_stim_pres_df["orientation"] + .apply(lambda x: f"gratings_{x}")) + else: + raise ValueError("All values for 'orentation' and 'image_name'" + " are null.") + + stimulus_metadata_df = get_stimulus_metadata(data) + + idx_name = raw_stim_pres_df.index.name + stimulus_index_df = ( + raw_stim_pres_df + .reset_index() + .merge(stimulus_metadata_df.reset_index(), on=["image_name"]) + .set_index(idx_name)) + stimulus_index_df = ( + stimulus_index_df[["image_set", "image_index", "start_time", + "phase", "spatial_frequency"]] + .rename(columns={"start_time": "timestamps"}) + .sort_index() + .set_index("timestamps", drop=True)) + stim_pres_df = raw_stim_pres_df.merge( + stimulus_index_df, left_on="start_time", right_index=True, + how="left") + if len(raw_stim_pres_df) != len(stim_pres_df): + raise ValueError("Length of `stim_pres_df` should not change after" + f" merge; was {len(raw_stim_pres_df)}, now " + f" {len(stim_pres_df)}.") + + stim_pres_df['is_change'] = is_change_event( + stimulus_presentations=stim_pres_df) + + # Sort columns then drop columns which contain only all NaN values + stim_pres_df = \ + stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all') + if limit_to_images is not None: + stim_pres_df = \ + stim_pres_df[stim_pres_df['image_name'].isin(limit_to_images)] + stim_pres_df.index = pd.Int64Index( + range(stim_pres_df.shape[0]), name=stim_pres_df.index.name) + stim_pres_df = cls._postprocess(presentations=stim_pres_df) + return Presentations(presentations=stim_pres_df) + + @classmethod + def _postprocess(cls, presentations: pd.DataFrame, + fill_omitted_values=True, + omitted_time_duration: float = 0.25) \ + -> pd.DataFrame: + """ + 1. Filter/rearrange columns + 2. Optionally fill missing values for omitted flashes (no need when + reading from NWB since already filled) + + Parameters + ---------- + presentations + Presentations df + fill_omitted_values + Whether to fill stop time and duration for omitted flashes + omitted_time_duration + Amount of time a stimuli is omitted for in seconds""" + + def _filter_arrange_columns(df: pd.DataFrame): + df = df.drop(['index'], axis=1, errors='ignore') + df = df[['start_time', 'stop_time', + 'duration', + 'image_name', 'image_index', + 'is_change', 'omitted', + 'start_frame', 'end_frame', + 'image_set']] + return df + + df = _filter_arrange_columns(df=presentations) + if fill_omitted_values: + cls._fill_missing_values_for_omitted_flashes( + df=df, omitted_time_duration=omitted_time_duration) + return df + + @staticmethod + def _fill_missing_values_for_omitted_flashes( + df: pd.DataFrame, omitted_time_duration: float = 0.25) \ + -> pd.DataFrame: + """ + This function sets the stop time for a row that is an omitted + stimulus. An omitted stimulus is a stimulus where a mouse is + shown only a grey screen and these last for 250 milliseconds. + These do not include a stop_time or end_frame like other stimuli in + the stimulus table due to design choices. + + Parameters + ---------- + df + Stimuli presentations dataframe + omitted_time_duration + Amount of time a stimulus is omitted for in seconds + """ + omitted = df['omitted'] + df.loc[omitted, 'stop_time'] = \ + df.loc[omitted, 'start_time'] + omitted_time_duration + df.loc[omitted, 'duration'] = omitted_time_duration + return df diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/stimuli.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/stimuli.py new file mode 100644 index 000000000..330e3a679 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/stimuli.py @@ -0,0 +1,62 @@ +from typing import Optional, List + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.stimuli.presentations \ + import \ + Presentations +from allensdk.brain_observatory.behavior.data_objects.stimuli.templates \ + import \ + Templates + + +class Stimuli(DataObject, StimulusFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, presentations: Presentations, + templates: Templates): + super().__init__(name='stimuli', value=self) + self._presentations = presentations + self._templates = templates + + @property + def presentations(self) -> Presentations: + return self._presentations + + @property + def templates(self) -> Templates: + return self._templates + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Stimuli": + p = Presentations.from_nwb(nwbfile=nwbfile) + t = Templates.from_nwb(nwbfile=nwbfile) + return Stimuli(presentations=p, templates=t) + + @classmethod + def from_stimulus_file( + cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps, + limit_to_images: Optional[List] = None) -> "Stimuli": + p = Presentations.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + limit_to_images=limit_to_images) + t = Templates.from_stimulus_file(stimulus_file=stimulus_file, + limit_to_images=limit_to_images) + return Stimuli(presentations=p, templates=t) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + nwbfile = self._templates.to_nwb( + nwbfile=nwbfile, stimulus_presentations=self._presentations) + nwbfile = self._presentations.to_nwb(nwbfile=nwbfile) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/stimulus_processing/stimulus_templates.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/stimulus_templates.py similarity index 99% rename from allensdk/brain_observatory/behavior/stimulus_processing/stimulus_templates.py rename to allensdk/brain_observatory/behavior/data_objects/stimuli/stimulus_templates.py index bad59e0b9..d5a7bfa69 100644 --- a/allensdk/brain_observatory/behavior/stimulus_processing/stimulus_templates.py +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/stimulus_templates.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from allensdk.brain_observatory.behavior.stimulus_processing.util import \ +from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \ convert_filepath_caseinsensitive from allensdk.brain_observatory.stimulus_info import BrainObservatoryMonitor diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/templates.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/templates.py new file mode 100644 index 000000000..168a9d367 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/templates.py @@ -0,0 +1,139 @@ +import os +import numpy as np +from typing import Optional, List + +import imageio +from pynwb import NWBFile + +from allensdk.brain_observatory import nwb +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.stimuli.presentations \ + import \ + Presentations +from allensdk.brain_observatory.behavior.stimulus_processing import \ + get_stimulus_templates +from allensdk.brain_observatory.behavior.data_objects.stimuli \ + .stimulus_templates import \ + StimulusTemplate, StimulusTemplateFactory +from allensdk.brain_observatory.behavior.write_nwb.extensions\ + .stimulus_template.ndx_stimulus_template import \ + StimulusTemplateExtension +from allensdk.internal.core.lims_utilities import safe_system_path + + +class Templates(DataObject, StimulusFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, templates: StimulusTemplate): + super().__init__(name='stimulus_templates', value=templates) + + @classmethod + def from_stimulus_file( + cls, stimulus_file: StimulusFile, + limit_to_images: Optional[List] = None) -> "Templates": + """Get stimulus templates (movies, scenes) for behavior session.""" + + # TODO: Eventually the `grating_images_dict` should be provided by the + # BehaviorLimsExtractor/BehaviorJsonExtractor classes. + # - NJM 2021/2/23 + + gratings_dir = "/allen/programs/braintv/production/visualbehavior" + gratings_dir = os.path.join(gratings_dir, + "prod5/project_VisualBehavior") + grating_images_dict = { + "gratings_0.0": { + "warped": np.asarray(imageio.imread( + safe_system_path(os.path.join(gratings_dir, + "warped_grating_0.png")))), + "unwarped": np.asarray(imageio.imread( + safe_system_path(os.path.join( + gratings_dir, "masked_unwarped_grating_0.png")))) + }, + "gratings_90.0": { + "warped": np.asarray(imageio.imread( + safe_system_path(os.path.join(gratings_dir, + "warped_grating_90.png")))), + "unwarped": np.asarray(imageio.imread( + safe_system_path(os.path.join( + gratings_dir, "masked_unwarped_grating_90.png")))) + }, + "gratings_180.0": { + "warped": np.asarray(imageio.imread( + safe_system_path(os.path.join(gratings_dir, + "warped_grating_180.png")))), + "unwarped": np.asarray(imageio.imread( + safe_system_path(os.path.join( + gratings_dir, "masked_unwarped_grating_180.png")))) + }, + "gratings_270.0": { + "warped": np.asarray(imageio.imread( + safe_system_path(os.path.join(gratings_dir, + "warped_grating_270.png")))), + "unwarped": np.asarray(imageio.imread( + safe_system_path(os.path.join( + gratings_dir, "masked_unwarped_grating_270.png")))) + } + } + + pkl = stimulus_file.data + t = get_stimulus_templates(pkl=pkl, + grating_images_dict=grating_images_dict, + limit_to_images=limit_to_images) + return Templates(templates=t) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "Templates": + image_set_name = list(nwbfile.stimulus_template.keys())[0] + image_data = list(nwbfile.stimulus_template.values())[0] + + image_attributes = [{'image_name': image_name} + for image_name in image_data.control_description] + t = StimulusTemplateFactory.from_processed( + image_set_name=image_set_name, image_attributes=image_attributes, + warped=image_data.data[:], unwarped=image_data.unwarped[:] + ) + return Templates(templates=t) + + def to_nwb(self, nwbfile: NWBFile, + stimulus_presentations: Presentations) -> NWBFile: + stimulus_templates = self.value + + unwarped_images = [] + warped_images = [] + image_names = [] + for image_name, image_data in stimulus_templates.items(): + image_names.append(image_name) + unwarped_images.append(image_data.unwarped) + warped_images.append(image_data.warped) + + image_index = np.zeros(len(image_names)) + image_index[:] = np.nan + + visual_stimulus_image_series = \ + StimulusTemplateExtension( + name=stimulus_templates.image_set_name, + data=warped_images, + unwarped=unwarped_images, + control=list(range(len(image_names))), + control_description=image_names, + unit='NA', + format='raw', + timestamps=image_index) + + nwbfile.add_stimulus_template(visual_stimulus_image_series) + + # Add index for this template to NWB in-memory object: + nwb_template = nwbfile.stimulus_template[ + stimulus_templates.image_set_name] + stimulus_index = stimulus_presentations.value[ + stimulus_presentations.value[ + 'image_set'] == nwb_template.name] + nwb.add_stimulus_index(nwbfile, stimulus_index, nwb_template) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py new file mode 100644 index 000000000..b3f38b7eb --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/util.py @@ -0,0 +1,63 @@ +import warnings +from pathlib import Path + +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.equipment import \ + Equipment +from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner + + +def convert_filepath_caseinsensitive(filename_in): + return filename_in.replace('TRAINING', 'training') + + +def get_image_set_name(image_set_path: str) -> str: + """ + Strips the stem from the image_set filename + """ + return Path(image_set_path).stem + + +def calculate_monitor_delay(sync_file: SyncFile, + equipment: Equipment) -> float: + """Calculates monitor delay using sync file. If that fails, looks up + monitor delay from known values for equipment. + + Raises + -------- + RuntimeError + If input equipment is unknown + """ + aligner = OphysTimeAligner(sync_file=sync_file.filepath) + + try: + delay = aligner.monitor_delay + except ValueError as ee: + equipment_name = equipment.value + + warning_msg = 'Monitory delay calculation failed ' + warning_msg += 'with ValueError\n' + warning_msg += f' "{ee}"' + warning_msg += '\nlooking monitor delay up from table ' + warning_msg += f'for rig: {equipment_name} ' + + # see + # https://github.com/AllenInstitute/AllenSDK/issues/1318 + # https://github.com/AllenInstitute/AllenSDK/issues/1916 + delay_lookup = {'CAM2P.1': 0.020842, + 'CAM2P.2': 0.037566, + 'CAM2P.3': 0.021390, + 'CAM2P.4': 0.021102, + 'CAM2P.5': 0.021192, + 'MESO.1': 0.03613} + + if equipment_name not in delay_lookup: + msg = warning_msg + msg += f'\nequipment_name {equipment_name} not in lookup table' + raise RuntimeError(msg) + delay = delay_lookup[equipment_name] + warning_msg += f'\ndelay: {delay} seconds' + warnings.warn(warning_msg) + + return delay diff --git a/allensdk/brain_observatory/behavior/data_objects/task_parameters.py b/allensdk/brain_observatory/behavior/data_objects/task_parameters.py new file mode 100644 index 000000000..a1780ff49 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/task_parameters.py @@ -0,0 +1,234 @@ +from enum import Enum +import numpy as np +from typing import List + +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.schemas import \ + BehaviorTaskParametersSchema +from allensdk.brain_observatory.nwb import load_pynwb_extension + + +class BehaviorStimulusType(Enum): + IMAGES = 'images' + GRATING = 'grating' + + +class StimulusDistribution(Enum): + EXPONENTIAL = 'exponential' + GEOMETRIC = 'geometric' + + +class TaskType(Enum): + CHANGE_DETECTION = 'change detection' + + +class TaskParameters(DataObject, StimulusFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, + blank_duration_sec: List[float], + stimulus_duration_sec: float, + omitted_flash_fraction: float, + response_window_sec: List[float], + reward_volume: float, + auto_reward_volume: float, + session_type: str, + stimulus: str, + stimulus_distribution: StimulusDistribution, + task_type: TaskType, + n_stimulus_frames: int): + super().__init__(name='task_parameters', value=self) + self._blank_duration_sec = blank_duration_sec + self._stimulus_duration_sec = stimulus_duration_sec + self._omitted_flash_fraction = omitted_flash_fraction + self._response_window_sec = response_window_sec + self._reward_volume = reward_volume + self._auto_reward_volume = auto_reward_volume + self._session_type = session_type + self._stimulus = BehaviorStimulusType(stimulus) + self._stimulus_distribution = StimulusDistribution( + stimulus_distribution) + self._task = TaskType(task_type) + self._n_stimulus_frames = n_stimulus_frames + + @property + def blank_duration_sec(self) -> List[float]: + return self._blank_duration_sec + + @property + def stimulus_duration_sec(self) -> float: + return self._stimulus_duration_sec + + @property + def omitted_flash_fraction(self) -> float: + return self._omitted_flash_fraction + + @property + def response_window_sec(self) -> List[float]: + return self._response_window_sec + + @property + def reward_volume(self) -> float: + return self._reward_volume + + @property + def auto_reward_volume(self) -> float: + return self._auto_reward_volume + + @property + def session_type(self) -> str: + return self._session_type + + @property + def stimulus(self) -> str: + return self._stimulus + + @property + def stimulus_distribution(self) -> float: + return self._stimulus_distribution + + @property + def task(self) -> TaskType: + return self._task + + @property + def n_stimulus_frames(self) -> int: + return self._n_stimulus_frames + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + nwb_extension = load_pynwb_extension( + BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys' + ) + task_parameters = self.to_dict()['task_parameters'] + task_parameters_clean = BehaviorTaskParametersSchema().dump( + task_parameters + ) + + new_task_parameters_dict = {} + for key, val in task_parameters_clean.items(): + if isinstance(val, list): + new_task_parameters_dict[key] = np.array(val) + else: + new_task_parameters_dict[key] = val + nwb_task_parameters = nwb_extension( + name='task_parameters', **new_task_parameters_dict) + nwbfile.add_lab_meta_data(nwb_task_parameters) + return nwbfile + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "TaskParameters": + metadata_nwb_obj = nwbfile.lab_meta_data['task_parameters'] + data = BehaviorTaskParametersSchema().dump(metadata_nwb_obj) + data['task_type'] = data['task'] + del data['task'] + return TaskParameters(**data) + + @classmethod + def from_stimulus_file(cls, + stimulus_file: StimulusFile) -> "TaskParameters": + data = stimulus_file.data + + behavior = data["items"]["behavior"] + config = behavior["config"] + doc = config["DoC"] + + blank_duration_sec = [float(x) for x in doc['blank_duration_range']] + stim_duration = cls._calculate_stimulus_duration( + stimulus_file=stimulus_file) + omitted_flash_fraction = \ + behavior['params'].get('flash_omit_probability', float('nan')) + response_window_sec = [float(x) for x in doc["response_window"]] + reward_volume = config["reward"]["reward_volume"] + auto_reward_volume = doc['auto_reward_volume'] + session_type = behavior["params"]["stage"] + stimulus = next(iter(behavior["stimuli"])) + stimulus_distribution = doc["change_time_dist"] + task = cls._parse_task(stimulus_file=stimulus_file) + n_stimulus_frames = cls._calculuate_n_stimulus_frames( + stimulus_file=stimulus_file) + return TaskParameters( + blank_duration_sec=blank_duration_sec, + stimulus_duration_sec=stim_duration, + omitted_flash_fraction=omitted_flash_fraction, + response_window_sec=response_window_sec, + reward_volume=reward_volume, + auto_reward_volume=auto_reward_volume, + session_type=session_type, + stimulus=stimulus, + stimulus_distribution=stimulus_distribution, + task_type=task, + n_stimulus_frames=n_stimulus_frames + ) + + @staticmethod + def _calculate_stimulus_duration(stimulus_file: StimulusFile) -> float: + data = stimulus_file.data + + behavior = data["items"]["behavior"] + stimuli = behavior['stimuli'] + + def _parse_stimulus_key(): + if 'images' in stimuli: + stim_key = 'images' + elif 'grating' in stimuli: + stim_key = 'grating' + else: + msg = "Cannot get stimulus_duration_sec\n" + msg += "'images' and/or 'grating' not a valid " + msg += "key in pickle file under " + msg += "['items']['behavior']['stimuli']\n" + msg += f"keys: {list(stimuli.keys())}" + raise RuntimeError(msg) + + return stim_key + stim_key = _parse_stimulus_key() + stim_duration = stimuli[stim_key]['flash_interval_sec'] + + # from discussion in + # https://github.com/AllenInstitute/AllenSDK/issues/1572 + # + # 'flash_interval' contains (stimulus_duration, gray_screen_duration) + # (as @matchings said above). That second value is redundant with + # 'blank_duration_range'. I'm not sure what would happen if they were + # set to be conflicting values in the params. But it looks like + # they're always consistent. It should always be (0.25, 0.5), + # except for TRAINING_0 and TRAINING_1, which have statically + # displayed stimuli (no flashes). + + if stim_duration is None: + stim_duration = np.NaN + else: + stim_duration = stim_duration[0] + return stim_duration + + @staticmethod + def _parse_task(stimulus_file: StimulusFile) -> TaskType: + data = stimulus_file.data + config = data["items"]["behavior"]["config"] + + task_id = config['behavior']['task_id'] + if 'DoC' in task_id: + task = TaskType.CHANGE_DETECTION + else: + msg = "metadata.get_task_parameters does not " + msg += f"know how to parse 'task_id' = {task_id}" + raise RuntimeError(msg) + return task + + @staticmethod + def _calculuate_n_stimulus_frames(stimulus_file: StimulusFile) -> int: + data = stimulus_file.data + behavior = data["items"]["behavior"] + + n_stimulus_frames = 0 + for stim_type, stim_table in behavior["stimuli"].items(): + n_stimulus_frames += sum(stim_table.get("draw_log", [])) + return n_stimulus_frames diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/__init__.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/ophys_timestamps.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/ophys_timestamps.py new file mode 100644 index 000000000..7e9ad3e24 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/timestamps/ophys_timestamps.py @@ -0,0 +1,104 @@ +import logging + +import numpy as np +from pynwb import NWBFile + +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + SyncFileReadableInterface, NwbReadableInterface + + +class OphysTimestamps(DataObject, SyncFileReadableInterface, + NwbReadableInterface): + _logger = logging.getLogger(__name__) + + def __init__(self, timestamps: np.ndarray): + """ + :param timestamps + ophys timestamps + """ + super().__init__(name='ophys_timestamps', value=timestamps) + + @classmethod + def from_sync_file(cls, sync_file: SyncFile) -> "OphysTimestamps": + ophys_timestamps = sync_file.data['ophys_frames'] + return cls(timestamps=ophys_timestamps) + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "OphysTimestamps": + ts = nwbfile.processing[ + 'ophys'].get_data_interface('dff').roi_response_series[ + 'traces'].timestamps[:] + return cls(timestamps=ts) + + def validate(self, number_of_frames: int) -> "OphysTimestamps": + """Validates that number of ophys timestamps do not exceed number of + dff traces. If so, truncates number of ophys timestamps to the same + length as dff traces + + :param number_of_frames + number of frames in the movie + + Notes + --------- + Modifies self._value if ophys timestamps exceed length of + number_of_frames + """ + # Scientifica data has extra frames in the sync file relative + # to the number of frames in the video. These sentinel frames + # should be removed. + # NOTE: This fix does not apply to mesoscope data. + # See http://confluence.corp.alleninstitute.org/x/9DVnAg + ophys_timestamps = self.value + num_of_timestamps = len(ophys_timestamps) + if number_of_frames < num_of_timestamps: + self._logger.info( + "Truncating acquisition frames ('ophys_frames') " + f"(len={num_of_timestamps}) to the number of frames " + f"in the df/f trace ({number_of_frames}).") + self._value = ophys_timestamps[:number_of_frames] + elif number_of_frames > num_of_timestamps: + raise RuntimeError( + f"dff_frames (len={number_of_frames}) is longer " + f"than timestamps (len={num_of_timestamps}).") + return self + + +class OphysTimestampsMultiplane(OphysTimestamps): + def __init__(self, timestamps: np.ndarray): + super().__init__(timestamps=timestamps) + + @classmethod + def from_sync_file(cls, sync_file: SyncFile, + group_count: int, + plane_group: int) -> "OphysTimestampsMultiplane": + if group_count == 0: + raise ValueError('Group count cannot be 0') + + ophys_timestamps = sync_file.data['ophys_frames'] + cls._logger.info( + "Mesoscope data detected. Splitting timestamps " + f"(len={len(ophys_timestamps)} over {group_count} " + "plane group(s).") + + # Resample if collecting multiple concurrent planes + # because the frames are interleaved + ophys_timestamps = ophys_timestamps[plane_group::group_count] + + return cls(timestamps=ophys_timestamps) + + def validate(self, number_of_frames: int) -> "OphysTimestampsMultiplane": + """ + Raises error if length of timestamps and number of frames are not equal + :param number_of_frames + See super().validate + """ + ophys_timestamps = self.value + num_of_timestamps = len(ophys_timestamps) + if number_of_frames != num_of_timestamps: + raise RuntimeError( + f"dff_frames (len={number_of_frames}) is not equal to " + f"number of split timestamps (len={num_of_timestamps}).") + return self diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/__init__.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/stimulus_timestamps.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/stimulus_timestamps.py new file mode 100644 index 000000000..2df1042c4 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/stimulus_timestamps.py @@ -0,0 +1,142 @@ + +import json +from typing import Optional + +from cachetools.keys import hashkey + +import numpy as np +from pynwb import NWBFile, ProcessingModule +from pynwb.base import TimeSeries + +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + JsonReadableInterface, LimsReadableInterface, NwbReadableInterface, \ + StimulusFileReadableInterface, SyncFileReadableInterface +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_files import ( + StimulusFile, SyncFile +) +from allensdk.brain_observatory.behavior.data_objects.base \ + .writable_interfaces import \ + JsonWritableInterface, NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .stimulus_timestamps.timestamps_processing import ( + get_behavior_stimulus_timestamps, get_ophys_stimulus_timestamps) +from allensdk.internal.api import PostgresQueryMixin + + +def from_json_cache_key(cls, dict_repr: dict): + return hashkey(json.dumps(dict_repr)) + + +def from_lims_cache_key( + cls, db, behavior_session_id: int, + ophys_experiment_id: Optional[int] = None +): + return hashkey(behavior_session_id, ophys_experiment_id) + + +class StimulusTimestamps(DataObject, StimulusFileReadableInterface, + SyncFileReadableInterface, JsonReadableInterface, + NwbReadableInterface, LimsReadableInterface, + NwbWritableInterface, JsonWritableInterface): + """A DataObject which contains properties and methods to load, process, + and represent visual behavior stimulus timestamp data. + + Stimulus timestamp data is represented as: + + Numpy array whose length is equal to the number of timestamps collected + and whose values are timestamps (in seconds) + """ + + def __init__( + self, + timestamps: np.ndarray, + stimulus_file: Optional[StimulusFile] = None, + sync_file: Optional[SyncFile] = None + ): + super().__init__(name="stimulus_timestamps", value=timestamps) + self._stimulus_file = stimulus_file + self._sync_file = sync_file + + @classmethod + def from_stimulus_file( + cls, + stimulus_file: StimulusFile) -> "StimulusTimestamps": + stimulus_timestamps = get_behavior_stimulus_timestamps( + stimulus_pkl=stimulus_file.data + ) + + return cls( + timestamps=stimulus_timestamps, + stimulus_file=stimulus_file + ) + + @classmethod + def from_sync_file(cls, sync_file: SyncFile) -> "StimulusTimestamps": + stimulus_timestamps = get_ophys_stimulus_timestamps( + sync_path=sync_file.filepath + ) + return cls( + timestamps=stimulus_timestamps, + sync_file=sync_file + ) + + @classmethod + def from_json(cls, dict_repr: dict) -> "StimulusTimestamps": + if 'sync_file' in dict_repr: + sync_file = SyncFile.from_json(dict_repr=dict_repr) + return cls.from_sync_file(sync_file=sync_file) + else: + stim_file = StimulusFile.from_json(dict_repr=dict_repr) + return cls.from_stimulus_file(stimulus_file=stim_file) + + def from_lims( + cls, + db: PostgresQueryMixin, + behavior_session_id: int, + ophys_experiment_id: Optional[int] = None + ) -> "StimulusTimestamps": + stimulus_file = StimulusFile.from_lims(db, behavior_session_id) + + if ophys_experiment_id: + sync_file = SyncFile.from_lims( + db=db, ophys_experiment_id=ophys_experiment_id) + return cls.from_sync_file(sync_file=sync_file) + else: + return cls.from_stimulus_file(stimulus_file=stimulus_file) + + def to_json(self) -> dict: + if self._stimulus_file is None: + raise RuntimeError( + "StimulusTimestamps DataObject lacks information about the " + "StimulusFile. This is likely due to instantiating from NWB " + "which prevents to_json() functionality" + ) + + output_dict = dict() + output_dict.update(self._stimulus_file.to_json()) + if self._sync_file is not None: + output_dict.update(self._sync_file.to_json()) + return output_dict + + @classmethod + def from_nwb(cls, nwbfile: NWBFile) -> "StimulusTimestamps": + stim_module = nwbfile.processing["stimulus"] + stim_ts_interface = stim_module.get_data_interface("timestamps") + stim_timestamps = stim_ts_interface.timestamps[:] + return cls(timestamps=stim_timestamps) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + stimulus_ts = TimeSeries( + data=self._value, + name="timestamps", + timestamps=self._value, + unit="s" + ) + + stim_mod = ProcessingModule("stimulus", "Stimulus Times processing") + stim_mod.add_data_interface(stimulus_ts) + nwbfile.add_processing_module(stim_mod) + + return nwbfile diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/timestamps_processing.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/timestamps_processing.py new file mode 100644 index 000000000..b34f9b0e1 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/timestamps/stimulus_timestamps/timestamps_processing.py @@ -0,0 +1,49 @@ +from typing import Union +from pathlib import Path + +import numpy as np + +from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner + + +def get_behavior_stimulus_timestamps(stimulus_pkl: dict) -> np.ndarray: + """Obtain visual behavior stimuli timing information from a behavior + stimulus *.pkl file. + + Parameters + ---------- + stimulus_pkl : dict + A dictionary containing stimulus presentation timing information + during a behavior session. Presentation timing info is stored as + an array of times between frames (frame time intervals) in + milliseconds. + + Returns + ------- + np.ndarray + Timestamps (in seconds) for presented stimulus frames during a session. + """ + vsyncs = stimulus_pkl["items"]["behavior"]["intervalsms"] + stimulus_timestamps = np.hstack((0, vsyncs)).cumsum() / 1000.0 + return stimulus_timestamps + + +def get_ophys_stimulus_timestamps(sync_path: Union[str, Path]) -> np.ndarray: + """Obtain visual behavior stimuli timing information from a sync *.h5 file. + + Parameters + ---------- + sync_path : Union[str, Path] + The path to a sync *.h5 file that contains global timing information + about multiple data streams (e.g. behavior, ophys, eye_tracking) + during a session. + + Returns + ------- + np.ndarray + Timestamps (in seconds) for presented stimulus frames during a + behavior + ophys session. + """ + aligner = OphysTimeAligner(sync_file=sync_path) + stimulus_timestamps, _ = aligner.clipped_stim_timestamps + return stimulus_timestamps diff --git a/allensdk/brain_observatory/behavior/data_objects/timestamps/util.py b/allensdk/brain_observatory/behavior/data_objects/timestamps/util.py new file mode 100644 index 000000000..8e162adde --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/timestamps/util.py @@ -0,0 +1,5 @@ +import numpy as np + + +def calc_frame_rate(timestamps: np.ndarray): + return np.round(1 / np.mean(np.diff(timestamps)), 0) diff --git a/allensdk/brain_observatory/behavior/data_objects/trials/__init__.py b/allensdk/brain_observatory/behavior/data_objects/trials/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/allensdk/brain_observatory/behavior/data_objects/trials/trial.py b/allensdk/brain_observatory/behavior/data_objects/trials/trial.py new file mode 100644 index 000000000..e8d372999 --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/trials/trial.py @@ -0,0 +1,423 @@ +from typing import List, Dict, Any, Tuple + +import numpy as np + +from allensdk import one +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.licks import Licks +from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards + + +class Trial: + def __init__(self, trial: dict, start: float, end: float, + behavior_stimulus_file: StimulusFile, + index: int, monitor_delay: float, + stimulus_timestamps: StimulusTimestamps, + licks: Licks, rewards: Rewards, stimuli: dict): + self._trial = trial + self._start = start + self._end = self._calculate_trial_end( + trial_end=end, behavior_stimulus_file=behavior_stimulus_file) + self._index = index + self._data = self._match_to_sync_timestamps( + monitor_delay=monitor_delay, + stimulus_timestamps=stimulus_timestamps, licks=licks, + rewards=rewards, stimuli=stimuli) + + @property + def data(self): + return self._data + + def _match_to_sync_timestamps( + self, monitor_delay: float, + stimulus_timestamps: StimulusTimestamps, + licks: Licks, rewards: Rewards, + stimuli: dict) -> Dict[str, Any]: + event_dict = { + (e[0], e[1]): { + 'timestamp': stimulus_timestamps.value[e[3]], + 'frame': e[3]} for e in self._trial['events'] + } + + tr_data = {"trial": self._trial["index"]} + lick_frames = licks.value['frame'].values + timestamps = stimulus_timestamps.value + reward_times = rewards.value['timestamps'].values + + # this block of code is trying to mimic + # https://github.com/AllenInstitute/visual_behavior_analysis + # /blob/master/visual_behavior/translator/foraging2 + # /stimulus_processing.py + # #L377-L381 + # https://github.com/AllenInstitute/visual_behavior_analysis + # /blob/master/visual_behavior/translator/foraging2 + # /extract_movies.py#L59-L94 + # https://github.com/AllenInstitute/visual_behavior_analysis + # /blob/master/visual_behavior/translator/core/annotate.py#L11-L36 + # + # In summary: there are cases where an "epilogue movie" is shown + # after the proper stimuli; we do not want licks that occur + # during this epilogue movie to be counted as belonging to + # the last trial + # https://github.com/AllenInstitute/visual_behavior_analysis + # /issues/482 + + # select licks that fall between trial_start and trial_end; + # licks on the boundary get assigned to the trial that is ending, + # rather than the trial that is starting + if self._end > 0: + valid_idx = np.where(np.logical_and(lick_frames > self._start, + lick_frames <= self._end)) + else: + valid_idx = np.where(lick_frames > self._start) + + valid_licks = lick_frames[valid_idx] + if len(valid_licks) > 0: + tr_data["lick_times"] = timestamps[valid_licks] + else: + tr_data["lick_times"] = np.array([], dtype=float) + + tr_data["reward_time"] = self._get_reward_time( + reward_times, + event_dict[('trial_start', '')]['timestamp'], + event_dict[('trial_end', '')]['timestamp'] + ) + tr_data.update(self._get_trial_data()) + tr_data.update(self._get_trial_timing( + event_dict, + tr_data['lick_times'], + tr_data['go'], + tr_data['catch'], + tr_data['auto_rewarded'], + tr_data['hit'], + tr_data['false_alarm'], + tr_data["aborted"], + timestamps, + monitor_delay + )) + tr_data.update(self._get_trial_image_names(stimuli)) + + self._validate_trial_condition_exclusivity(tr_data=tr_data) + + return tr_data + + @staticmethod + def _get_reward_time(rebased_reward_times, + start_time, + stop_time) -> float: + """extract reward times in time range""" + reward_times = rebased_reward_times[np.where(np.logical_and( + rebased_reward_times >= start_time, + rebased_reward_times <= stop_time + ))] + return float('nan') if len(reward_times) == 0 else one( + reward_times) + + @staticmethod + def _calculate_trial_end(trial_end, + behavior_stimulus_file: StimulusFile) -> int: + if trial_end < 0: + bhv = behavior_stimulus_file.data['items']['behavior']['items'] + if 'fingerprint' in bhv.keys(): + trial_end = bhv['fingerprint']['starting_frame'] + return trial_end + + def _get_trial_data(self) -> Dict[str, Any]: + """ + Infer trial logic from trial log. Returns a dictionary. + + * reward volume: volume of water delivered on the trial, in mL + + Each of the following values is boolean: + + Trial category values are mutually exclusive + * go: trial was a go trial (trial with a stimulus change) + * catch: trial was a catch trial (trial with a sham stimulus change) + + stimulus_change/sham_change are mutually exclusive + * stimulus_change: did the stimulus change (True on 'go' trials) + * sham_change: stimulus did not change, but response was evaluated + (True on 'catch' trials) + + Each trial can be one (and only one) of the following: + * hit (stimulus changed, animal responded in response window) + * miss (stimulus changed, animal did not respond in response window) + * false_alarm (stimulus did not change, + animal responded in response window) + * correct_reject (stimulus did not change, + animal did not respond in response window) + * aborted (animal responded before change time) + * auto_rewarded (reward was automatically delivered following the + change. + This will bias the animals choice and should not be + categorized as hit/miss) + """ + trial_event_names = [val[0] for val in self._trial['events']] + hit = 'hit' in trial_event_names + false_alarm = 'false_alarm' in trial_event_names + miss = 'miss' in trial_event_names + sham_change = 'sham_change' in trial_event_names + stimulus_change = 'stimulus_changed' in trial_event_names + aborted = 'abort' in trial_event_names + + if aborted: + go = catch = auto_rewarded = False + else: + catch = self._trial["trial_params"]["catch"] is True + auto_rewarded = self._trial["trial_params"]["auto_reward"] + go = not catch and not auto_rewarded + + correct_reject = catch and not false_alarm + + if auto_rewarded: + hit = miss = correct_reject = false_alarm = False + + return { + "reward_volume": sum([ + r[0] for r in self._trial.get("rewards", [])]), + "hit": hit, + "false_alarm": false_alarm, + "miss": miss, + "sham_change": sham_change, + "stimulus_change": stimulus_change, + "aborted": aborted, + "go": go, + "catch": catch, + "auto_rewarded": auto_rewarded, + "correct_reject": correct_reject, + } + + @staticmethod + def _get_trial_timing( + event_dict: dict, + licks: List[float], go: bool, catch: bool, auto_rewarded: bool, + hit: bool, false_alarm: bool, aborted: bool, + timestamps: np.ndarray, + monitor_delay: float) -> Dict[str, Any]: + """ + Extract a dictionary of trial timing data. + See trial_data_from_log for a description of the trial types. + + Parameters + ========== + event_dict: dict + Dictionary of trial events in the well-known `pkl` file + licks: List[float] + list of lick timestamps, from the `get_licks` response for + the BehaviorOphysExperiment.api. + go: bool + True if "go" trial, False otherwise. Mutually exclusive with + `catch`. + catch: bool + True if "catch" trial, False otherwise. Mutually exclusive + with `go.` + auto_rewarded: bool + True if "auto_rewarded" trial, False otherwise. + hit: bool + True if "hit" trial, False otherwise + false_alarm: bool + True if "false_alarm" trial, False otherwise + aborted: bool + True if "aborted" trial, False otherwise + timestamps: np.ndarray[1d] + Array of ground truth timestamps for the session + (sync times, if available) + monitor_delay: float + The monitor delay in seconds associated with the session + + Returns + ======= + dict + start_time: float + The time the trial started (in seconds elapsed from + recording start) + stop_time: float + The time the trial ended (in seconds elapsed from + recording start) + trial_length: float + Duration of the trial in seconds + response_time: float + The response time, for non-aborted trials. This is equal + to the first lick in the trial. For aborted trials or trials + without licks, `response_time` is NaN. + change_frame: int + The frame number that the stimulus changed + change_time: float + The time in seconds that the stimulus changed + response_latency: float or None + The time in seconds between the stimulus change and the + animal's lick response, if the trial is a "go", "catch", or + "auto_rewarded" type. If the animal did not respond, + return `float("inf")`. In all other cases, return None. + + Notes + ===== + The following parameters are mutually exclusive (exactly one can + be true): + hit, miss, false_alarm, aborted, auto_rewarded + """ + assert not (aborted and (hit or false_alarm or auto_rewarded)), ( + "'aborted' trials cannot be 'hit', 'false_alarm', " + "or 'auto_rewarded'") + assert not (hit and false_alarm), ( + "both `hit` and `false_alarm` cannot be True, they are mutually " + "exclusive categories") + assert not (go and catch), ( + "both `go` and `catch` cannot be True, they are mutually " + "exclusive " + "categories") + assert not (go and auto_rewarded), ( + "both `go` and `auto_rewarded` cannot be True, they are mutually " + "exclusive categories") + + def _get_response_time(licks: List[float], aborted: bool) -> float: + """ + Return the time the first lick occurred in a non-"aborted" trial. + A response time is not returned for on an "aborted trial", since by + definition, the animal licked before the change stimulus. + """ + if aborted: + return float("nan") + if len(licks): + return licks[0] + else: + return float("nan") + + start_time = event_dict["trial_start", ""]['timestamp'] + stop_time = event_dict["trial_end", ""]['timestamp'] + + response_time = _get_response_time(licks, aborted) + + if go or auto_rewarded: + change_frame = event_dict.get(('stimulus_changed', ''))['frame'] + change_time = timestamps[change_frame] + monitor_delay + elif catch: + change_frame = event_dict.get(('sham_change', ''))['frame'] + change_time = timestamps[change_frame] + monitor_delay + else: + change_time = float("nan") + change_frame = float("nan") + + if not (go or catch or auto_rewarded): + response_latency = None + elif len(licks) > 0: + response_latency = licks[0] - change_time + else: + response_latency = float("inf") + + return { + "start_time": start_time, + "stop_time": stop_time, + "trial_length": stop_time - start_time, + "response_time": response_time, + "change_frame": change_frame, + "change_time": change_time, + "response_latency": response_latency, + } + + def _get_trial_image_names(self, stimuli) -> Dict[str, str]: + """ + Gets the name of the stimulus presented at the beginning of the + trial and + what is it changed to at the end of the trial. + Parameters + ---------- + stimuli: The stimuli presentation log for the behavior session + + Returns + ------- + A dictionary indicating the starting_stimulus and what the + stimulus is + changed to. + + """ + grating_oris = {'horizontal', 'vertical'} + trial_start_frame = self._trial["events"][0][3] + initial_image_category_name, _, initial_image_name = \ + self._resolve_initial_image( + stimuli, trial_start_frame) + if len(self._trial["stimulus_changes"]) == 0: + change_image_name = initial_image_name + else: + ((from_set, from_name), + (to_set, to_name), + _, _) = self._trial["stimulus_changes"][0] + + # do this to fix names if the stimuli is a grating + if from_set in grating_oris: + from_name = f'gratings_{from_name}' + if to_set in grating_oris: + to_name = f'gratings_{to_name}' + assert from_name == initial_image_name + change_image_name = to_name + + return { + "initial_image_name": initial_image_name, + "change_image_name": change_image_name + } + + @staticmethod + def _resolve_initial_image(stimuli, start_frame) -> Tuple[str, str, str]: + """Attempts to resolve the initial image for a given start_frame for + a trial + + Parameters + ---------- + stimuli: Mapping + foraging2 shape stimuli mapping + start_frame: int + start frame of the trial + + Returns + ------- + initial_image_category_name: str + stimulus category of initial image + initial_image_group: str + group name of the initial image + initial_image_name: str + name of the initial image + """ + max_frame = float("-inf") + initial_image_group = '' + initial_image_name = '' + initial_image_category_name = '' + + for stim_category_name, stim_dict in stimuli.items(): + for set_event in stim_dict["set_log"]: + set_frame = set_event[3] + if start_frame >= set_frame >= max_frame: + # hack assumes initial_image_group == initial_image_name, + # only initial_image_name is present for natual_scenes + initial_image_group = initial_image_name = set_event[1] + initial_image_category_name = stim_category_name + if initial_image_category_name == 'grating': + initial_image_name = f'gratings_{initial_image_name}' + max_frame = set_frame + + return initial_image_category_name, initial_image_group, \ + initial_image_name + + def _validate_trial_condition_exclusivity(self, tr_data: dict) -> None: + """ensure that only one of N possible mutually + exclusive trial conditions is True""" + trial_conditions = {} + for key in ['hit', + 'miss', + 'false_alarm', + 'correct_reject', + 'auto_rewarded', + 'aborted']: + trial_conditions[key] = tr_data[key] + + on = [] + for condition, value in trial_conditions.items(): + if value: + on.append(condition) + + if len(on) != 1: + all_conditions = list(trial_conditions.keys()) + msg = f"expected exactly 1 trial condition out of " \ + f"{all_conditions} " + msg += f"to be True, instead {on} were True (trial {self._index})" + raise AssertionError(msg) diff --git a/allensdk/brain_observatory/behavior/data_objects/trials/trial_table.py b/allensdk/brain_observatory/behavior/data_objects/trials/trial_table.py new file mode 100644 index 000000000..6650020bd --- /dev/null +++ b/allensdk/brain_observatory/behavior/data_objects/trials/trial_table.py @@ -0,0 +1,150 @@ +from typing import List, Tuple + +import pandas as pd +from pynwb import NWBFile + +from allensdk.brain_observatory import dict_to_indexed_array +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import DataObject, \ + StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.base \ + .readable_interfaces import \ + StimulusFileReadableInterface, NwbReadableInterface +from allensdk.brain_observatory.behavior.data_objects.base\ + .writable_interfaces import \ + NwbWritableInterface +from allensdk.brain_observatory.behavior.data_objects.licks import Licks +from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards +from allensdk.brain_observatory.behavior.data_objects.trials.trial import Trial + + +class TrialTable(DataObject, StimulusFileReadableInterface, + NwbReadableInterface, NwbWritableInterface): + def __init__(self, trials: pd.DataFrame): + super().__init__(name='trials', value=trials) + + def to_nwb(self, nwbfile: NWBFile) -> NWBFile: + trials = self.value + order = list(trials.index) + for _, row in trials[['start_time', 'stop_time']].iterrows(): + row_dict = row.to_dict() + nwbfile.add_trial(**row_dict) + + for c in trials.columns: + if c in ['start_time', 'stop_time']: + continue + index, data = dict_to_indexed_array(trials[c].to_dict(), order) + if data.dtype == ' "TrialTable": + trials = nwbfile.trials.to_dataframe() + if 'lick_events' in trials.columns: + trials.drop('lick_events', inplace=True, axis=1) + trials.index = trials.index.rename('trials_id') + return TrialTable(trials=trials) + + @classmethod + def from_stimulus_file(cls, stimulus_file: StimulusFile, + stimulus_timestamps: StimulusTimestamps, + licks: Licks, + rewards: Rewards, + monitor_delay: float + ) -> "TrialTable": + bsf = stimulus_file.data + + stimuli = bsf["items"]["behavior"]["stimuli"] + trial_log = bsf["items"]["behavior"]["trial_log"] + + trial_bounds = cls._get_trial_bounds(trial_log=trial_log) + + all_trial_data = [None] * len(trial_log) + + for idx, trial in enumerate(trial_log): + trial_start, trial_end = trial_bounds[idx] + t = Trial(trial=trial, start=trial_start, end=trial_end, + behavior_stimulus_file=stimulus_file, + index=idx, + monitor_delay=monitor_delay, + stimulus_timestamps=stimulus_timestamps, + licks=licks, rewards=rewards, + stimuli=stimuli + ) + all_trial_data[idx] = t.data + + trials = pd.DataFrame(all_trial_data).set_index('trial') + trials.index = trials.index.rename('trials_id') + + # Order/Filter columns + trials = trials[['initial_image_name', 'change_image_name', + 'stimulus_change', 'change_time', + 'go', 'catch', 'lick_times', 'response_time', + 'response_latency', 'reward_time', 'reward_volume', + 'hit', 'false_alarm', 'miss', 'correct_reject', + 'aborted', 'auto_rewarded', 'change_frame', + 'start_time', 'stop_time', 'trial_length']] + + return TrialTable(trials=trials) + + @staticmethod + def _get_trial_bounds(trial_log: List) -> List[Tuple[int, int]]: + """ + Adjust trial boundaries from a trial_log so that there is no dead time + between trials. + + Parameters + ---------- + trial_log: list + The trial_log read in from the well known behavior stimulus + pickle file + + Returns + ------- + list + Each element in the list is a tuple of the form + (start_frame, end_frame) so that the ith element + of the list gives the start and end frames of + the ith trial. The endframe of the last trial will + be -1, indicating that it should map to the last + timestamp in the session + """ + start_frames = [] + + for trial in trial_log: + start_f = None + for event in trial['events']: + if event[0] == 'trial_start': + start_f = event[-1] + break + if start_f is None: + msg = "Could not find a 'trial_start' event " + msg += "for all trials in the trial log\n" + msg += f"{trial}" + raise ValueError(msg) + + if len(start_frames) > 0 and start_f < start_frames[-1]: + msg = "'trial_start' frames in trial log " + msg += "are not in ascending order" + msg += f"\ntrial_log: {trial_log}" + raise ValueError(msg) + + start_frames.append(start_f) + + end_frames = [idx for idx in start_frames[1:] + [-1]] + return list([(s, e) for s, e in zip(start_frames, end_frames)]) diff --git a/allensdk/brain_observatory/behavior/image_api.py b/allensdk/brain_observatory/behavior/image_api.py index 509a39f92..8418d1bcd 100644 --- a/allensdk/brain_observatory/behavior/image_api.py +++ b/allensdk/brain_observatory/behavior/image_api.py @@ -16,7 +16,7 @@ class Image(NamedTuple): data: np.ndarray spacing: tuple - unit: int = 0 + unit: str = 'mm' def __eq__(self, other): a = np.array_equal(self.data, other.data) @@ -27,6 +27,7 @@ def __eq__(self, other): def __array__(self): return np.array(self.data) + class ImageApi: @staticmethod diff --git a/allensdk/brain_observatory/behavior/metadata/behavior_ophys_metadata.py b/allensdk/brain_observatory/behavior/metadata/behavior_ophys_metadata.py deleted file mode 100644 index 537e6b34c..000000000 --- a/allensdk/brain_observatory/behavior/metadata/behavior_ophys_metadata.py +++ /dev/null @@ -1,96 +0,0 @@ -import numpy as np -from typing import Optional - -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_ophys_data_extractor_base import \ - BehaviorOphysDataExtractorBase - - -class BehaviorOphysMetadata(BehaviorMetadata): - """Container class for behavior ophys metadata""" - def __init__(self, extractor: BehaviorOphysDataExtractorBase, - stimulus_timestamps: np.ndarray, - ophys_timestamps: np.ndarray, - behavior_stimulus_file: dict): - - super().__init__(extractor=extractor, - stimulus_timestamps=stimulus_timestamps, - behavior_stimulus_file=behavior_stimulus_file) - self._extractor = extractor - self._ophys_timestamps = ophys_timestamps - - # project_code needs to be excluded from comparison - # since it's only exposed internally - self._exclude_from_equals = {'project_code'} - - @property - def indicator(self) -> Optional[str]: - """Parses indicator from reporter""" - reporter_line = self.reporter_line - return self.parse_indicator(reporter_line=reporter_line, warn=True) - - @property - def emission_lambda(self) -> float: - return 520.0 - - @property - def excitation_lambda(self) -> float: - return 910.0 - - # TODO rename to ophys_container_id - @property - def experiment_container_id(self) -> int: - return self._extractor.get_ophys_container_id() - - @property - def field_of_view_height(self) -> int: - return self._extractor.get_field_of_view_shape()['height'] - - @property - def field_of_view_width(self) -> int: - return self._extractor.get_field_of_view_shape()['width'] - - @property - def imaging_depth(self) -> int: - return self._extractor.get_imaging_depth() - - @property - def imaging_plane_group(self) -> Optional[int]: - return self._extractor.get_imaging_plane_group() - - @property - def imaging_plane_group_count(self) -> int: - return self._extractor.get_plane_group_count() - - @property - def ophys_experiment_id(self) -> int: - return self._extractor.get_ophys_experiment_id() - - @property - def ophys_frame_rate(self) -> float: - return self._get_frame_rate(timestamps=self._ophys_timestamps) - - @property - def ophys_session_id(self) -> int: - return self._extractor.get_ophys_session_id() - - @property - def project_code(self) -> Optional[str]: - try: - project_code = self._extractor.get_project_code() - except NotImplementedError: - # Project code only returned by LIMS - project_code = None - return project_code - - @property - def targeted_structure(self) -> str: - return self._extractor.get_targeted_structure() - - def to_dict(self) -> dict: - """Returns dict representation of all properties in class""" - vars_ = vars(BehaviorOphysMetadata) - d = self._get_properties(vars_=vars_) - return {**super().to_dict(), **d} diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_data_extractor_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_data_extractor_base.py deleted file mode 100644 index 6ca354a75..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_data_extractor_base.py +++ /dev/null @@ -1,92 +0,0 @@ -import abc -import pandas as pd -from datetime import datetime -from typing import List - -from allensdk.api.warehouse_cache.cache import memoize - - -class BehaviorDataExtractorBase(abc.ABC): - """Abstract base class implementing required methods for extracting - data (from LIMS or from JSON) that will be transformed or passed on to - fill behavior session data. - """ - - @abc.abstractmethod - def get_behavior_session_id(self) -> int: - """Get the ID of the behavior session""" - raise NotImplementedError() - - @abc.abstractmethod - def get_foraging_id(self) -> int: - """Get the foraging ID for the behavior session""" - raise NotImplementedError() - - @abc.abstractmethod - def get_equipment_name(self) -> str: - """Get the name of the experiment rig (ex: CAM2P.3)""" - raise NotImplementedError() - - @abc.abstractmethod - def get_sex(self) -> str: - """Get the sex of the subject (ex: 'M', 'F', or 'unknown')""" - raise NotImplementedError() - - @abc.abstractmethod - def get_age(self) -> str: - """Get the age code of the subject (ie P123)""" - raise NotImplementedError() - - @memoize - def get_stimulus_name(self) -> str: - """Get the stimulus set used from the behavior session pkl file - :rtype: str - """ - behavior_stimulus_path = self.get_behavior_stimulus_file() - pkl = pd.read_pickle(behavior_stimulus_path) - - try: - stimulus_name = pkl["items"]["behavior"]["cl_params"]["stage"] - except KeyError: - raise RuntimeError( - f"Could not obtain stimulus_name/stage information from " - f"the *.pkl file ({behavior_stimulus_path}) " - f"for the behavior session to save as NWB! The " - f"following series of nested keys did not work: " - f"['items']['behavior']['cl_params']['stage']" - ) - return stimulus_name - - @abc.abstractmethod - def get_reporter_line(self) -> List[str]: - """Get the (gene) reporter line(s) for the subject associated with a - behavior or behavior + ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_driver_line(self) -> List[str]: - """Get the (gene) driver line(s) for the subject associated with a - behavior or behavior + ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_full_genotype(self) -> str: - """Get the full genotype of the subject associated with a - behavior or behavior + ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_behavior_stimulus_file(self) -> str: - """Get the filepath to the StimulusPickle file for the session""" - raise NotImplementedError() - - @abc.abstractmethod - def get_mouse_id(self) -> int: - """Get the mouse id (LabTracks ID) for the subject - associated with a behavior experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_date_of_acquisition(self) -> datetime: - """Get the acquisition date of an experiment in UTC""" - raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_ophys_data_extractor_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_ophys_data_extractor_base.py deleted file mode 100644 index cf63a111d..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/abcs/data_extractor_base/behavior_ophys_data_extractor_base.py +++ /dev/null @@ -1,142 +0,0 @@ -import abc -from typing import Dict, Optional - -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_data_extractor_base import \ - BehaviorDataExtractorBase - - -class BehaviorOphysDataExtractorBase(BehaviorDataExtractorBase): - """Abstract base class implementing required methods for extracting - data (from LIMS or from JSON) that will be transformed or passed on to - fill behavior + ophys session data. - """ - - @abc.abstractmethod - def get_ophys_experiment_id(self) -> int: - """Return the ophys experiment id (experiments are an internal alias - for an imaging plane)""" - raise NotImplementedError() - - @abc.abstractmethod - def get_ophys_session_id(self) -> int: - """Return the ophys session id""" - raise NotImplementedError() - - @abc.abstractmethod - def get_surface_2p_pixel_size_um(self) -> float: - """Get the pixel size for 2-photon movies in micrometers""" - raise NotImplementedError() - - @abc.abstractmethod - def get_sync_file(self) -> str: - """Get the filepath of the sync timing file associated with the - ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_field_of_view_shape(self) -> Dict[str, int]: - """Get a field of view dictionary for a given ophys experiment. - ex: {"width": int, "height": int} - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_ophys_container_id(self) -> int: - """Get the experiment container id associated with an ophys - experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_targeted_structure(self) -> str: - """Get the targeted structure (acronym) for an ophys experiment - (ex: "Visp")""" - raise NotImplementedError() - - @abc.abstractmethod - def get_imaging_depth(self) -> int: - """Get the imaging depth for an ophys experiment - (ex: 400, 500, etc.)""" - raise NotImplementedError() - - @abc.abstractmethod - def get_dff_file(self) -> str: - """Get the filepath of the dff trace file associated with an ophys - experiment.""" - raise NotImplementedError() - - @abc.abstractmethod - def get_event_detection_filepath(self) -> str: - """Get the filepath of the .h5 events file associated with an ophys - experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_ophys_cell_segmentation_run_id(self) -> int: - """Get the ophys cell segmentation run id associated with an - ophys experiment id""" - raise NotImplementedError() - - @abc.abstractmethod - def get_raw_cell_specimen_table_dict(self) -> dict: - """Get the cell_rois table from LIMS in dictionary form""" - raise NotImplementedError() - - @abc.abstractmethod - def get_demix_file(self) -> str: - """Get the filepath of the demixed traces file associated with an - ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_average_intensity_projection_image_file(self) -> str: - """Get the avg intensity project image filepath associated with an - ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_max_projection_file(self) -> str: - """Get the filepath of the max projection image associated with the - ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_rigid_motion_transform_file(self) -> str: - """Get the filepath for the motion transform file (.csv) associated - with an ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_imaging_plane_group(self) -> Optional[int]: - """Get the imaging plane group number. This is a numeric index - that indicates the order that the frames were acquired when - there is more than one frame acquired concurrently. Relevant for - mesoscope data timestamps, as the laser jumps between plane - groups during the scan. Will be None for non-mesoscope data. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_plane_group_count(self) -> int: - """Gets the total number of plane groups in the session. - This is required for resampling ophys timestamps for mesoscope - data. Will be 0 if the scope did not capture multiple concurrent - frames. See `get_imaging_plane_group` for more info. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_eye_tracking_rig_geometry(self) -> dict: - """Get the eye tracking rig geometry associated with an - ophys experiment""" - raise NotImplementedError() - - @abc.abstractmethod - def get_eye_tracking_filepath(self) -> dict: - """Get the eye tracking filepath containing ellipse fits""" - raise NotImplementedError() - - @abc.abstractmethod - def get_project_code(self) -> str: - """Get the project code.""" - raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_base.py deleted file mode 100644 index 3a3411af9..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_base.py +++ /dev/null @@ -1,158 +0,0 @@ -import abc -from typing import Union - -import numpy as np -import pandas as pd - -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.brain_observatory.behavior.stimulus_processing import \ - StimulusTemplate - - -class BehaviorBase(abc.ABC): - """Abstract base class implementing required methods for interacting with - behavior session data. - - Child classes should be instantiated with a fetch API that implements these - methods. - """ - @abc.abstractmethod - def get_behavior_session_id(self) -> int: - """Returns the behavior_session_id associated with this experiment, - if applicable. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_licks(self) -> pd.DataFrame: - """Get lick data from pkl file. - - Returns - ------- - pd.Dataframe - A dataframe containing lick timestamps. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_rewards(self) -> pd.DataFrame: - """Get reward data from pkl file. - - Returns - ------- - pd.DataFrame - A dataframe containing timestamps of delivered rewards. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_running_acquisition_df(self) -> pd.DataFrame: - """Get running speed acquisition data from a behavior pickle file. - - Returns - ------- - pd.DataFrame - Dataframe with an index of timestamps and the following columns: - "speed": computed running speed - "dx": angular change, computed during data collection - "v_sig": voltage signal from the encoder - "v_in": the theoretical maximum voltage that the encoder - will reach prior to "wrapping". This should - theoretically be 5V (after crossing 5V goes to 0V, or - vice versa). In practice the encoder does not always - reach this value before wrapping, which can cause - transient spikes in speed at the voltage "wraps". - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_running_speed(self) -> pd.DataFrame: - """Get running speed using timestamps from - self.get_stimulus_timestamps. - - NOTE: Do not correct for monitor delay. - - Returns - ------- - pd.DataFrame - timestamps : np.ndarray - index consisting of timestamps of running speed data samples - speed : np.ndarray - Running speed of the experimental subject (in cm / s). - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_stimulus_presentations(self) -> pd.DataFrame: - """Get stimulus presentation data. - - NOTE: Uses timestamps that do not account for monitor delay. - - Returns - ------- - pd.DataFrame - Table whose rows are stimulus presentations - (i.e. a given image, for a given duration, typically 250 ms) - and whose columns are presentation characteristics. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_stimulus_templates(self) -> StimulusTemplate: - """Get stimulus templates (movies, scenes) for behavior session. - - Returns - ------- - StimulusTemplate - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_stimulus_timestamps(self) -> np.ndarray: - """Get stimulus timestamps from pkl file. - - NOTE: Located with behavior_session_id - - Returns - ------- - np.ndarray - Timestamps associated with stimulus presentations on the monitor - that do no account for monitor delay. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_task_parameters(self) -> dict: - """Get task parameters from pkl file. - - Returns - ------- - dict - A dictionary containing parameters used to define the task runtime - behavior. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_trials(self) -> pd.DataFrame: - """Get trials from pkl file - - Returns - ------- - pd.DataFrame - A dataframe containing behavioral trial start/stop times, - and trial data - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_metadata(self) -> Union[BehaviorMetadata, dict]: - """Get metadata for Session - - Returns - ------- - dict if NWB - BehaviorMetadata otherwise - """ - raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_ophys_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_ophys_base.py deleted file mode 100644 index 328f0c5df..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/abcs/session_base/behavior_ophys_base.py +++ /dev/null @@ -1,195 +0,0 @@ -import abc -from typing import Optional, Union - -import numpy as np -import pandas as pd - -from allensdk.brain_observatory.behavior.metadata.behavior_ophys_metadata \ - import BehaviorOphysMetadata -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - session_base.behavior_base import BehaviorBase -from allensdk.brain_observatory.behavior.image_api import Image - - -class BehaviorOphysBase(BehaviorBase): - """Abstract base class implementing required methods for interacting with - behavior + ophys session data. - - Child classes should be instantiated with a fetch API that implements these - methods. - """ - - @abc.abstractmethod - def get_ophys_experiment_id(self) -> Optional[int]: - """Returns the ophys_experiment_id for the instantiated BehaviorOphys - Session (or BehaviorOphys data fetcher) if applicable.""" - raise NotImplementedError() - - @abc.abstractmethod - def get_ophys_session_id(self) -> Optional[int]: - """Returns the behavior + ophys_session_id associated with this - experiment, if applicable. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_average_projection(self) -> Image: - """Get an image whose values are the average obtained values at - each pixel of the ophys movie over time. - - Returns - ---------- - allensdk.brain_observatory.behavior.image_api.Image: - Array-like interface to avg projection image data and metadata. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_max_projection(self) -> Image: - """Get an image whose values are the maximum obtained values at - each pixel of the ophys movie over time. - - Returns - ---------- - allensdk.brain_observatory.behavior.image_api.Image: - Array-like interface to max projection image data and metadata. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_cell_specimen_table(self) -> pd.DataFrame: - """Get a cell specimen dataframe containing ROI information about - cells identified in an ophys experiment. - - Returns - ------- - pd.DataFrame - Cell ROI information organized into a dataframe. - Index is the cell ROI IDs. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_corrected_fluorescence_traces(self) -> pd.DataFrame: - """Get motion-corrected fluorescence traces. - - Returns - ------- - pd.DataFrame - Motion-corrected fluorescence traces organized into a dataframe. - Index is the cell ROI IDs. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_dff_traces(self) -> pd.DataFrame: - """Get a table of delta fluorescence over fluorescence traces. - - Returns - ------- - pd.DataFrame - The traces of dff (normalized fluorescence) organized into a - dataframe. Index is the cell ROI IDs. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_metadata(self) -> Union[BehaviorOphysMetadata, dict]: - """Get behavior+ophys session metadata. - - Returns - ------- - dict if NWB - BehaviorOphysMetadata otherwise - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_motion_correction(self) -> pd.DataFrame: - """Get motion correction trace data. - - Returns - ------- - pd.DataFrame - A dataframe containing trace data used during motion - correction computation. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_ophys_timestamps(self) -> np.ndarray: - """Get optical physiology frame timestamps. - - Returns - ------- - np.ndarray - Timestamps associated with frames captured by the microscope. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_stimulus_timestamps(self) -> np.ndarray: - """Get stimulus timestamps. - - Returns - ------- - np.ndarray - Timestamps associated with stimulus presentations on the monitor - after accounting for monitor delay. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_stimulus_presentations(self) -> pd.DataFrame: - """Get stimulus presentation data. - - NOTE: Uses monitor delay corrected stimulus timestamps. - - Returns - ------- - pd.DataFrame - Table whose rows are stimulus presentations - (i.e. a given image, for a given duration, typically 250 ms) - and whose columns are presentation characteristics. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_eye_tracking(self) -> Optional[pd.DataFrame]: - """Get eye tracking data from behavior + ophys session. - - Returns - ------- - pd.DataFrame - A refined eye tracking dataframe that contains information - about eye tracking ellipse fits, frame times, eye areas, - pupil areas, and frames with likely blinks/outliers. - """ - raise NotImplementedError() - - @abc.abstractmethod - def get_events(self) -> pd.DataFrame: - """Get event detection data - - Returns - ------- - pd.DataFrame - index: - cell_specimen_id: int - cell_roi_id: int - events: np.array - filtered_events: np.array - lambdas: float64 - noise_stds: float64 - """ - raise NotImplementedError() - - def get_eye_tracking_rig_geometry(self) -> dict: - """Get eye tracking rig metadata from behavior + ophys session. - - Returns - ------- - dict - Includes geometry of monitor, camera, LED - """ - raise NotImplementedError() diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py deleted file mode 100644 index 094280db9..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# extractor class for behavior -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_lims_api import BehaviorLimsExtractor # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_json_api import BehaviorJsonExtractor # noqa: F401, E501 - -# extractor + transform classes for behavior only -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_nwb_api import BehaviorNwbApi # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_lims_api import BehaviorLimsApi # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_json_api import BehaviorJsonApi # noqa: F401, E501 - -# extractor class for ophys -from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api import OphysLimsExtractor # noqa: F401, E501 - -# extractor class for behavior + ophys -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_lims_api import BehaviorOphysLimsExtractor # noqa: F401, E501 - -# extractor + transform classes for behavior + ophys -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_lims_api import BehaviorOphysLimsApi # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_json_api import BehaviorOphysJsonApi # noqa: F401, E501 diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_json_api.py deleted file mode 100644 index 3540675ce..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_json_api.py +++ /dev/null @@ -1,86 +0,0 @@ -import logging -from datetime import datetime - -import pytz -from allensdk.brain_observatory.behavior.session_apis.abcs. \ - data_extractor_base.behavior_data_extractor_base import \ - BehaviorDataExtractorBase -from allensdk.brain_observatory.behavior.session_apis.data_transforms import \ - BehaviorDataTransforms - - -class BehaviorJsonApi(BehaviorDataTransforms): - """A data fetching and processing class that serves processed data from - a specified raw data source (extractor). Contains all methods - needed to fill a BehaviorSession.""" - - def __init__(self, data): - extractor = BehaviorJsonExtractor(data=data) - super().__init__(extractor=extractor) - - -class BehaviorJsonExtractor(BehaviorDataExtractorBase): - """A class which 'extracts' data from a json file. The extracted data - is necessary (but not sufficient) for populating a 'BehaviorSession'. - - Most data provided by this extractor needs to be processed by - BehaviorDataTransforms methods in order to usable by 'BehaviorSession's. - - This class is used by the write_nwb module for behavior sessions. - """ - - def __init__(self, data): - self.data = data - self.logger = logging.getLogger(self.__class__.__name__) - - def get_behavior_session_id(self) -> int: - return self.data['behavior_session_id'] - - def get_foraging_id(self) -> int: - return self.data['foraging_id'] - - def get_equipment_name(self) -> str: - """Get the name of the experiment rig (ex: CAM2P.3)""" - return self.data['rig_name'] - - def get_sex(self) -> str: - """Get the sex of the subject (ex: 'M', 'F', or 'unknown')""" - return self.data['sex'] - - def get_age(self) -> str: - """Get the age code of the subject (ie P123)""" - return self.data['age'] - - def get_reporter_line(self) -> str: - """Get the (gene) reporter line for the subject associated with an - experiment""" - return self.data['reporter_line'] - - def get_driver_line(self) -> str: - """Get the (gene) driver line for the subject associated with an - experiment""" - return self.data['driver_line'] - - def get_full_genotype(self) -> str: - """Get the full genotype of the subject associated with an - experiment""" - return self.data['full_genotype'] - - def get_behavior_stimulus_file(self) -> str: - """Get the filepath to the StimulusPickle file for the session""" - return self.data['behavior_stimulus_file'] - - def get_mouse_id(self) -> int: - """Get the external specimen id (LabTracks ID) for the subject - associated with a behavior experiment""" - return int(self.data['external_specimen_name']) - - def get_date_of_acquisition(self) -> datetime: - """Get the acquisition date of an experiment (in UTC) - - NOTE: LIMS writes to JSON in local time. Needs to be converted to UTC - """ - tz = pytz.timezone("America/Los_Angeles") - return tz.localize(datetime.strptime(self.data['date_of_acquisition'], - "%Y-%m-%d %H:%M:%S")).astimezone( - pytz.utc) diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py deleted file mode 100644 index 0adcd8416..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py +++ /dev/null @@ -1,342 +0,0 @@ -import logging -import uuid -from datetime import datetime -from typing import Dict, List, Optional, Union -import pytz - -from allensdk.api.warehouse_cache.cache import memoize -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_data_extractor_base import \ - BehaviorDataExtractorBase -from allensdk.brain_observatory.behavior.session_apis.data_transforms import \ - BehaviorDataTransforms -from allensdk.core.auth_config import (LIMS_DB_CREDENTIAL_MAP, - MTRAIN_DB_CREDENTIAL_MAP) -from allensdk.core.authentication import DbCredentials -from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin -from allensdk.internal.api import (OneOrMoreResultExpectedError, - OneResultExpectedError, - db_connection_creator) -from allensdk.internal.core.lims_utilities import safe_system_path - - -class BehaviorLimsApi(BehaviorDataTransforms, CachedInstanceMethodMixin): - """A data fetching and processing class that serves processed data from - a specified raw data source (extractor). Contains all methods - needed to fill a BehaviorSession.""" - - def __init__(self, - behavior_session_id: Optional[int] = None, - lims_credentials: Optional[DbCredentials] = None, - mtrain_credentials: Optional[DbCredentials] = None, - extractor: Optional[BehaviorDataExtractorBase] = None): - - if extractor is None: - if behavior_session_id is not None: - extractor = BehaviorLimsExtractor( - behavior_session_id, - lims_credentials, - mtrain_credentials) - else: - raise RuntimeError( - "BehaviorLimsApi must be provided either an instantiated " - "'extractor' or a 'behavior_session_id'!") - - super().__init__(extractor=extractor) - - -class BehaviorLimsExtractor(BehaviorDataExtractorBase): - """A data fetching class that serves as an API for fetching 'raw' - data from LIMS necessary (but not sufficient) for filling a - 'BehaviorSession'. - - Most 'raw' data provided by this API needs to be processed by - BehaviorDataTransforms methods in order to usable by 'BehaviorSession's - """ - def __init__(self, behavior_session_id: int, - lims_credentials: Optional[DbCredentials] = None, - mtrain_credentials: Optional[DbCredentials] = None): - - self.logger = logging.getLogger(self.__class__.__name__) - - self.mtrain_db = db_connection_creator( - credentials=mtrain_credentials, - fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP) - - self.lims_db = db_connection_creator( - credentials=lims_credentials, - fallback_credentials=LIMS_DB_CREDENTIAL_MAP) - - self.behavior_session_id = behavior_session_id - ids = self._get_ids() - self.ophys_experiment_ids = ids.get("ophys_experiment_ids") - self.ophys_session_id = ids.get("ophys_session_id") - self.foraging_id = ids.get("foraging_id") - self.ophys_container_id = ids.get("ophys_container_id") - - @classmethod - def from_foraging_id(cls, - foraging_id: Union[str, uuid.UUID, int], - lims_credentials: Optional[DbCredentials] = None - ) -> "BehaviorLimsApi": - """Create a BehaviorLimsAPI instance from a foraging_id instead of - a behavior_session_id. - - NOTE: 'foraging_id' in the LIMS behavior_session table should be - the same as the 'behavior_session_uuid' in mtrain which should - also be the same as the 'session_uuid' field in the .pkl - returned by 'get_behavior_stimulus_file()'. - """ - - lims_db = db_connection_creator( - credentials=lims_credentials, - fallback_credentials=LIMS_DB_CREDENTIAL_MAP) - - if isinstance(foraging_id, uuid.UUID): - foraging_id = str(foraging_id) - elif isinstance(foraging_id, int): - foraging_id = str(uuid.UUID(int=foraging_id)) - - query = f""" - SELECT id - FROM behavior_sessions - WHERE foraging_id = '{foraging_id}'; - """ - session_id = lims_db.fetchone(query, strict=True) - return cls(session_id, lims_credentials=lims_credentials) - - def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]: - """Fetch ids associated with this behavior_session_id. If there is no - id, return None. - :returns: Dictionary of ids with the following keys: - ophys_session_id: int - ophys_experiment_ids: List[int] -- only if have ophys_session_id - foraging_id: int - :rtype: dict - """ - # Get all ids from the behavior_sessions table - query = f""" - SELECT - ophys_session_id, foraging_id - FROM - behavior_sessions - WHERE - behavior_sessions.id = {self.behavior_session_id}; - """ - ids_response = self.lims_db.select(query) - if len(ids_response) > 1 or len(ids_response) < 1: - raise OneResultExpectedError( - f"Expected length one result, received: " - f"{ids_response} results from query") - ids_dict = ids_response.iloc[0].to_dict() - - # Get additional ids if also an ophys session - # (experiment_id, container_id) - if ids_dict.get("ophys_session_id"): - oed_query = f""" - SELECT id - FROM ophys_experiments - WHERE ophys_session_id = {ids_dict["ophys_session_id"]}; - """ - oed = self.lims_db.fetchall(oed_query) - if len(oed) == 0: - oed = None - - container_query = f""" - SELECT DISTINCT - visual_behavior_experiment_container_id id - FROM - ophys_experiments_visual_behavior_experiment_containers - WHERE - ophys_experiment_id IN ({",".join(set(map(str, oed)))}); - """ - try: - container_id = self.lims_db.fetchone(container_query, - strict=True) - except OneResultExpectedError: - container_id = None - - ids_dict.update({"ophys_experiment_ids": oed, - "ophys_container_id": container_id}) - else: - ids_dict.update({"ophys_experiment_ids": None, - "ophys_container_id": None}) - return ids_dict - - def get_behavior_session_id(self) -> int: - """Getter to be consistent with BehaviorOphysLimsApi.""" - return self.behavior_session_id - - def get_ophys_experiment_ids(self) -> Optional[List[int]]: - return self.ophys_experiment_ids - - def get_ophys_session_id(self) -> Optional[int]: - return self.ophys_session_id - - def get_foraging_id(self) -> int: - return self.foraging_id - - def get_ophys_container_id(self) -> Optional[int]: - return self.ophys_container_id - - def get_behavior_stimulus_file(self) -> str: - """Return the path to the StimulusPickle file for a session. - :rtype: str - """ - query = f""" - SELECT - stim.storage_directory || stim.filename AS stim_file - FROM - well_known_files stim - WHERE - stim.attachable_id = {self.behavior_session_id} - AND stim.attachable_type = 'BehaviorSession' - AND stim.well_known_file_type_id IN ( - SELECT id - FROM well_known_file_types - WHERE name = 'StimulusPickle'); - """ - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_birth_date(self) -> datetime: - """Returns the birth date of the animal. - :rtype: datetime.date - """ - query = f""" - SELECT d.date_of_birth - FROM behavior_sessions bs - JOIN donors d on d.id = bs.donor_id - WHERE bs.id = {self.behavior_session_id}; - """ - return self.lims_db.fetchone(query, strict=True).date() - - @memoize - def get_sex(self) -> str: - """Returns sex of the animal (M/F) - :rtype: str - """ - query = f""" - SELECT g.name AS sex - FROM behavior_sessions bs - JOIN donors d ON bs.donor_id = d.id - JOIN genders g ON g.id = d.gender_id - WHERE bs.id = {self.behavior_session_id}; - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_age(self) -> str: - """Return the age code of the subject (ie P123) - :rtype: str - """ - query = f""" - SELECT a.name AS age - FROM behavior_sessions bs - JOIN donors d ON d.id = bs.donor_id - JOIN ages a ON a.id = d.age_id - WHERE bs.id = {self.behavior_session_id}; - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_equipment_name(self) -> str: - """Returns the name of the experimental rig. - :rtype: str - """ - query = f""" - SELECT e.name AS device_name - FROM behavior_sessions bs - JOIN equipment e ON e.id = bs.equipment_id - WHERE bs.id = {self.behavior_session_id}; - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_reporter_line(self) -> List[str]: - """Returns the genotype name(s) of the reporter line(s). - :rtype: list - """ - query = f""" - SELECT g.name AS reporter_line - FROM behavior_sessions bs - JOIN donors d ON bs.donor_id=d.id - JOIN donors_genotypes dg ON dg.donor_id=d.id - JOIN genotypes g ON g.id=dg.genotype_id - JOIN genotype_types gt - ON gt.id=g.genotype_type_id AND gt.name = 'reporter' - WHERE bs.id={self.behavior_session_id}; - """ - result = self.lims_db.fetchall(query) - if result is None or len(result) < 1: - raise OneOrMoreResultExpectedError( - f"Expected one or more, but received: '{result}' " - f"from query:\n'{query}'") - return result - - @memoize - def get_driver_line(self) -> List[str]: - """Returns the genotype name(s) of the driver line(s). - :rtype: list - """ - query = f""" - SELECT g.name AS driver_line - FROM behavior_sessions bs - JOIN donors d ON bs.donor_id=d.id - JOIN donors_genotypes dg ON dg.donor_id=d.id - JOIN genotypes g ON g.id=dg.genotype_id - JOIN genotype_types gt - ON gt.id=g.genotype_type_id AND gt.name = 'driver' - WHERE bs.id={self.behavior_session_id}; - """ - result = self.lims_db.fetchall(query) - if result is None or len(result) < 1: - raise OneOrMoreResultExpectedError( - f"Expected one or more, but received: '{result}' " - f"from query:\n'{query}'") - return result - - @memoize - def get_mouse_id(self) -> int: - """Returns the LabTracks ID - :rtype: int - """ - # TODO: Should this even be included? - # Found sometimes there were entries with NONE which is - # why they are filtered out; also many entries in the table - # match the donor_id, which is why used DISTINCT - query = f""" - SELECT DISTINCT(sp.external_specimen_name) - FROM behavior_sessions bs - JOIN donors d ON bs.donor_id=d.id - JOIN specimens sp ON sp.donor_id=d.id - WHERE bs.id={self.behavior_session_id} - AND sp.external_specimen_name IS NOT NULL; - """ - return int(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_full_genotype(self) -> str: - """Return the name of the subject's genotype - :rtype: str - """ - query = f""" - SELECT d.full_genotype - FROM behavior_sessions bs - JOIN donors d ON d.id=bs.donor_id - WHERE bs.id= {self.behavior_session_id}; - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_date_of_acquisition(self) -> datetime: - """Get the acquisition date of a behavior_session in UTC - :rtype: datetime""" - query = """ - SELECT bs.date_of_acquisition - FROM behavior_sessions bs - WHERE bs.id = {}; - """.format(self.behavior_session_id) - - experiment_date = self.lims_db.fetchone(query, strict=True) - return pytz.utc.localize(experiment_date) diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_nwb_api.py deleted file mode 100644 index 637a4704f..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_nwb_api.py +++ /dev/null @@ -1,303 +0,0 @@ -import datetime -import uuid -from typing import Optional - -import numpy as np -import pandas as pd -import pytz - -from pynwb import NWBHDF5IO, NWBFile - -import allensdk.brain_observatory.nwb as nwb -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import ( - get_expt_description, BehaviorMetadata -) -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - session_base.behavior_base import BehaviorBase -from allensdk.brain_observatory.behavior.schemas import ( - BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema) -from allensdk.brain_observatory.behavior.stimulus_processing import \ - StimulusTemplate, StimulusTemplateFactory, is_change_event -from allensdk.brain_observatory.behavior.trials_processing import ( - TRIAL_COLUMN_DESCRIPTION_DICT -) -from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension -from allensdk.brain_observatory.nwb.nwb_api import NwbApi -from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time - -load_pynwb_extension(OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys') -load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys') - - -class BehaviorNwbApi(NwbApi, BehaviorBase): - """A data fetching class that serves as an API for fetching 'raw' - data from an NWB file that is both necessary and sufficient for filling - a 'BehaviorOphysExperiment'. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._behavior_session_id = None - - def save(self, session_object): - - session_metadata: BehaviorMetadata = \ - session_object.api.get_metadata() - - session_type = str(session_metadata.session_type) - - nwbfile = NWBFile( - session_description=session_type, - identifier=str(session_object.behavior_session_id), - session_start_time=session_metadata.date_of_acquisition, - file_create_date=pytz.utc.localize(datetime.datetime.now()), - institution="Allen Institute for Brain Science", - keywords=["visual", "behavior", "task"], - experiment_description=get_expt_description(session_type) - ) - - # Add stimulus_timestamps to NWB in-memory object: - nwb.add_stimulus_timestamps(nwbfile, - session_object.stimulus_timestamps) - - # Add running acquisition ('dx', 'v_sig', 'v_in') data to NWB - # This data should be saved to NWB but not accessible directly from - # Sessions - nwb.add_running_acquisition_to_nwbfile( - nwbfile, - session_object.api.get_running_acquisition_df()) - - # Add running data to NWB in-memory object: - nwb.add_running_speed_to_nwbfile(nwbfile, - session_object.running_speed, - name="speed", - from_dataframe=True) - nwb.add_running_speed_to_nwbfile(nwbfile, - session_object.raw_running_speed, - name="speed_unfiltered", - from_dataframe=True) - - # Add stimulus template data to NWB in-memory object: - # Use the semi-private _stimulus_templates attribute because it is - # a StimulusTemplate object. The public stimulus_templates property - # of the session_object returns a DataFrame. - session_stimulus_templates = session_object._stimulus_templates - self._add_stimulus_templates( - nwbfile=nwbfile, - stimulus_templates=session_stimulus_templates, - stimulus_presentations=session_object.stimulus_presentations) - - # search for omitted rows and add stop_time before writing to NWB file - set_omitted_stop_time( - stimulus_table=session_object.stimulus_presentations) - - # Add stimulus presentations data to NWB in-memory object: - nwb.add_stimulus_presentations(nwbfile, - session_object.stimulus_presentations) - - # Add trials data to NWB in-memory object: - nwb.add_trials(nwbfile, session_object.trials, - TRIAL_COLUMN_DESCRIPTION_DICT) - - # Add licks data to NWB in-memory object: - if len(session_object.licks) > 0: - nwb.add_licks(nwbfile, session_object.licks) - - # Add rewards data to NWB in-memory object: - if len(session_object.rewards) > 0: - nwb.add_rewards(nwbfile, session_object.rewards) - - # Add metadata to NWB in-memory object: - nwb.add_metadata(nwbfile, session_object.metadata, - behavior_only=True) - - # Add task parameters to NWB in-memory object: - nwb.add_task_parameters(nwbfile, session_object.task_parameters) - - # Write the file: - with NWBHDF5IO(self.path, 'w') as nwb_file_writer: - nwb_file_writer.write(nwbfile) - - return nwbfile - - def get_behavior_session_id(self) -> int: - if self._behavior_session_id is None: - self.get_metadata() - return self._behavior_session_id - - def get_running_acquisition_df(self) -> pd.DataFrame: - """Get running speed acquisition data. - - Returns - ------- - pd.DataFrame - Dataframe with an index of timestamps and the following columns: - "dx": angular change, computed during data collection - "v_sig": voltage signal from the encoder - "v_in": the theoretical maximum voltage that the encoder - will reach prior to "wrapping". This should - theoretically be 5V (after crossing 5V goes to 0V, or - vice versa). In practice the encoder does not always - reach this value before wrapping, which can cause - transient spikes in speed at the voltage "wraps". - """ - running_module = self.nwbfile.modules['running'] - dx_interface = running_module.get_data_interface('dx') - - timestamps = dx_interface.timestamps[:] - dx = dx_interface.data - v_in = self.nwbfile.get_acquisition('v_in').data - v_sig = self.nwbfile.get_acquisition('v_sig').data - - running_acq_df = pd.DataFrame( - { - 'dx': dx, - 'v_in': v_in, - 'v_sig': v_sig - }, - index=pd.Index(timestamps, name='timestamps')) - - return running_acq_df - - def get_running_speed(self, lowpass: bool = True) -> pd.DataFrame: - """ - Gets running speed data - - NOTE: Overrides the inherited method from: - allensdk.brain_observatory.nwb.nwb_api - - Parameters - ---------- - lowpass: bool - Whether to return running speed with or without low pass filter - applied - zscore_threshold: float - The threshold to use for removing outlier running speeds which - might be noise and not true signal - - Returns - ------- - pd.DataFrame: - Dataframe containing various signals used to compute running - speed, and the filtered or unfiltered speed. - """ - running_module = self.nwbfile.modules['running'] - interface_name = 'speed' if lowpass else 'speed_unfiltered' - running_interface = running_module.get_data_interface(interface_name) - values = running_interface.data[:] - timestamps = running_interface.timestamps[:] - - running_speed_df = pd.DataFrame( - { - 'timestamps': timestamps, - 'speed': values - }, - ) - return running_speed_df - - def get_stimulus_templates(self, **kwargs) -> Optional[StimulusTemplate]: - - # If we have a session where only gratings were presented - # there will be no stimulus_template dict in the nwbfile - if len(self.nwbfile.stimulus_template) == 0: - return None - - image_set_name = list(self.nwbfile.stimulus_template.keys())[0] - image_data = list(self.nwbfile.stimulus_template.values())[0] - - image_attributes = [{'image_name': image_name} - for image_name in image_data.control_description] - return StimulusTemplateFactory.from_processed( - image_set_name=image_set_name, image_attributes=image_attributes, - warped=image_data.data[:], unwarped=image_data.unwarped[:] - ) - - def get_stimulus_presentations(self) -> pd.DataFrame: - df = super().get_stimulus_presentations() - df['is_change'] = is_change_event(stimulus_presentations=df) - return df - - def get_stimulus_timestamps(self) -> np.ndarray: - stim_module = self.nwbfile.processing['stimulus'] - return stim_module.get_data_interface('timestamps').timestamps[:] - - def get_trials(self) -> pd.DataFrame: - trials = self.nwbfile.trials.to_dataframe() - if 'lick_events' in trials.columns: - trials.drop('lick_events', inplace=True, axis=1) - trials.index = trials.index.rename('trials_id') - return trials - - def get_licks(self) -> np.ndarray: - if 'licking' in self.nwbfile.processing: - lick_module = self.nwbfile.processing['licking'] - licks = lick_module.get_data_interface('licks') - - return pd.DataFrame({ - 'timestamps': licks.timestamps[:], - 'frame': licks.data[:] - }) - else: - return pd.DataFrame({'time': [], 'frame': []}) - - def get_rewards(self) -> np.ndarray: - if 'rewards' in self.nwbfile.processing: - rewards = self.nwbfile.processing['rewards'] - time = rewards.get_data_interface('autorewarded').timestamps[:] - autorewarded = rewards.get_data_interface('autorewarded').data[:] - volume = rewards.get_data_interface('volume').data[:] - return pd.DataFrame({ - 'volume': volume, 'timestamps': time, - 'autorewarded': autorewarded}) - else: - return pd.DataFrame({ - 'volume': [], 'timestamps': [], - 'autorewarded': []}) - - def get_metadata(self) -> dict: - - metadata_nwb_obj = self.nwbfile.lab_meta_data['metadata'] - data = OphysBehaviorMetadataSchema( - exclude=['date_of_acquisition']).dump(metadata_nwb_obj) - self._behavior_session_id = data["behavior_session_id"] - - # Add pyNWB Subject metadata to behavior session metadata - nwb_subject = self.nwbfile.subject - data['mouse_id'] = int(nwb_subject.subject_id) - data['sex'] = nwb_subject.sex - data['age_in_days'] = BehaviorMetadata.parse_age_in_days( - age=nwb_subject.age) - data['full_genotype'] = nwb_subject.genotype - data['reporter_line'] = nwb_subject.reporter_line - data['driver_line'] = sorted(list(nwb_subject.driver_line)) - data['cre_line'] = BehaviorMetadata.parse_cre_line( - full_genotype=nwb_subject.genotype) - - # Add other metadata stored in nwb file to behavior session meta - data['date_of_acquisition'] = self.nwbfile.session_start_time - data['behavior_session_uuid'] = uuid.UUID( - data['behavior_session_uuid']) - return data - - def get_task_parameters(self) -> dict: - - metadata_nwb_obj = self.nwbfile.lab_meta_data['task_parameters'] - data = BehaviorTaskParametersSchema().dump(metadata_nwb_obj) - return data - - @staticmethod - def _add_stimulus_templates(nwbfile: NWBFile, - stimulus_templates: StimulusTemplate, - stimulus_presentations: pd.DataFrame): - nwb.add_stimulus_template( - nwbfile=nwbfile, stimulus_template=stimulus_templates) - - # Add index for this template to NWB in-memory object: - nwb_template = nwbfile.stimulus_template[ - stimulus_templates.image_set_name] - stimulus_index = stimulus_presentations[ - stimulus_presentations[ - 'image_set'] == nwb_template.name] - nwb.add_stimulus_index(nwbfile, stimulus_index, nwb_template) - - return nwbfile diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py deleted file mode 100644 index 8f295ab31..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py +++ /dev/null @@ -1,143 +0,0 @@ -import logging -from typing import Optional - -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_ophys_data_extractor_base import \ - BehaviorOphysDataExtractorBase -from allensdk.brain_observatory.behavior.session_apis.data_io import \ - BehaviorJsonExtractor -from allensdk.brain_observatory.behavior.session_apis.data_transforms import \ - BehaviorOphysDataTransforms - - -class BehaviorOphysJsonApi(BehaviorOphysDataTransforms): - """A data fetching and processing class that serves processed data from - a specified raw data source (extractor). Contains all methods - needed to fill a BehaviorOphysExperiment.""" - - def __init__(self, data: dict, skip_eye_tracking: bool = False): - extractor = BehaviorOphysJsonExtractor(data=data) - super().__init__(extractor=extractor, - skip_eye_tracking=skip_eye_tracking) - - -class BehaviorOphysJsonExtractor(BehaviorJsonExtractor, - BehaviorOphysDataExtractorBase): - """A class which 'extracts' data from a json file. The extracted data - is necessary (but not sufficient) for populating a 'BehaviorOphysExperiment'. - - Most data provided by this extractor needs to be processed by - BehaviorOphysDataTransforms methods in order to usable by - 'BehaviorOphysExperiment's. - - This class is used by the write_nwb module for behavior ophys sessions. - """ - - def __init__(self, data): - super().__init__(data) - self.logger = logging.getLogger(self.__class__.__name__) - - def get_ophys_experiment_id(self) -> int: - return self.data['ophys_experiment_id'] - - def get_ophys_session_id(self): - return self.data['ophys_session_id'] - - def get_surface_2p_pixel_size_um(self) -> float: - """Get the pixel size for 2-photon movies in micrometers""" - return self.data['surface_2p_pixel_size_um'] - - def get_max_projection_file(self) -> str: - """Get the filepath of the max projection image associated with the - ophys experiment""" - return self.data['max_projection_file'] - - def get_sync_file(self) -> str: - """Get the filepath of the sync timing file associated with the - ophys experiment""" - return self.data['sync_file'] - - def get_field_of_view_shape(self) -> dict: - """Get a field of view dictionary for a given ophys experiment. - ex: {"width": int, "height": int} - """ - return {'height': self.data['movie_height'], - 'width': self.data['movie_width']} - - def get_ophys_container_id(self) -> int: - """Get the experiment container id associated with an ophys - experiment""" - return self.data['container_id'] - - def get_targeted_structure(self) -> str: - """Get the targeted structure (acronym) for an ophys experiment - (ex: "Visp")""" - return self.data['targeted_structure'] - - def get_imaging_depth(self) -> int: - """Get the imaging depth for an ophys experiment - (ex: 400, 500, etc.)""" - return self.data['targeted_depth'] - - def get_dff_file(self) -> str: - """Get the filepath of the dff trace file associated with an ophys - experiment.""" - return self.data['dff_file'] - - def get_ophys_cell_segmentation_run_id(self) -> int: - """Get the ophys cell segmentation run id associated with an - ophys experiment id""" - return self.data['ophys_cell_segmentation_run_id'] - - def get_raw_cell_specimen_table_dict(self) -> dict: - """Get the cell_rois table from LIMS in dictionary form""" - return self.data['cell_specimen_table_dict'] - - def get_demix_file(self) -> str: - """Get the filepath of the demixed traces file associated with an - ophys experiment""" - return self.data['demix_file'] - - def get_average_intensity_projection_image_file(self) -> str: - """Get the avg intensity project image filepath associated with an - ophys experiment""" - return self.data['average_intensity_projection_image_file'] - - def get_rigid_motion_transform_file(self) -> str: - """Get the filepath for the motion transform file (.csv) associated - with an ophys experiment""" - return self.data['rigid_motion_transform_file'] - - def get_imaging_plane_group(self) -> Optional[int]: - """Get the imaging plane group number. This is a numeric index - that indicates the order that the frames were acquired when - there is more than one frame acquired concurrently. Relevant for - mesoscope data timestamps, as the laser jumps between plane - groups during the scan. Will be None for non-mesoscope data. - """ - return self.data["imaging_plane_group"] - - def get_plane_group_count(self) -> int: - """Gets the total number of plane groups in the session. - This is required for resampling ophys timestamps for mesoscope - data. Will be 0 if the scope did not capture multiple concurrent - frames (e.g. data from Scientifica microscope). - """ - return self.data["plane_group_count"] - - def get_eye_tracking_rig_geometry(self) -> dict: - """Get the eye tracking rig geometry associated with an ophys - experiment""" - return self.data['eye_tracking_rig_geometry'] - - def get_eye_tracking_filepath(self) -> dict: - """Get the eye tracking filepath containing ellipse fits""" - return self.data['eye_tracking_filepath'] - - def get_event_detection_filepath(self) -> str: - """Get the filepath of the .h5 events file associated with an ophys - experiment""" - return self.data['events_file'] - - def get_project_code(self) -> str: - raise NotImplementedError('Not exposed externally') diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py deleted file mode 100644 index c26f4525b..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py +++ /dev/null @@ -1,314 +0,0 @@ -import logging -from typing import List, Optional - -import pandas as pd -from allensdk.api.warehouse_cache.cache import memoize -from allensdk.brain_observatory.behavior.session_apis.abcs. \ - data_extractor_base.behavior_ophys_data_extractor_base import \ - BehaviorOphysDataExtractorBase -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsExtractor, OphysLimsExtractor) -from allensdk.brain_observatory.behavior.session_apis.data_transforms import \ - BehaviorOphysDataTransforms -from allensdk.core.auth_config import (LIMS_DB_CREDENTIAL_MAP, - MTRAIN_DB_CREDENTIAL_MAP) -from allensdk.core.authentication import DbCredentials -from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin -from allensdk.internal.api import db_connection_creator -from allensdk.internal.core.lims_utilities import safe_system_path - - -class BehaviorOphysLimsApi(BehaviorOphysDataTransforms, - CachedInstanceMethodMixin): - """A data fetching and processing class that serves processed data from - a specified data source (extractor). Contains all methods - needed to populate a BehaviorOphysExperiment.""" - - def __init__(self, - ophys_experiment_id: Optional[int] = None, - lims_credentials: Optional[DbCredentials] = None, - mtrain_credentials: Optional[DbCredentials] = None, - extractor: Optional[BehaviorOphysDataExtractorBase] = None, - skip_eye_tracking: bool = False): - - if extractor is None: - if ophys_experiment_id is not None: - extractor = BehaviorOphysLimsExtractor( - ophys_experiment_id, - lims_credentials, - mtrain_credentials) - else: - raise RuntimeError( - "BehaviorOphysLimsApi must be provided either an " - "instantiated 'extractor' or an 'ophys_experiment_id'!") - - super().__init__(extractor=extractor, - skip_eye_tracking=skip_eye_tracking) - - -class BehaviorOphysLimsExtractor(OphysLimsExtractor, BehaviorLimsExtractor, - BehaviorOphysDataExtractorBase): - """A data fetching class that serves as an API for fetching 'raw' - data from LIMS necessary (but not sufficient) for filling - a 'BehaviorOphysExperiment'. - - Most 'raw' data provided by this API needs to be processed by - BehaviorOphysDataTransforms methods in order to usable by - 'BehaviorOphysExperiment's. - """ - - def __init__(self, ophys_experiment_id: int, - lims_credentials: Optional[DbCredentials] = None, - mtrain_credentials: Optional[DbCredentials] = None): - - self.logger = logging.getLogger(self.__class__.__name__) - - self.lims_db = db_connection_creator( - credentials=lims_credentials, - fallback_credentials=LIMS_DB_CREDENTIAL_MAP) - - self.mtrain_db = db_connection_creator( - credentials=mtrain_credentials, - fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP) - - self.ophys_experiment_id = ophys_experiment_id - self.behavior_session_id = self.get_behavior_session_id() - - def get_ophys_experiment_id(self) -> int: - return self.ophys_experiment_id - - @memoize - def get_ophys_session_id(self) -> int: - """Get the ophys session id associated with the ophys experiment - id used to initialize the API""" - query = """ - SELECT os.id FROM ophys_sessions os - JOIN ophys_experiments oe ON oe.ophys_session_id = os.id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_project_code(self) -> str: - """Get the project code""" - query = f""" - SELECT projects.code AS project_code - FROM ophys_sessions - JOIN projects ON projects.id = ophys_sessions.project_id - WHERE ophys_sessions.id = {self.get_ophys_session_id()} - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_ophys_container_id(self) -> int: - """Get the experiment container id associated with the ophys - experiment id used to initialize the API""" - query = """ - SELECT visual_behavior_experiment_container_id - FROM ophys_experiments_visual_behavior_experiment_containers - WHERE ophys_experiment_id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=False) - - @memoize - def get_behavior_stimulus_file(self) -> str: - """Get the filepath to the StimulusPickle file for the session - associated with the ophys experiment id used to initialize the API""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS stim_file - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN behavior_sessions bs ON bs.ophys_session_id = os.id - LEFT JOIN well_known_files wkf ON wkf.attachable_id = bs.id - JOIN well_known_file_types wkft - ON wkf.well_known_file_type_id = wkft.id - WHERE wkf.attachable_type = 'BehaviorSession' - AND wkft.name = 'StimulusPickle' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_nwb_filepath(self) -> str: - """Get the filepath of the nwb file associated with the ophys - experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS nwb_file - FROM ophys_experiments oe - JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft - ON wkf.well_known_file_type_id = wkft.id - WHERE wkft.name ='BehaviorOphysNwb' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_eye_tracking_filepath(self) -> str: - """Get the filepath of the eye tracking file (*.h5) associated with the - ophys experiment""" - query = f""" - SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file - FROM ophys_experiments oe - LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.ophys_session_id - JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id - WHERE wkf.attachable_type = 'OphysSession' - AND wkft.name = 'EyeTracking Ellipses' - AND oe.id = {self.get_ophys_experiment_id()}; - """ # noqa E501 - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_eye_tracking_rig_geometry(self) -> Optional[dict]: - """Get the eye tracking rig geometry metadata""" - ophys_experiment_id = self.get_ophys_experiment_id() - - query = f''' - SELECT oec.*, oect.name as config_type, equipment.name as equipment_name - FROM ophys_sessions os - JOIN observatory_experiment_configs oec ON oec.equipment_id = os.equipment_id - JOIN observatory_experiment_config_types oect ON oect.id = oec.observatory_experiment_config_type_id - JOIN ophys_experiments oe ON oe.ophys_session_id = os.id - JOIN equipment ON equipment.id = oec.equipment_id - WHERE oe.id = {ophys_experiment_id} AND - oec.active_date <= os.date_of_acquisition AND - oect.name IN ('eye camera position', 'led position', 'screen position') - ''' # noqa E501 - # Get the raw data - rig_geometry = pd.read_sql(query, self.lims_db.get_connection()) - - if rig_geometry.empty: - # There is no rig geometry for this experiment - return None - - return self._process_eye_tracking_rig_geometry( - rig_geometry=rig_geometry - ) - - def get_ophys_experiment_df(self) -> pd.DataFrame: - """Get a DataFrame of metadata for ophys experiments""" - query = """ - SELECT - - oec.visual_behavior_experiment_container_id as container_id, - oec.ophys_experiment_id, - oe.workflow_state, - d.full_genotype as full_genotype, - id.depth as imaging_depth, - st.acronym as targeted_structure, - os.name as session_name, - equipment.name as equipment_name - - FROM ophys_experiments_visual_behavior_experiment_containers oec - LEFT JOIN ophys_experiments oe ON oe.id = oec.ophys_experiment_id - LEFT JOIN ophys_sessions os ON oe.ophys_session_id = os.id - LEFT JOIN specimens sp ON sp.id = os.specimen_id - LEFT JOIN donors d ON d.id = sp.donor_id - LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id - LEFT JOIN structures st ON st.id = oe.targeted_structure_id - LEFT JOIN equipment ON equipment.id = os.equipment_id; - """ - - return pd.read_sql(query, self.lims_db.get_connection()) - - def get_containers_df(self, only_passed=True) -> pd.DataFrame: - """Get a DataFrame of experiment containers""" - if only_passed is True: - query = """ - SELECT * - FROM visual_behavior_experiment_containers vbc - WHERE workflow_state IN ('container_qc','publish'); - """ - else: - query = """ - SELECT * - FROM visual_behavior_experiment_containers vbc; - """ - - return pd.read_sql(query, self.lims_db.get_connection()).rename( - columns={'id': 'container_id'})[['container_id', - 'specimen_id', - 'workflow_state']] - - @staticmethod - def _process_eye_tracking_rig_geometry(rig_geometry: pd.DataFrame) -> dict: - """ - Processes the raw eye tracking rig geometry returned by LIMS - """ - # Map the config types to new names - rig_geometry_config_type_map = { - 'eye camera position': 'camera', - 'screen position': 'monitor', - 'led position': 'led' - } - rig_geometry['config_type'] = rig_geometry['config_type'] \ - .map(rig_geometry_config_type_map) - - # Select the most recent config - # that precedes the date_of_acquisition for this experiment - rig_geometry = rig_geometry.sort_values('active_date', ascending=False) - rig_geometry = rig_geometry.groupby('config_type') \ - .apply(lambda x: x.iloc[0]) - - # Construct dictionary for positions - position = rig_geometry[['center_x_mm', 'center_y_mm', 'center_z_mm']] - position.index = [ - f'{v}_position_mm' if v != 'led' - else f'{v}_position' for v in position.index] - position = position.to_dict(orient='index') - position = { - config_type: [ - values['center_x_mm'], - values['center_y_mm'], - values['center_z_mm'] - ] - for config_type, values in position.items() - } - - # Construct dictionary for rotations - rotation = rig_geometry[['rotation_x_deg', 'rotation_y_deg', - 'rotation_z_deg']] - rotation = rotation[rotation.index != 'led'] - rotation.index = [f'{v}_rotation_deg' for v in rotation.index] - rotation = rotation.to_dict(orient='index') - rotation = { - config_type: [ - values['rotation_x_deg'], - values['rotation_y_deg'], - values['rotation_z_deg'] - ] for config_type, values in rotation.items() - } - - # Combine the dictionaries - return { - **position, - **rotation, - 'equipment': rig_geometry['equipment_name'].iloc[0] - } - - @classmethod - def get_api_list_by_container_id(cls, container_id - ) -> List["BehaviorOphysLimsApi"]: - """Return a list of BehaviorOphysLimsApi instances for all - ophys experiments""" - df = cls.get_ophys_experiment_df() - container_selector = df['container_id'] == container_id - oeid_list = df[container_selector]['ophys_experiment_id'].values - return [cls(oeid) for oeid in oeid_list] - - @memoize - def get_event_detection_filepath(self) -> str: - """Gets the filepath to the event detection data""" - query = f''' - SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath - FROM ophys_experiments oe - LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id - WHERE wkft.name = 'OphysEventTraceFile' - AND oe.id = {self.get_ophys_experiment_id()}; - ''' # noqa E501 - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - -if __name__ == "__main__": - print(BehaviorOphysLimsApi.get_ophys_experiment_df()) diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py deleted file mode 100644 index 84789ffb1..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py +++ /dev/null @@ -1,637 +0,0 @@ -import datetime -import warnings -from typing import Optional - -import numpy as np -import pandas as pd -import pynwb -import pytz -import SimpleITK as sitk -from hdmf.backends.hdf5 import H5DataIO - -from pynwb import NWBHDF5IO, NWBFile - -from allensdk.brain_observatory.behavior.metadata.behavior_ophys_metadata \ - import BehaviorOphysMetadata -from allensdk.brain_observatory.behavior.event_detection import \ - filter_events_array -import allensdk.brain_observatory.nwb as nwb -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import ( - get_expt_description -) -from allensdk.brain_observatory.behavior.session_apis.abcs.session_base. \ - behavior_ophys_base import BehaviorOphysBase -from allensdk.brain_observatory.behavior.schemas import ( - BehaviorTaskParametersSchema, OphysEyeTrackingRigMetadataSchema) -from allensdk.brain_observatory.behavior.trials_processing import ( - TRIAL_COLUMN_DESCRIPTION_DICT -) -from allensdk.brain_observatory.nwb import TimeSeries -from allensdk.brain_observatory.nwb.eye_tracking.ndx_ellipse_eye_tracking import ( # noqa: E501 - EllipseEyeTracking, EllipseSeries) -from allensdk.brain_observatory.behavior.write_nwb.extensions \ - .event_detection.ndx_ophys_events import OphysEventDetection -from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorNwbApi -) -from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time -from allensdk.brain_observatory.behavior.eye_tracking_processing import ( - determine_outliers, determine_likely_blinks -) - -load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys') - - -class BehaviorOphysNwbApi(BehaviorNwbApi, BehaviorOphysBase): - """A data fetching class that serves as an API for fetching 'raw' - data from an NWB file that is both necessary and sufficient for filling - a 'BehaviorOphysExperiment'. - """ - - def __init__(self, *args, **kwargs): - self.filter_invalid_rois = kwargs.pop("filter_invalid_rois", False) - super().__init__(*args, **kwargs) - - def save(self, session_object): - # Cannot type session_object due to a circular dependency - # TODO fix circular dependency and add type - - session_metadata: BehaviorOphysMetadata = \ - session_object.api.get_metadata() - - session_type = session_metadata.session_type - - nwbfile = NWBFile( - session_description=session_type, - identifier=str(session_object.ophys_experiment_id), - session_start_time=session_metadata.date_of_acquisition, - file_create_date=pytz.utc.localize(datetime.datetime.now()), - institution="Allen Institute for Brain Science", - keywords=["2-photon", "calcium imaging", "visual cortex", - "behavior", "task"], - experiment_description=get_expt_description(session_type) - ) - - # Add stimulus_timestamps to NWB in-memory object: - nwb.add_stimulus_timestamps(nwbfile, - session_object.stimulus_timestamps) - - # Add running acquisition ('dx', 'v_sig', 'v_in') data to NWB - # This data should be saved to NWB but not accessible directly from - # Sessions - nwb.add_running_acquisition_to_nwbfile( - nwbfile, - session_object.api.get_running_acquisition_df()) - - # Add running data to NWB in-memory object: - nwb.add_running_speed_to_nwbfile(nwbfile, - session_object.running_speed, - name="speed", - from_dataframe=True) - nwb.add_running_speed_to_nwbfile(nwbfile, - session_object.raw_running_speed, - name="speed_unfiltered", - from_dataframe=True) - - # Add stimulus template data to NWB in-memory object: - # Use the semi-private _stimulus_templates attribute because it is - # a StimulusTemplate object. The public stimulus_templates property - # of the session_object returns a DataFrame. - session_stimulus_templates = session_object._stimulus_templates - self._add_stimulus_templates( - nwbfile=nwbfile, - stimulus_templates=session_stimulus_templates, - stimulus_presentations=session_object.stimulus_presentations) - - # search for omitted rows and add stop_time before writing to NWB file - set_omitted_stop_time( - stimulus_table=session_object.stimulus_presentations) - - # Add stimulus presentations data to NWB in-memory object: - nwb.add_stimulus_presentations(nwbfile, - session_object.stimulus_presentations) - - # Add trials data to NWB in-memory object: - nwb.add_trials(nwbfile, session_object.trials, - TRIAL_COLUMN_DESCRIPTION_DICT) - - # Add licks data to NWB in-memory object: - if len(session_object.licks) > 0: - nwb.add_licks(nwbfile, session_object.licks) - - # Add rewards data to NWB in-memory object: - if len(session_object.rewards) > 0: - nwb.add_rewards(nwbfile, session_object.rewards) - - # Add max_projection image data to NWB in-memory object: - nwb.add_max_projection(nwbfile, session_object.max_projection) - - # Add average_image image data to NWB in-memory object: - nwb.add_average_image(nwbfile, session_object.average_projection) - - # Add segmentation_mask_image image data to NWB in-memory object: - nwb.add_segmentation_mask_image(nwbfile, - session_object.segmentation_mask_image) - - # Add metadata to NWB in-memory object: - nwb.add_metadata(nwbfile, session_object.metadata, - behavior_only=False) - - # Add task parameters to NWB in-memory object: - nwb.add_task_parameters(nwbfile, session_object.task_parameters) - - # Add roi metrics to NWB in-memory object: - nwb.add_cell_specimen_table(nwbfile, - session_object.cell_specimen_table, - session_object.metadata) - - # Add dff to NWB in-memory object: - nwb.add_dff_traces(nwbfile, session_object.dff_traces, - session_object.ophys_timestamps) - - # Add corrected_fluorescence to NWB in-memory object: - nwb.add_corrected_fluorescence_traces( - nwbfile, - session_object.corrected_fluorescence_traces) - - # Add motion correction to NWB in-memory object: - nwb.add_motion_correction(nwbfile, session_object.motion_correction) - - # Add eye tracking and rig geometry to NWB in-memory object - # if eye_tracking data exists. - if session_object.eye_tracking is not None: - self.add_eye_tracking_data_to_nwb( - nwbfile, - session_object.eye_tracking, - session_object.eye_tracking_rig_geometry) - - # Add events - self.add_events(nwbfile=nwbfile, events=session_object.events) - - # Write the file: - with NWBHDF5IO(self.path, 'w') as nwb_file_writer: - nwb_file_writer.write(nwbfile) - - return nwbfile - - def get_behavior_session_id(self) -> int: - return self.get_metadata()['behavior_session_id'] - - def get_ophys_session_id(self) -> int: - return self.get_metadata()['ophys_session_id'] - - def get_ophys_experiment_id(self) -> int: - return int(self.nwbfile.identifier) - - def get_eye_tracking(self, - z_threshold: float = 3.0, - dilation_frames: int = 2) -> Optional[pd.DataFrame]: - """ - Gets corneal, eye, and pupil ellipse fit data - - Parameters - ---------- - z_threshold : float, optional - The z-threshold when determining which frames likely contain - outliers for eye or pupil areas. Influences which frames - are considered 'likely blinks'. By default 3.0 - dilation_frames : int, optional - Determines the number of additional adjacent frames to mark as - 'likely_blink', by default 2. - - Returns - ------- - pd.DataFrame - *_area - *_center_x - *_center_y - *_height - *_phi - *_width - likely_blink - where "*" can be "corneal", "pupil" or "eye" - or None if no eye tracking data - Note: `pupil_area` is set to NaN where `likely_blink` == True - use `pupil_area_raw` column to access unfiltered pupil data - """ - try: - eye_tracking_acquisition = self.nwbfile.acquisition['EyeTracking'] - except KeyError as e: - warnings.warn("This ophys session " - f"'{int(self.nwbfile.identifier)}' has no eye " - f"tracking data. (NWB error: {e})") - return None - - eye_tracking = eye_tracking_acquisition.eye_tracking - pupil_tracking = eye_tracking_acquisition.pupil_tracking - corneal_reflection_tracking = \ - eye_tracking_acquisition.corneal_reflection_tracking - - eye_tracking_dict = { - "timestamps": eye_tracking.timestamps[:], - "cr_area": corneal_reflection_tracking.area_raw[:], - "eye_area": eye_tracking.area_raw[:], - "pupil_area": pupil_tracking.area_raw[:], - "likely_blink": eye_tracking_acquisition.likely_blink.data[:], - - "eye_center_x": eye_tracking.data[:, 0], - "eye_center_y": eye_tracking.data[:, 1], - "eye_area_raw": eye_tracking.area_raw[:], - "eye_height": eye_tracking.height[:], - "eye_width": eye_tracking.width[:], - "eye_phi": eye_tracking.angle[:], - - "pupil_center_x": pupil_tracking.data[:, 0], - "pupil_center_y": pupil_tracking.data[:, 1], - "pupil_area_raw": pupil_tracking.area_raw[:], - "pupil_height": pupil_tracking.height[:], - "pupil_width": pupil_tracking.width[:], - "pupil_phi": pupil_tracking.angle[:], - - "cr_center_x": corneal_reflection_tracking.data[:, 0], - "cr_center_y": corneal_reflection_tracking.data[:, 1], - "cr_area_raw": corneal_reflection_tracking.area_raw[:], - "cr_height": corneal_reflection_tracking.height[:], - "cr_width": corneal_reflection_tracking.width[:], - "cr_phi": corneal_reflection_tracking.angle[:], - } - - eye_tracking_data = pd.DataFrame(eye_tracking_dict) - eye_tracking_data.index = eye_tracking_data.index.rename('frame') - - # re-calculate likely blinks for new z_threshold and dilate_frames - area_df = eye_tracking_data[['eye_area_raw', 'pupil_area_raw']] - outliers = determine_outliers(area_df, z_threshold=z_threshold) - likely_blinks = determine_likely_blinks( - eye_tracking_data['eye_area_raw'], - eye_tracking_data['pupil_area_raw'], - outliers, - dilation_frames=dilation_frames) - - eye_tracking_data["likely_blink"] = likely_blinks - eye_tracking_data.at[likely_blinks, "eye_area"] = np.nan - eye_tracking_data.at[likely_blinks, "pupil_area"] = np.nan - eye_tracking_data.at[likely_blinks, "cr_area"] = np.nan - - return eye_tracking_data - - def get_eye_tracking_rig_geometry(self) -> Optional[dict]: - try: - et_mod = \ - self.nwbfile.get_processing_module("eye_tracking_rig_metadata") - except KeyError as e: - warnings.warn("This ophys session " - f"'{int(self.nwbfile.identifier)}' has no eye " - f"tracking rig metadata. (NWB error: {e})") - return None - - meta = et_mod.get_data_interface("eye_tracking_rig_metadata") - - monitor_position = meta.monitor_position[:] - monitor_position = (monitor_position.tolist() - if isinstance(monitor_position, np.ndarray) - else monitor_position) - - monitor_rotation = meta.monitor_rotation[:] - monitor_rotation = (monitor_rotation.tolist() - if isinstance(monitor_rotation, np.ndarray) - else monitor_rotation) - - camera_position = meta.camera_position[:] - camera_position = (camera_position.tolist() - if isinstance(camera_position, np.ndarray) - else camera_position) - - camera_rotation = meta.camera_rotation[:] - camera_rotation = (camera_rotation.tolist() - if isinstance(camera_rotation, np.ndarray) - else camera_rotation) - - led_position = meta.led_position[:] - led_position = (led_position.tolist() - if isinstance(led_position, np.ndarray) - else led_position) - - rig_geometry = { - f"monitor_position_{meta.monitor_position__unit_of_measurement}": - monitor_position, - f"camera_position_{meta.camera_position__unit_of_measurement}": - camera_position, - "led_position": led_position, - f"monitor_rotation_{meta.monitor_rotation__unit_of_measurement}": - monitor_rotation, - f"camera_rotation_{meta.camera_rotation__unit_of_measurement}": - camera_rotation, - "equipment": meta.equipment - } - - return rig_geometry - - def get_ophys_timestamps(self) -> np.ndarray: - return self.nwbfile.processing[ - 'ophys'].get_data_interface('dff').roi_response_series[ - 'traces'].timestamps[:] - - def get_max_projection(self, image_api=None) -> sitk.Image: - return self.get_image('max_projection', 'ophys', image_api=image_api) - - def get_average_projection(self, image_api=None) -> sitk.Image: - return self.get_image('average_image', 'ophys', image_api=image_api) - - def get_segmentation_mask_image(self, image_api=None) -> sitk.Image: - return self.get_image('segmentation_mask_image', - 'ophys', image_api=image_api) - - def get_metadata(self) -> dict: - data = super().get_metadata() - - # Add pyNWB OpticalChannel and ImagingPlane metadata to behavior ophys - # session metadata - try: - ophys_module = self.nwbfile.processing['ophys'] - except KeyError: - warnings.warn("Could not locate 'ophys' module in " - "NWB file. The following metadata fields will be " - "missing: 'ophys_frame_rate', 'indicator', " - "'targeted_structure', 'excitation_lambda', " - "'emission_lambda'") - else: - image_seg = ophys_module.data_interfaces['image_segmentation'] - imaging_plane = image_seg.plane_segmentations[ - 'cell_specimen_table'].imaging_plane - optical_channel = imaging_plane.optical_channel[0] - - data['ophys_frame_rate'] = imaging_plane.imaging_rate - data['indicator'] = imaging_plane.indicator - data['targeted_structure'] = imaging_plane.location - data['excitation_lambda'] = imaging_plane.excitation_lambda - data['emission_lambda'] = optical_channel.emission_lambda - - # Because nwb can't store imaging_plane_group as None - nwb_imaging_plane_group = data['imaging_plane_group'] - if nwb_imaging_plane_group == -1: - data["imaging_plane_group"] = None - else: - data["imaging_plane_group"] = nwb_imaging_plane_group - - return data - - def get_cell_specimen_table(self) -> pd.DataFrame: - # NOTE: ROI masks are stored in full frame width and height arrays - df = self.nwbfile.processing[ - 'ophys'].data_interfaces[ - 'image_segmentation'].plane_segmentations[ - 'cell_specimen_table'].to_dataframe() - - # Because pynwb stores this field as "image_mask", it is renamed here - df = df.rename(columns={'image_mask': 'roi_mask'}) - - df.index.rename('cell_roi_id', inplace=True) - df['cell_specimen_id'] = [None if csid == -1 else csid - for csid in df['cell_specimen_id'].values] - - df.reset_index(inplace=True) - df.set_index('cell_specimen_id', inplace=True) - - if self.filter_invalid_rois: - df = df[df["valid_roi"]] - - return df - - def get_dff_traces(self) -> pd.DataFrame: - dff_nwb = self.nwbfile.processing[ - 'ophys'].data_interfaces['dff'].roi_response_series['traces'] - # dff traces stored as timepoints x rois in NWB - # We want rois x timepoints, hence the transpose - dff_traces = dff_nwb.data[:].T - number_of_cells, number_of_dff_frames = dff_traces.shape - num_of_timestamps = len(self.get_ophys_timestamps()) - assert num_of_timestamps == number_of_dff_frames - - df = pd.DataFrame({'dff': dff_traces.tolist()}, - index=pd.Index(data=dff_nwb.rois.table.id[:], - name='cell_roi_id')) - cell_specimen_table = self.get_cell_specimen_table() - df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') - return df - - def get_corrected_fluorescence_traces(self) -> pd.DataFrame: - corr_fluorescence_nwb = self.nwbfile.processing[ - 'ophys'].data_interfaces[ - 'corrected_fluorescence'].roi_response_series['traces'] - # f traces stored as timepoints x rois in NWB - # We want rois x timepoints, hence the transpose - f_traces = corr_fluorescence_nwb.data[:].T - df = pd.DataFrame({'corrected_fluorescence': f_traces.tolist()}, - index=pd.Index( - data=corr_fluorescence_nwb.rois.table.id[:], - name='cell_roi_id')) - - cell_specimen_table = self.get_cell_specimen_table() - df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') - return df - - def get_motion_correction(self) -> pd.DataFrame: - ophys_module = self.nwbfile.processing['ophys'] - - motion_correction_data = {} - motion_correction_data['x'] = ophys_module.get_data_interface( - 'ophys_motion_correction_x').data[:] - motion_correction_data['y'] = ophys_module.get_data_interface( - 'ophys_motion_correction_y').data[:] - - return pd.DataFrame(motion_correction_data) - - def add_eye_tracking_data_to_nwb(self, nwbfile: NWBFile, - eye_tracking_df: pd.DataFrame, - eye_tracking_rig_geometry: Optional[dict] - ) -> NWBFile: - # 1. Add rig geometry - if eye_tracking_rig_geometry: - self.add_eye_tracking_rig_geometry_data_to_nwbfile( - nwbfile=nwbfile, - eye_tracking_rig_geometry=eye_tracking_rig_geometry) - - # 2. Add eye tracking - eye_tracking = EllipseSeries( - name='eye_tracking', - reference_frame='nose', - data=eye_tracking_df[['eye_center_x', 'eye_center_y']].values, - area=eye_tracking_df['eye_area'].values, - area_raw=eye_tracking_df['eye_area_raw'].values, - width=eye_tracking_df['eye_width'].values, - height=eye_tracking_df['eye_height'].values, - angle=eye_tracking_df['eye_phi'].values, - timestamps=eye_tracking_df['timestamps'].values - ) - - pupil_tracking = EllipseSeries( - name='pupil_tracking', - reference_frame='nose', - data=eye_tracking_df[['pupil_center_x', 'pupil_center_y']].values, - area=eye_tracking_df['pupil_area'].values, - area_raw=eye_tracking_df['pupil_area_raw'].values, - width=eye_tracking_df['pupil_width'].values, - height=eye_tracking_df['pupil_height'].values, - angle=eye_tracking_df['pupil_phi'].values, - timestamps=eye_tracking - ) - - corneal_reflection_tracking = EllipseSeries( - name='corneal_reflection_tracking', - reference_frame='nose', - data=eye_tracking_df[['cr_center_x', 'cr_center_y']].values, - area=eye_tracking_df['cr_area'].values, - area_raw=eye_tracking_df['cr_area_raw'].values, - width=eye_tracking_df['cr_width'].values, - height=eye_tracking_df['cr_height'].values, - angle=eye_tracking_df['cr_phi'].values, - timestamps=eye_tracking - ) - - likely_blink = TimeSeries(timestamps=eye_tracking, - data=eye_tracking_df['likely_blink'].values, - name='likely_blink', - description='blinks', - unit='N/A') - - ellipse_eye_tracking = EllipseEyeTracking( - eye_tracking=eye_tracking, - pupil_tracking=pupil_tracking, - corneal_reflection_tracking=corneal_reflection_tracking, - likely_blink=likely_blink - ) - - nwbfile.add_acquisition(ellipse_eye_tracking) - - return nwbfile - - @staticmethod - def add_eye_tracking_rig_geometry_data_to_nwbfile( - nwbfile: NWBFile, eye_tracking_rig_geometry: dict) -> NWBFile: - """ Rig geometry dict should consist of the following fields: - monitor_position_mm: [x, y, z] - monitor_rotation_deg: [x, y, z] - camera_position_mm: [x, y, z] - camera_rotation_deg: [x, y, z] - led_position: [x, y, z] - equipment: A string describing rig - """ - eye_tracking_rig_mod = pynwb.ProcessingModule( - name='eye_tracking_rig_metadata', - description='Eye tracking rig metadata module') - - ophys_eye_tracking_rig_metadata = load_pynwb_extension( - OphysEyeTrackingRigMetadataSchema, 'ndx-aibs-behavior-ophys') - - rig_metadata = ophys_eye_tracking_rig_metadata( - name="eye_tracking_rig_metadata", - equipment=eye_tracking_rig_geometry['equipment'], - monitor_position=eye_tracking_rig_geometry['monitor_position_mm'], - monitor_position__unit_of_measurement="mm", - camera_position=eye_tracking_rig_geometry['camera_position_mm'], - camera_position__unit_of_measurement="mm", - led_position=eye_tracking_rig_geometry['led_position'], - led_position__unit_of_measurement="mm", - monitor_rotation=eye_tracking_rig_geometry['monitor_rotation_deg'], - monitor_rotation__unit_of_measurement="deg", - camera_rotation=eye_tracking_rig_geometry['camera_rotation_deg'], - camera_rotation__unit_of_measurement="deg" - ) - - eye_tracking_rig_mod.add_data_interface(rig_metadata) - nwbfile.add_processing_module(eye_tracking_rig_mod) - - return nwbfile - - def get_events(self, filter_scale: float = 2, - filter_n_time_steps: int = 20) -> pd.DataFrame: - """ - Parameters - ---------- - filter_scale: float - See filter_events_array for description - filter_n_time_steps: int - See filter_events_array for description - - Returns - ------- - Events dataframe: - columns: - events: np.array - lambda: float - noise_std: float - cell_roi_id: int - - index: - cell_specimen_id: int - - """ - event_detection = self.nwbfile.processing['ophys']['event_detection'] - # NOTE: The rois with events are stored in event detection - partial_cell_specimen_table = event_detection.rois.to_dataframe() - - events = event_detection.data[:] - - # events stored time x roi. Change back to roi x time - events = events.T - - filtered_events = filter_events_array( - arr=events, scale=filter_scale, n_time_steps=filter_n_time_steps) - - # Convert to list to that it can be stored in a single column - events = [x for x in events] - filtered_events = [x for x in filtered_events] - - return pd.DataFrame({ - 'cell_roi_id': partial_cell_specimen_table.index, - 'events': events, - 'filtered_events': filtered_events, - 'lambda': event_detection.lambdas[:], - 'noise_std': event_detection.noise_stds[:] - }, index=pd.Index(partial_cell_specimen_table['cell_specimen_id'])) - - @staticmethod - def add_events(nwbfile: NWBFile, events: pd.DataFrame) -> NWBFile: - """ - Adds events to NWB file from dataframe - - Returns - ------- - NWBFile: - The NWBFile with events added - """ - events_data = np.vstack(events['events']) - - ophys_module = nwbfile.processing['ophys'] - dff_interface = ophys_module.data_interfaces['dff'] - traces = dff_interface.roi_response_series['traces'] - seg_interface = ophys_module.data_interfaces['image_segmentation'] - - cell_specimen_table = ( - seg_interface.plane_segmentations['cell_specimen_table']) - cell_specimen_df = cell_specimen_table.to_dataframe() - cell_specimen_df = cell_specimen_df.set_index('cell_specimen_id') - # We only want to store the subset of rois that have events data - rois_with_events_indices = [cell_specimen_df.index.get_loc(label) - for label in events.index] - roi_table_region = cell_specimen_table.create_roi_table_region( - description="Cells with detected events", - region=rois_with_events_indices) - - events = OphysEventDetection( - # time x rois instead of rois x time - # store using compression since sparse - data=H5DataIO(events_data.T, compression=True), - - lambdas=events['lambda'].values, - noise_stds=events['noise_std'].values, - unit='N/A', - rois=roi_table_region, - timestamps=traces.timestamps - ) - - ophys_module.add_data_interface(events) - - return nwbfile diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py deleted file mode 100644 index 7025587cf..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py +++ /dev/null @@ -1,508 +0,0 @@ -import pytz -from datetime import datetime -import pandas as pd -from typing import Optional - -from allensdk.internal.api import ( - OneOrMoreResultExpectedError, db_connection_creator) -from allensdk.api.warehouse_cache.cache import memoize -from allensdk.internal.core.lims_utilities import safe_system_path -from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin -from allensdk.core.authentication import DbCredentials -from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP - - -class OphysLimsExtractor(CachedInstanceMethodMixin): - """A data fetching class that serves as an API for fetching 'raw' - data from LIMS for filling optical physiology data. This data is - is necessary (but not sufficient) to fill the 'Ophys' portion of a - BehaviorOphysExperiment. - - This class needs to be inherited by the BehaviorOphysLimsApi and also - have methods from BehaviorOphysDataTransforms in order to be usable by a - BehaviorOphysExperiment. - """ - - def __init__(self, ophys_experiment_id: int, - lims_credentials: Optional[DbCredentials] = None): - self.ophys_experiment_id = ophys_experiment_id - - self.lims_db = db_connection_creator( - credentials=lims_credentials, - fallback_credentials=LIMS_DB_CREDENTIAL_MAP) - - def get_ophys_experiment_id(self): - return self.ophys_experiment_id - - @memoize - def get_plane_group_count(self) -> int: - """Gets the total number of plane groups in the session. - This is required for resampling ophys timestamps for mesoscope - data. Will be 0 if the scope did not capture multiple concurrent - frames. See `get_imaging_plane_group` for more info. - """ - query = f""" - -- Get the session ID for an experiment - WITH sess AS ( - SELECT os.id from ophys_experiments oe - JOIN ophys_sessions os ON os.id = oe.ophys_session_id - WHERE oe.id = {self.ophys_experiment_id} - ) - SELECT COUNT(DISTINCT(pg.group_order)) AS planes - FROM ophys_sessions os - JOIN ophys_experiments oe ON os.id = oe.ophys_session_id - JOIN ophys_imaging_plane_groups pg - ON pg.id = oe.ophys_imaging_plane_group_id - WHERE - -- only 1 session for an experiment - os.id = (SELECT id from sess limit 1); - """ - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_imaging_plane_group(self) -> Optional[int]: - """Get the imaging plane group number. This is a numeric index - that indicates the order that the frames were acquired when - there is more than one frame acquired concurrently. Relevant for - mesoscope data timestamps, as the laser jumps between plane - groups during the scan. Will be None for non-mesoscope data. - """ - query = f""" - SELECT pg.group_order - FROM ophys_experiments oe - JOIN ophys_imaging_plane_groups pg - ON pg.id = oe.ophys_imaging_plane_group_id - WHERE oe.id = {self.get_ophys_experiment_id()}; - """ - # Non-mesoscope data will not have results - group_order = self.lims_db.fetchall(query) - if len(group_order): - return group_order[0] - else: - return None - - @memoize - def get_behavior_session_id(self) -> Optional[int]: - """Returns the behavior_session_id associated with this experiment, - if applicable. - """ - query = f""" - SELECT bs.id - FROM ophys_experiments oe - -- every ophys_experiment should have an ophys_session - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - -- but not every ophys_session has a behavior_session - LEFT JOIN behavior_sessions bs ON os.id = bs.ophys_session_id - WHERE oe.id = {self.get_ophys_experiment_id()}; - """ - response = self.lims_db.fetchall(query) # Can be null - if not len(response): - return None - else: - return response[0] - - @memoize - def get_ophys_experiment_dir(self) -> str: - """Get the storage directory associated with the ophys experiment""" - query = """ - SELECT oe.storage_directory - FROM ophys_experiments oe - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_nwb_filepath(self) -> str: - """Get the filepath of the nwb file associated with the ophys - experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS nwb_file - FROM ophys_experiments oe - JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkft.name = 'NWBOphys' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_sync_file(self, ophys_experiment_id=None) -> str: - """Get the filepath of the sync timing file associated with the - ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS sync_file - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN well_known_files wkf ON wkf.attachable_id = os.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkf.attachable_type = 'OphysSession' - AND wkft.name = 'OphysRigSync' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_max_projection_file(self) -> str: - """Get the filepath of the max projection image associated with the - ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS maxint_file - FROM ophys_experiments oe - JOIN ophys_cell_segmentation_runs ocsr - ON ocsr.ophys_experiment_id = oe.id - JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE ocsr.current = 't' - AND wkf.attachable_type = 'OphysCellSegmentationRun' - AND wkft.name = 'OphysMaxIntImage' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_targeted_structure(self) -> str: - """Get the targeted structure (acronym) for an ophys experiment - (ex: "Visp")""" - query = """ - SELECT st.acronym - FROM ophys_experiments oe - LEFT JOIN structures st ON st.id = oe.targeted_structure_id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_imaging_depth(self) -> int: - """Get the imaging depth for an ophys experiment - (ex: 400, 500, etc.)""" - query = """ - SELECT id.depth - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_stimulus_name(self) -> str: - """Get the name of the stimulus presented for an ophys experiment""" - query = """ - SELECT os.stimulus_name - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - stimulus_name = self.lims_db.fetchone(query, strict=False) - stimulus_name = 'Unknown' if stimulus_name is None else stimulus_name - return stimulus_name - - @memoize - def get_date_of_acquisition(self) -> datetime: - """Get the acquisition date of an ophys experiment""" - query = """ - SELECT os.date_of_acquisition - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - - experiment_date = self.lims_db.fetchone(query, strict=True) - return pytz.utc.localize(experiment_date) - - @memoize - def get_reporter_line(self) -> str: - """Get the (gene) reporter line for the subject associated with an - ophys experiment - """ - query = """ - SELECT g.name as reporter_line - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id = os.specimen_id - JOIN donors d ON d.id = sp.donor_id - JOIN donors_genotypes dg ON dg.donor_id = d.id - JOIN genotypes g ON g.id = dg.genotype_id - JOIN genotype_types gt ON gt.id = g.genotype_type_id - WHERE gt.name = 'reporter' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - result = self.lims_db.fetchall(query) - if result is None or len(result) < 1: - raise OneOrMoreResultExpectedError( - f"Expected one or more, but received: '{result}' from query") - return result - - @memoize - def get_driver_line(self) -> str: - """Get the (gene) driver line for the subject associated with an ophys - experiment""" - query = """ - SELECT g.name as driver_line - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id = os.specimen_id - JOIN donors d ON d.id = sp.donor_id - JOIN donors_genotypes dg ON dg.donor_id = d.id - JOIN genotypes g ON g.id = dg.genotype_id - JOIN genotype_types gt ON gt.id = g.genotype_type_id - WHERE gt.name = 'driver' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - result = self.lims_db.fetchall(query) - if result is None or len(result) < 1: - raise OneOrMoreResultExpectedError( - f"Expected one or more, but received: '{result}' from query") - return result - - @memoize - def get_external_specimen_name(self) -> int: - """Get the external specimen id (LabTracks ID) for the subject - associated with an ophys experiment""" - query = """ - SELECT sp.external_specimen_name - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id = os.specimen_id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return int(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_full_genotype(self) -> str: - """Get the full genotype of the subject associated with an ophys - experiment""" - query = """ - SELECT d.full_genotype - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id = os.specimen_id - JOIN donors d ON d.id = sp.donor_id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_dff_file(self) -> str: - """Get the filepath of the dff trace file associated with an ophys - experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS dff_file - FROM ophys_experiments oe - JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkft.name = 'OphysDffTraceFile' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_objectlist_file(self) -> str: - """Get the objectlist.txt filepath associated with an ophys experiment - - NOTE: Although this will be deprecated for visual behavior it will - still be valid for visual coding. - """ - query = """ - SELECT wkf.storage_directory || wkf.filename AS obj_file - FROM ophys_experiments oe - LEFT JOIN ophys_cell_segmentation_runs ocsr - ON ocsr.ophys_experiment_id = oe.id - JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkft.name = 'OphysSegmentationObjects' - AND wkf.attachable_type = 'OphysCellSegmentationRun' - AND ocsr.current = 't' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_demix_file(self) -> str: - """Get the filepath of the demixed traces file associated with an - ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS demix_file - FROM ophys_experiments oe - JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkf.attachable_type = 'OphysExperiment' - AND wkft.name = 'DemixedTracesFile' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_average_intensity_projection_image_file(self) -> str: - """Get the avg intensity project image filepath associated with an - ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS avgint_file - FROM ophys_experiments oe - JOIN ophys_cell_segmentation_runs ocsr - ON ocsr.ophys_experiment_id = oe.id - JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE ocsr.current = 't' - AND wkf.attachable_type = 'OphysCellSegmentationRun' - AND wkft.name = 'OphysAverageIntensityProjectionImage' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_rigid_motion_transform_file(self) -> str: - """Get the filepath for the motion transform file (.csv) associated - with an ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename AS transform_file - FROM ophys_experiments oe - JOIN well_known_files wkf ON wkf.attachable_id = oe.id - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkf.attachable_type = 'OphysExperiment' - AND wkft.name = 'OphysMotionXyOffsetData' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_motion_corrected_image_stack_file(self) -> str: - """Get the filepath for the motion corrected image stack associated - with a an ophys experiment""" - query = """ - SELECT wkf.storage_directory || wkf.filename - FROM well_known_files wkf - JOIN well_known_file_types wkft - ON wkft.id = wkf.well_known_file_type_id - WHERE wkft.name = 'MotionCorrectedImageStack' - AND wkf.attachable_id = {}; - """.format(self.get_ophys_experiment_id()) - - return safe_system_path(self.lims_db.fetchone(query, strict=True)) - - @memoize - def get_foraging_id(self) -> str: - """Get the foraging id associated with an ophys experiment. This - id is obtained in str format but can be interpreted as a UUID. - (ex: 6448125b-5d18-4bda-94b6-fb4eb6613979)""" - query = """ - SELECT os.foraging_id - FROM ophys_experiments oe - LEFT JOIN ophys_sessions os ON oe.ophys_session_id = os.id - WHERE oe.id= {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_field_of_view_shape(self) -> dict: - """Get a field of view dictionary for a given ophys experiment. - ex: {"width": int, "height": int} - """ - query = """ - SELECT {} - FROM ophys_experiments oe - WHERE oe.id = {}; - """ - - fov_shape = dict() - ophys_expt_id = self.get_ophys_experiment_id() - for dim in ['width', 'height']: - select_col = f'oe.movie_{dim}' - formatted_query = query.format(select_col, ophys_expt_id) - fov_shape[dim] = self.lims_db.fetchone(formatted_query, - strict=True) - return fov_shape - - @memoize - def get_ophys_cell_segmentation_run_id(self) -> int: - """Get the ophys cell segmentation run id associated with an - ophys experiment id""" - query = """ - SELECT oseg.id - FROM ophys_experiments oe - JOIN ophys_cell_segmentation_runs oseg - ON oe.id = oseg.ophys_experiment_id - WHERE oseg.current = 't' - AND oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_raw_cell_specimen_table_dict(self) -> dict: - """Get the cell_rois table from LIMS in dictionary form""" - ophys_cell_seg_run_id = self.get_ophys_cell_segmentation_run_id() - query = """ - SELECT * - FROM cell_rois cr - WHERE cr.ophys_cell_segmentation_run_id = {}; - """.format(ophys_cell_seg_run_id) - initial_cs_table = pd.read_sql(query, self.lims_db.get_connection()) - cell_specimen_table = initial_cs_table.rename( - columns={'id': 'cell_roi_id', 'mask_matrix': 'roi_mask'}) - cell_specimen_table.drop(['ophys_experiment_id', - 'ophys_cell_segmentation_run_id'], - inplace=True, axis=1) - return cell_specimen_table.to_dict() - - @memoize - def get_surface_2p_pixel_size_um(self) -> float: - """Get the pixel size for 2-photon movies in micrometers""" - query = """ - SELECT sc.resolution - FROM ophys_experiments oe - JOIN scans sc ON sc.image_id=oe.ophys_primary_image_id - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_workflow_state(self) -> str: - """Get the workflow state of an ophys experiment (ex: 'failed')""" - query = """ - SELECT oe.workflow_state - FROM ophys_experiments oe - WHERE oe.id = {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_sex(self) -> str: - """Get the sex of the subject (ex: 'M', 'F', or 'unknown')""" - query = """ - SELECT g.name as sex - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id=os.specimen_id - JOIN donors d ON d.id=sp.donor_id - JOIN genders g ON g.id=d.gender_id - WHERE oe.id= {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - @memoize - def get_age(self) -> str: - """Get the age of the subject (ex: 'P15', 'Adult', etc...)""" - query = """ - SELECT a.name as age - FROM ophys_experiments oe - JOIN ophys_sessions os ON oe.ophys_session_id = os.id - JOIN specimens sp ON sp.id=os.specimen_id - JOIN donors d ON d.id=sp.donor_id - JOIN ages a ON a.id=d.age_id - WHERE oe.id= {}; - """.format(self.get_ophys_experiment_id()) - return self.lims_db.fetchone(query, strict=True) - - -if __name__ == "__main__": - - api = OphysLimsExtractor(789359614) - print(api.get_age()) diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py deleted file mode 100644 index c19c30a26..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from allensdk.brain_observatory.behavior.session_apis.data_transforms.behavior_data_transforms import BehaviorDataTransforms # noqa: F401, E501 -from allensdk.brain_observatory.behavior.session_apis.data_transforms.behavior_ophys_data_transforms import BehaviorOphysDataTransforms # noqa: F401, E501 diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_transforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_transforms.py deleted file mode 100644 index 6fa8c19a4..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_transforms.py +++ /dev/null @@ -1,351 +0,0 @@ -import logging -from typing import Optional - -import imageio -import numpy as np -import pandas as pd -import os -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - get_task_parameters, BehaviorMetadata -from allensdk.api.warehouse_cache.cache import memoize -from allensdk.internal.core.lims_utilities import safe_system_path -from allensdk.brain_observatory.behavior.rewards_processing import get_rewards -from allensdk.brain_observatory.behavior.running_processing import \ - get_running_df -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - session_base.behavior_base import BehaviorBase -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_data_extractor_base import \ - BehaviorDataExtractorBase -from allensdk.brain_observatory.behavior.stimulus_processing import ( - get_stimulus_metadata, get_stimulus_presentations, get_stimulus_templates, - StimulusTemplate, is_change_event) -from allensdk.brain_observatory.behavior.trials_processing import ( - get_extended_trials, get_trials_from_data_transform) -from allensdk.core.exceptions import DataFrameIndexError - - -class BehaviorDataTransforms(BehaviorBase): - """This class provides methods that transform data extracted from - LIMS or JSON data sources into final data products necessary for - populating a BehaviorSession. - """ - - def __init__(self, extractor: BehaviorDataExtractorBase): - self.extractor: BehaviorDataExtractorBase = extractor - self.logger = logging.getLogger(self.__class__.__name__) - - def get_behavior_session_id(self): - return self.extractor.get_behavior_session_id() - - @memoize - def _behavior_stimulus_file(self) -> dict: - """Helper method to cache stimulus pkl file in memory since it takes - about a second to load (and is used in many methods). - """ - return pd.read_pickle(self.extractor.get_behavior_stimulus_file()) - - @memoize - def get_licks(self) -> pd.DataFrame: - """Get lick data from pkl file. - This function assumes that the first sensor in the list of - lick_sensors is the desired lick sensor. If this changes we need - to update to get the proper line. - - Since licks can occur outside of a trial context, the lick times - are extracted from the vsyncs and the frame number in `lick_events`. - Since we don't have a timestamp for when in "experiment time" the - vsync stream starts (from self.get_stimulus_timestamps), we compute - it by fitting a linear regression (frame number x time) for the - `start_trial` and `end_trial` events in the `trial_log`, to true - up these time streams. - - :returns: pd.DataFrame - Two columns: "time", which contains the sync time - of the licks that occurred in this session and "frame", - the frame numbers of licks that occurred in this session - """ - # Get licks from pickle file instead of sync - data = self._behavior_stimulus_file() - stimulus_timestamps = self.get_stimulus_timestamps() - lick_frames = (data["items"]["behavior"]["lick_sensors"][0] - ["lick_events"]) - - # there's an occasional bug where the number of logged - # frames is one greater than the number of vsync intervals. - # If the animal licked on this last frame it will cause an - # error here. This fixes the problem. - # see: https://github.com/AllenInstitute/visual_behavior_analysis/issues/572 # noqa: E501 - # & https://github.com/AllenInstitute/visual_behavior_analysis/issues/379 # noqa:E501 - # - # This bugfix copied from - # https://github.com/AllenInstitute/visual_behavior_analysis/blob/master/visual_behavior/translator/foraging2/extract.py#L640-L647 - - if len(lick_frames) > 0: - if lick_frames[-1] == len(stimulus_timestamps): - lick_frames = lick_frames[:-1] - self.logger.error('removed last lick - ' - 'it fell outside of stimulus_timestamps ' - 'range') - - lick_times = [stimulus_timestamps[frame] for frame in lick_frames] - return pd.DataFrame({"timestamps": lick_times, "frame": lick_frames}) - - @memoize - def get_rewards(self) -> pd.DataFrame: - """Get reward data from pkl file, based on pkl file timestamps - (not sync file). - - :returns: pd.DataFrame -- A dataframe containing timestamps of - delivered rewards. - """ - data = self._behavior_stimulus_file() - timestamps = self.get_stimulus_timestamps() - return get_rewards(data, timestamps) - - def get_running_acquisition_df(self, lowpass=True, - zscore_threshold=10.0) -> pd.DataFrame: - """Get running speed acquisition data from a behavior pickle file. - - NOTE: Rebases timestamps with the self.get_stimulus_timestamps() - method which varies between the BehaviorDataTransformer and the - BehaviorOphysDataTransformer. - - Parameters - ---------- - lowpass: bool (default=True) - Whether to apply a 10Hz low-pass filter to the running speed - data. - zscore_threshold: float - The threshold to use for removing outlier running speeds which - might be noise and not true signal - - Returns - ------- - pd.DataFrame - Dataframe with an index of timestamps and the following columns: - "speed": computed running speed - "dx": angular change, computed during data collection - "v_sig": voltage signal from the encoder - "v_in": the theoretical maximum voltage that the encoder - will reach prior to "wrapping". This should - theoretically be 5V (after crossing 5V goes to 0V, or - vice versa). In practice the encoder does not always - reach this value before wrapping, which can cause - transient spikes in speed at the voltage "wraps". - """ - stimulus_timestamps = self.get_stimulus_timestamps() - data = self._behavior_stimulus_file() - return get_running_df(data, stimulus_timestamps, lowpass=lowpass, - zscore_threshold=zscore_threshold) - - def get_running_speed(self, lowpass=True) -> pd.DataFrame: - """Get running speed using timestamps from - self.get_stimulus_timestamps. - - NOTE: Do not correct for monitor delay. - - :returns: pd.DataFrame - index: timestamps - speed : subject's running speeds (in cm/s) - """ - running_data_df = self.get_running_acquisition_df(lowpass=lowpass) - if running_data_df.index.name != "timestamps": - raise DataFrameIndexError( - f"Expected index to be named 'timestamps' but got " - f"'{running_data_df.index.name}'.") - return pd.DataFrame({ - "timestamps": running_data_df.index.values, - "speed": running_data_df.speed.values}) - - def get_stimulus_frame_rate(self) -> float: - stimulus_timestamps = self.get_stimulus_timestamps() - return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0) - - def get_stimulus_presentations(self) -> pd.DataFrame: - """Get stimulus presentation data. - - NOTE: Uses timestamps that do not account for monitor delay. - - :returns: pd.DataFrame -- - Table whose rows are stimulus presentations - (i.e. a given image, for a given duration, typically 250 ms) - and whose columns are presentation characteristics. - """ - stimulus_timestamps = self.get_stimulus_timestamps() - data = self._behavior_stimulus_file() - raw_stim_pres_df = get_stimulus_presentations( - data, stimulus_timestamps) - - # Fill in nulls for image_name - # This makes two assumptions: - # 1. Nulls in `image_name` should be "gratings_" - # 2. Gratings are only present (or need to be fixed) when all - # values for `image_name` are null. - if pd.isnull(raw_stim_pres_df["image_name"]).all(): - if ~pd.isnull(raw_stim_pres_df["orientation"]).all(): - raw_stim_pres_df["image_name"] = ( - raw_stim_pres_df["orientation"] - .apply(lambda x: f"gratings_{x}")) - else: - raise ValueError("All values for 'orentation' and 'image_name'" - " are null.") - - stimulus_metadata_df = get_stimulus_metadata(data) - - idx_name = raw_stim_pres_df.index.name - stimulus_index_df = ( - raw_stim_pres_df - .reset_index() - .merge(stimulus_metadata_df.reset_index(), on=["image_name"]) - .set_index(idx_name)) - stimulus_index_df = ( - stimulus_index_df[["image_set", "image_index", "start_time", - "phase", "spatial_frequency"]] - .rename(columns={"start_time": "timestamps"}) - .sort_index() - .set_index("timestamps", drop=True)) - stim_pres_df = raw_stim_pres_df.merge( - stimulus_index_df, left_on="start_time", right_index=True, - how="left") - if len(raw_stim_pres_df) != len(stim_pres_df): - raise ValueError("Length of `stim_pres_df` should not change after" - f" merge; was {len(raw_stim_pres_df)}, now " - f" {len(stim_pres_df)}.") - - stim_pres_df['is_change'] = is_change_event( - stimulus_presentations=stim_pres_df) - - # Sort columns then drop columns which contain only all NaN values - return stim_pres_df[sorted(stim_pres_df)].dropna(axis=1, how='all') - - def get_stimulus_templates(self) -> Optional[StimulusTemplate]: - """Get stimulus templates (movies, scenes) for behavior session. - - Returns - ------- - StimulusTemplate or None if there are no images for the experiment - """ - - # TODO: Eventually the `grating_images_dict` should be provided by the - # BehaviorLimsExtractor/BehaviorJsonExtractor classes. - # - NJM 2021/2/23 - - gratings_dir = "/allen/programs/braintv/production/visualbehavior" - gratings_dir = os.path.join(gratings_dir, - "prod5/project_VisualBehavior") - grating_images_dict = { - "gratings_0.0": { - "warped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "warped_grating_0.png")))), - "unwarped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "masked_unwarped_grating_0.png")))) - }, - "gratings_90.0": { - "warped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "warped_grating_90.png")))), - "unwarped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "masked_unwarped_grating_90.png")))) - }, - "gratings_180.0": { - "warped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "warped_grating_180.png")))), - "unwarped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "masked_unwarped_grating_180.png")))) - }, - "gratings_270.0": { - "warped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "warped_grating_270.png")))), - "unwarped": np.asarray(imageio.imread( - safe_system_path(os.path.join(gratings_dir, - "masked_unwarped_grating_270.png")))) - } - } - - pkl = self._behavior_stimulus_file() - return get_stimulus_templates(pkl=pkl, - grating_images_dict=grating_images_dict) - - def get_monitor_delay(self) -> float: - """ - Return monitor delay for behavior only sessions - (in seconds) - """ - # This is the median estimate across all rigs - # as discussed in - # https://github.com/AllenInstitute/AllenSDK/issues/1318 - return 0.02115 - - def get_stimulus_timestamps(self) -> np.ndarray: - """Get stimulus timestamps (vsyncs) from pkl file. Align to the - (frame, time) points in the trial events. - - NOTE: Located with behavior_session_id. Does not use the sync_file - which requires ophys_session_id. - - Returns - ------- - np.ndarray - Timestamps associated with stimulus presentations on the monitor - that do no account for monitor delay. - """ - data = self._behavior_stimulus_file() - vsyncs = data["items"]["behavior"]["intervalsms"] - cum_sum = np.hstack((0, vsyncs)).cumsum() / 1000.0 # cumulative time - return cum_sum - - def get_task_parameters(self) -> dict: - """Get task parameters from pkl file. - - Returns - ------- - dict - A dictionary containing parameters used to define the task runtime - behavior. - """ - data = self._behavior_stimulus_file() - return get_task_parameters(data) - - @memoize - def get_trials(self) -> pd.DataFrame: - """Get trials from pkl file - - Returns - ------- - pd.DataFrame - A dataframe containing behavioral trial start/stop times, - and trial data - """ - - trial_df = get_trials_from_data_transform(self) - - return trial_df - - def get_extended_trials(self) -> pd.DataFrame: - """Get extended trials from pkl file - - Returns - ------- - pd.DataFrame - A dataframe containing extended behavior trial information. - """ - data = self._behavior_stimulus_file() - return get_extended_trials(data) - - def get_metadata(self) -> BehaviorMetadata: - """Return metadata about the session. - :rtype: BehaviorMetadata - """ - metadata = BehaviorMetadata( - extractor=self.extractor, - stimulus_timestamps=self.get_stimulus_timestamps(), - behavior_stimulus_file=self._behavior_stimulus_file() - ) - return metadata diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_transforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_transforms.py deleted file mode 100644 index 23c7885dc..000000000 --- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_transforms.py +++ /dev/null @@ -1,552 +0,0 @@ -import logging -from pathlib import Path -from typing import Iterable, Optional, Union - -import h5py -import matplotlib.image as mpimg # NOQA: E402 -import numpy as np -import xarray as xr -import pandas as pd - -import warnings - -from allensdk.api.warehouse_cache.cache import memoize -from allensdk.brain_observatory.behavior.metadata.behavior_ophys_metadata \ - import BehaviorOphysMetadata -from allensdk.brain_observatory.behavior.event_detection import \ - filter_events_array -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - data_extractor_base.behavior_ophys_data_extractor_base import \ - BehaviorOphysDataExtractorBase -from allensdk.brain_observatory.behavior.session_apis.abcs.\ - session_base.behavior_ophys_base import \ - BehaviorOphysBase - -from allensdk.brain_observatory.behavior.sync import get_sync_data -from allensdk.brain_observatory.sync_dataset import Dataset -from allensdk.brain_observatory import sync_utilities -from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner -from allensdk.brain_observatory.behavior.rewards_processing import get_rewards -from allensdk.brain_observatory.behavior.eye_tracking_processing import ( - load_eye_tracking_hdf, process_eye_tracking_data) -from allensdk.brain_observatory.behavior.image_api import ImageApi -import allensdk.brain_observatory.roi_masks as roi -from allensdk.brain_observatory.behavior.session_apis.data_transforms import ( - BehaviorDataTransforms -) - - -class BehaviorOphysDataTransforms(BehaviorDataTransforms, BehaviorOphysBase): - """This class provides methods that transform data extracted from - LIMS or JSON data sources into final data products necessary for - populating a BehaviorOphysExperiment - """ - - def __init__(self, - extractor: BehaviorOphysDataExtractorBase, - skip_eye_tracking: bool): - super().__init__(extractor=extractor) - - # Type checker not able to resolve that self.extractor is a - # BehaviorOphysDataExtractorBase. Explicitly adding as instance - # attribute fixes the issue. - self.extractor = extractor - self._skip_eye_tracking = skip_eye_tracking - self.logger = logging.getLogger(self.__class__.__name__) - - def get_ophys_session_id(self): - return self.extractor.get_ophys_session_id() - - def get_ophys_experiment_id(self): - return self.extractor.get_ophys_experiment_id() - - def get_eye_tracking_rig_geometry(self) -> Optional[dict]: - if self._skip_eye_tracking: - return None - else: - return self.extractor.get_eye_tracking_rig_geometry() - - @memoize - def get_cell_specimen_table(self): - raw_cell_specimen_table = ( - self.extractor.get_raw_cell_specimen_table_dict()) - - cell_specimen_table = pd.DataFrame.from_dict( - raw_cell_specimen_table).set_index( - 'cell_roi_id').sort_index() - fov_shape = self.extractor.get_field_of_view_shape() - fov_width = fov_shape['width'] - fov_height = fov_shape['height'] - - # Convert cropped ROI masks to uncropped versions - roi_mask_list = [] - for cell_roi_id, table_row in cell_specimen_table.iterrows(): - # Deserialize roi data into AllenSDK RoiMask object - curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height, - label=None, mask_group=-1) - curr_roi.x = table_row['x'] - curr_roi.y = table_row['y'] - curr_roi.width = table_row['width'] - curr_roi.height = table_row['height'] - curr_roi.mask = np.array(table_row['roi_mask']) - roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool)) - - cell_specimen_table['roi_mask'] = roi_mask_list - cell_specimen_table = cell_specimen_table[ - sorted(cell_specimen_table.columns)] - - cell_specimen_table.index.rename('cell_roi_id', inplace=True) - cell_specimen_table.reset_index(inplace=True) - cell_specimen_table.set_index('cell_specimen_id', inplace=True) - return cell_specimen_table - - @memoize - def get_ophys_timestamps(self): - ophys_timestamps = self.get_sync_data()['ophys_frames'] - - dff_traces = self.get_raw_dff_data() - - plane_group = self.extractor.get_imaging_plane_group() - - number_of_cells, number_of_dff_frames = dff_traces.shape - # Scientifica data has extra frames in the sync file relative - # to the number of frames in the video. These sentinel frames - # should be removed. - # NOTE: This fix does not apply to mesoscope data. - # See http://confluence.corp.alleninstitute.org/x/9DVnAg - if plane_group is None: # non-mesoscope - num_of_timestamps = len(ophys_timestamps) - if (number_of_dff_frames < num_of_timestamps): - self.logger.info( - "Truncating acquisition frames ('ophys_frames') " - f"(len={num_of_timestamps}) to the number of frames " - f"in the df/f trace ({number_of_dff_frames}).") - ophys_timestamps = ophys_timestamps[:number_of_dff_frames] - elif number_of_dff_frames > num_of_timestamps: - raise RuntimeError( - f"dff_frames (len={number_of_dff_frames}) is longer " - f"than timestamps (len={num_of_timestamps}).") - # Mesoscope data - # Resample if collecting multiple concurrent planes (e.g. mesoscope) - # because the frames are interleaved - else: - group_count = self.extractor.get_plane_group_count() - self.logger.info( - "Mesoscope data detected. Splitting timestamps " - f"(len={len(ophys_timestamps)} over {group_count} " - "plane group(s).") - ophys_timestamps = self._process_ophys_plane_timestamps( - ophys_timestamps, plane_group, group_count) - num_of_timestamps = len(ophys_timestamps) - if number_of_dff_frames != num_of_timestamps: - raise RuntimeError( - f"dff_frames (len={number_of_dff_frames}) is not equal to " - f"number of split timestamps (len={num_of_timestamps}).") - return ophys_timestamps - - @memoize - def get_sync_data(self): - sync_path = self.extractor.get_sync_file() - return get_sync_data(sync_path) - - def _load_stimulus_timestamps_and_delay(self): - """ - Load the stimulus timestamps (uncorrected for - monitor delay) and the monitor delay - """ - sync_path = self.extractor.get_sync_file() - aligner = OphysTimeAligner(sync_file=sync_path) - (self._stimulus_timestamps, - delta) = aligner.clipped_stim_timestamps - - try: - delay = aligner.monitor_delay - except ValueError as ee: - equipment_name = self.get_metadata().equipment_name - - warning_msg = 'Monitory delay calculation failed ' - warning_msg += 'with ValueError\n' - warning_msg += f' "{ee}"' - warning_msg += '\nlooking monitor delay up from table ' - warning_msg += f'for rig: {equipment_name} ' - - # see - # https://github.com/AllenInstitute/AllenSDK/issues/1318 - # https://github.com/AllenInstitute/AllenSDK/issues/1916 - delay_lookup = {'CAM2P.1': 0.020842, - 'CAM2P.2': 0.037566, - 'CAM2P.3': 0.021390, - 'CAM2P.4': 0.021102, - 'CAM2P.5': 0.021192, - 'MESO.1': 0.03613} - - if equipment_name not in delay_lookup: - msg = warning_msg - msg += f'\nequipment_name {equipment_name} not in lookup table' - raise RuntimeError(msg) - delay = delay_lookup[equipment_name] - warning_msg += f'\ndelay: {delay} seconds' - warnings.warn(warning_msg) - - self._monitor_delay = delay - - def get_stimulus_timestamps(self): - """ - Return a numpy array of stimulus timestamps uncorrected - for monitor delay (in seconds) - """ - if not hasattr(self, '_stimulus_timestamps'): - self._load_stimulus_timestamps_and_delay() - return self._stimulus_timestamps - - def get_monitor_delay(self): - """ - Return the monitor delay (in seconds) - """ - if not hasattr(self, '_monitor_delay'): - self._load_stimulus_timestamps_and_delay() - return self._monitor_delay - - @staticmethod - def _process_ophys_plane_timestamps( - ophys_timestamps: np.ndarray, plane_group: Optional[int], - group_count: int): - """ - On mesoscope rigs each frame corresponds to a different imaging plane; - the laser moves between N pairs of planes. So, every Nth 2P - frame time in the sync file corresponds to a given plane (and - its multiplexed pair). The order in which the planes are - acquired dictates which timestamps should be assigned to which - plane pairs. The planes are acquired in ascending order, where - plane_group=0 is the first group of planes. - - If the plane group is None (indicating it does not belong to - a plane group), then the plane was not collected concurrently - and the data do not need to be resampled. This is the case for - Scientifica 2p data, for example. - - Parameters - ---------- - ophys_timestamps: np.ndarray - Array of timestamps for 2p data - plane_group: int - The plane group this experiment belongs to. Signals the - order of acquisition. - group_count: int - The total number of plane groups acquired. - """ - if (group_count == 0) or (plane_group is None): - return ophys_timestamps - resampled = ophys_timestamps[plane_group::group_count] - return resampled - - @memoize - def get_metadata(self) -> BehaviorOphysMetadata: - """Return metadata about the session. - :rtype: BehaviorOphysMetadata - """ - metadata = BehaviorOphysMetadata( - extractor=self.extractor, - stimulus_timestamps=self.get_stimulus_timestamps(), - ophys_timestamps=self.get_ophys_timestamps(), - behavior_stimulus_file=self._behavior_stimulus_file() - ) - - return metadata - - @memoize - def get_cell_roi_ids(self): - cell_specimen_table = self.get_cell_specimen_table() - assert cell_specimen_table.index.name == 'cell_specimen_id' - return cell_specimen_table['cell_roi_id'].values - - def get_raw_dff_data(self): - dff_path = self.extractor.get_dff_file() - - # guarantee that DFF traces are ordered the same - # way as ROIs in the cell_specimen_table - cell_roi_id_list = self.get_cell_roi_ids() - dt = cell_roi_id_list.dtype - - with h5py.File(dff_path, 'r') as raw_file: - raw_dff_traces = np.asarray(raw_file['data']) - roi_names = np.asarray(raw_file['roi_names']).astype(dt) - - if not np.in1d(roi_names, cell_roi_id_list).all(): - raise RuntimeError("DFF traces contains ROI IDs that " - "are not in cell_specimen_table.cell_roi_id") - if not np.in1d(cell_roi_id_list, roi_names).all(): - raise RuntimeError("cell_specimen_table contains ROI IDs " - "that are not in DFF traces file") - - dff_traces = np.zeros(raw_dff_traces.shape, dtype=float) - for raw_trace, roi_id in zip(raw_dff_traces, roi_names): - idx = np.where(cell_roi_id_list == roi_id)[0][0] - dff_traces[idx, :] = raw_trace - - return dff_traces - - @memoize - def get_dff_traces(self): - dff_traces = self.get_raw_dff_data() - - cell_roi_id_list = self.get_cell_roi_ids() - - df = pd.DataFrame({'dff': [x for x in dff_traces]}, - index=pd.Index(cell_roi_id_list, - name='cell_roi_id')) - - cell_specimen_table = self.get_cell_specimen_table() - df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') - return df - - @memoize - def get_sync_licks(self): - lick_times = self.get_sync_data()['lick_times'] - return pd.DataFrame({'time': lick_times}) - - @memoize - def get_rewards(self): - data = self._behavior_stimulus_file() - timestamps = self.get_stimulus_timestamps() - return get_rewards(data, timestamps) - - @memoize - def get_corrected_fluorescence_traces(self): - demix_file = self.extractor.get_demix_file() - - cell_roi_id_list = self.get_cell_roi_ids() - dt = cell_roi_id_list.dtype - - with h5py.File(demix_file, 'r') as in_file: - corrected_fluorescence_traces = in_file['data'][()] - corrected_fluorescence_roi_id = in_file['roi_names'][()].astype(dt) - - if not np.in1d(corrected_fluorescence_roi_id, cell_roi_id_list).all(): - raise RuntimeError("corrected_fluorescence_traces contains ROI " - "IDs not present in cell_specimen_table") - if not np.in1d(cell_roi_id_list, corrected_fluorescence_roi_id).all(): - raise RuntimeError("cell_specimen_table contains ROI IDs " - "not present in corrected_fluorescence_traces") - - ophys_timestamps = self.get_ophys_timestamps() - - num_trace_timepoints = corrected_fluorescence_traces.shape[1] - assert num_trace_timepoints == ophys_timestamps.shape[0] - df = pd.DataFrame( - {'corrected_fluorescence': list(corrected_fluorescence_traces)}, - index=pd.Index(corrected_fluorescence_roi_id, - name='cell_roi_id')) - - cell_specimen_table = self.get_cell_specimen_table() - df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id') - return df - - @memoize - def get_max_projection(self, image_api=None): - - if image_api is None: - image_api = ImageApi - - maxInt_a13_file = self.extractor.get_max_projection_file() - pixel_size = self.extractor.get_surface_2p_pixel_size_um() - max_projection = mpimg.imread(maxInt_a13_file) - return ImageApi.serialize(max_projection, [pixel_size / 1000., - pixel_size / 1000.], 'mm') - - @memoize - def get_average_projection(self, image_api=None): - - if image_api is None: - image_api = ImageApi - - avgint_a1X_file = ( - self.extractor.get_average_intensity_projection_image_file()) - pixel_size = self.extractor.get_surface_2p_pixel_size_um() - average_image = mpimg.imread(avgint_a1X_file) - return ImageApi.serialize(average_image, [pixel_size / 1000., - pixel_size / 1000.], 'mm') - - @memoize - def get_motion_correction(self): - motion_corr_file = self.extractor.get_rigid_motion_transform_file() - motion_correction = pd.read_csv(motion_corr_file) - return motion_correction[['x', 'y']] - - @memoize - def get_eye_tracking(self, - z_threshold: float = 3.0, - dilation_frames: int = 2) -> Optional[pd.DataFrame]: - """Gets corneal, eye, and pupil ellipse fit data - - Parameters - ---------- - z_threshold : float, optional - The z-threshold when determining which frames likely contain - outliers for eye or pupil areas. Influences which frames - are considered 'likely blinks'. By default 3.0 - dilation_frames : int, optional - Determines the number of additional adjacent frames to mark as - 'likely_blink', by default 2. - - Returns - ------- - Optional[pd.DataFrame] - *_area - *_center_x - *_center_y - *_height - *_phi - *_width - likely_blink - where "*" can be "corneal", "pupil" or "eye" - - Will return None if class attr _skip_eye_tracking is True. - """ - if self._skip_eye_tracking: - return None - - self.logger.info(f"Getting eye_tracking_data with " - f"'z_threshold={z_threshold}', " - f"'dilation_frames={dilation_frames}'") - - filepath = Path(self.extractor.get_eye_tracking_filepath()) - sync_path = Path(self.extractor.get_sync_file()) - - eye_tracking_data = load_eye_tracking_hdf(filepath) - frame_times = sync_utilities.get_synchronized_frame_times( - session_sync_file=sync_path, - sync_line_label_keys=Dataset.EYE_TRACKING_KEYS, - trim_after_spike=False) - - eye_tracking_data = process_eye_tracking_data(eye_tracking_data, - frame_times, - z_threshold, - dilation_frames) - - return eye_tracking_data - - def get_events(self, filter_scale: float = 2, - filter_n_time_steps: int = 20) -> pd.DataFrame: - """ - Returns events in dataframe format - - Parameters - ---------- - filter_scale: float - See filter_events_array for description - filter_n_time_steps: int - See filter_events_array for description - - See behavior_ophys_experiment.events for return type - """ - events_file = self.extractor.get_event_detection_filepath() - with h5py.File(events_file, 'r') as f: - events = f['events'][:] - lambdas = f['lambdas'][:] - noise_stds = f['noise_stds'][:] - roi_ids = f['roi_names'][:] - - filtered_events = filter_events_array( - arr=events, scale=filter_scale, n_time_steps=filter_n_time_steps) - - # Convert matrix to list of 1d arrays so that it can be stored - # in a single column of the dataframe - events = [x for x in events] - filtered_events = [x for x in filtered_events] - - df = pd.DataFrame({ - 'events': events, - 'filtered_events': filtered_events, - 'lambda': lambdas, - 'noise_std': noise_stds, - 'cell_roi_id': roi_ids - }) - - # Set index as cell_specimen_id from cell_specimen_table - cell_specimen_table = self.get_cell_specimen_table() - cell_specimen_table = cell_specimen_table.reset_index() - df = cell_specimen_table[['cell_roi_id', 'cell_specimen_id']]\ - .merge(df, on='cell_roi_id') - df = df.set_index('cell_specimen_id') - - return df - - def get_roi_masks_by_cell_roi_id( - self, - cell_roi_ids: Optional[Union[int, Iterable[int]]] = None): - """ Obtains boolean masks indicating the location of one - or more ROIs in this session. - - Parameters - ---------- - cell_roi_ids : array-like of int, optional - ROI masks for these rois will be returned. - The default behavior is to return masks for all rois. - - Returns - ------- - result : xr.DataArray - dimensions are: - - roi_id : which roi is described by this mask? - - row : index within the underlying image - - column : index within the image - values are 1 where an ROI was present, otherwise 0. - - Notes - ----- - This method helps Allen Institute scientists to look at sessions - that have not yet had cell specimen ids assigned. You probably want - to use get_roi_masks instead. - """ - - cell_specimen_table = self.get_cell_specimen_table() - - if cell_roi_ids is None: - cell_roi_ids = cell_specimen_table["cell_roi_id"].unique() - elif isinstance(cell_roi_ids, int): - cell_roi_ids = np.array([int(cell_roi_ids)]) - elif np.issubdtype(type(cell_roi_ids), np.integer): - cell_roi_ids = np.array([int(cell_roi_ids)]) - else: - cell_roi_ids = np.array(cell_roi_ids) - - table = cell_specimen_table.copy() - table.set_index("cell_roi_id", inplace=True) - table = table.loc[cell_roi_ids, :] - - full_image_shape = table.iloc[0]["roi_mask"].shape - output = np.zeros((len(cell_roi_ids), - full_image_shape[0], - full_image_shape[1]), dtype=np.uint8) - - for ii, (_, row) in enumerate(table.iterrows()): - output[ii, :, :] = row["roi_mask"] - - # Pixel spacing and units of mask image will match either the - # max or avg projection image of 2P movie. - max_projection_image = ImageApi.deserialize(self.get_max_projection()) - # Spacing is in (col_spacing, row_spacing) order - # Coordinates also start spacing_dim / 2 for first element in a - # dimension. See: - # https://simpleitk.readthedocs.io/en/master/fundamentalConcepts.html - pixel_spacing = max_projection_image.spacing - unit = max_projection_image.unit - - return xr.DataArray( - data=output, - dims=("cell_roi_id", "row", "column"), - coords={ - "cell_roi_id": cell_roi_ids, - "row": (np.arange(full_image_shape[0]) - * pixel_spacing[1] - + (pixel_spacing[1] / 2)), - "column": (np.arange(full_image_shape[1]) - * pixel_spacing[0] - + (pixel_spacing[0] / 2)) - }, - attrs={ - "spacing": pixel_spacing, - "unit": unit - } - ).squeeze(drop=True) diff --git a/allensdk/brain_observatory/behavior/stimulus_processing/__init__.py b/allensdk/brain_observatory/behavior/stimulus_processing.py similarity index 95% rename from allensdk/brain_observatory/behavior/stimulus_processing/__init__.py rename to allensdk/brain_observatory/behavior/stimulus_processing.py index b6e52f5c0..a95b04865 100644 --- a/allensdk/brain_observatory/behavior/stimulus_processing/__init__.py +++ b/allensdk/brain_observatory/behavior/stimulus_processing.py @@ -5,10 +5,10 @@ import numpy as np import pandas as pd -from allensdk.brain_observatory.behavior.stimulus_processing \ +from allensdk.brain_observatory.behavior.data_objects.stimuli\ .stimulus_templates import \ StimulusTemplate, StimulusTemplateFactory -from allensdk.brain_observatory.behavior.stimulus_processing.util import \ +from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \ convert_filepath_caseinsensitive, get_image_set_name @@ -165,9 +165,9 @@ def get_gratings_metadata(stimuli: Dict, start_idx: int = 0) -> pd.DataFrame: return grating_df -def get_stimulus_templates(pkl: dict, - grating_images_dict: Optional[dict] = None - ) -> Optional[StimulusTemplate]: +def get_stimulus_templates( + pkl: dict, grating_images_dict: Optional[dict] = None, + limit_to_images: Optional[List] = None) -> Optional[StimulusTemplate]: """ Gets images presented during experiments from the behavior stimulus file (*.pkl) @@ -185,6 +185,8 @@ def get_stimulus_templates(pkl: dict, get_gratings_metadata(). Sub-nested dicts are expected to have 'warped' and 'unwarped' keys where values are numpy image arrays of aforementioned warped or unwarped grating stimuli. + limit_to_images: Optional[list] + Only return images given by these image names Returns ------- @@ -202,10 +204,19 @@ def get_stimulus_templates(pkl: dict, image_set_name = convert_filepath_caseinsensitive( image_set_name) + attrs = images['image_attributes'] + image_values = images['images'] + if limit_to_images is not None: + keep_idxs = [ + i for i in range(len(images)) if + attrs[i]['image_name'] in limit_to_images] + attrs = [attrs[i] for i in keep_idxs] + image_values = [image_values[i] for i in keep_idxs] + return StimulusTemplateFactory.from_unprocessed( image_set_name=image_set_name, - image_attributes=images['image_attributes'], - images=images['images'] + image_attributes=attrs, + images=image_values ) elif 'grating' in pkl_stimuli: if (grating_images_dict is None) or (not grating_images_dict): diff --git a/allensdk/brain_observatory/behavior/stimulus_processing/util.py b/allensdk/brain_observatory/behavior/stimulus_processing/util.py deleted file mode 100644 index fb019b09c..000000000 --- a/allensdk/brain_observatory/behavior/stimulus_processing/util.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path - - -def convert_filepath_caseinsensitive(filename_in): - return filename_in.replace('TRAINING', 'training') - - -def get_image_set_name(image_set_path: str) -> str: - """ - Strips the stem from the image_set filename - """ - return Path(image_set_path).stem \ No newline at end of file diff --git a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py index e34e9b214..9c63015f5 100644 --- a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py +++ b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py @@ -5,7 +5,8 @@ import re from allensdk import one -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_metadata import \ BehaviorMetadata from allensdk.brain_observatory.behavior.session_apis.data_io import ( BehaviorOphysNwbApi) diff --git a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py index 574372f34..f8969af2d 100644 --- a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py +++ b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py @@ -1,20 +1,19 @@ import sys import os import numpy as np -import pandas as pd from allensdk.brain_observatory.behavior.behavior_ophys_experiment import ( BehaviorOphysExperiment) from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi, BehaviorOphysLimsApi) + BehaviorOphysNwbApi) import behavior_project_cache as bpc from importlib import reload reload(bpc) -def time_from_last(flash_times, other_times): +def time_from_last(flash_times, other_times): last_other_index = np.searchsorted(a=other_times, v=flash_times) - 1 time_from_last_other = flash_times - other_times[last_other_index] @@ -23,10 +22,13 @@ def time_from_last(flash_times, other_times): return time_from_last_other + def trace_average(values, timestamps, start_time, stop_time): - values_this_range = values[((timestamps >= start_time) & (timestamps < stop_time))] + values_this_range = values[ + ((timestamps >= start_time) & (timestamps < stop_time))] return values_this_range.mean() + test_values = np.array([1, 2, 3, 4, 5, 6]) test_timestamps = np.array([1, 2, 3, 4, 5, 6]) test_start_times = np.array([0, 0, 2.5]) @@ -36,8 +38,9 @@ def trace_average(values, timestamps, start_time, stop_time): def find_change(image_index, omitted_index): ''' - Args: - image_index (pd.Series): The index of the presented image for each flash + Args: + image_index (pd.Series): The index of the presented image for each + flash omitted_index (int): The index value for omitted stimuli Returns: @@ -45,7 +48,8 @@ def find_change(image_index, omitted_index): ''' change = np.diff(image_index) != 0 - change = np.concatenate([np.array([False]), change]) # First flash not a change + change = np.concatenate( + [np.array([False]), change]) # First flash not a change omitted = image_index == omitted_index omitted_inds = np.flatnonzero(omitted) change[omitted_inds] = False @@ -55,11 +59,11 @@ def find_change(image_index, omitted_index): change[omitted_inds[:-1] + 1] = False else: change[omitted_inds + 1] = False - + return change -def get_extended_stimulus_presentations(session): +def get_extended_stimulus_presentations(session): intermediate_df = session.stimulus_presentations.copy() lick_times = session.licks["time"].values @@ -70,12 +74,12 @@ def get_extended_stimulus_presentations(session): # Time from last other for each flash - if len(lick_times) < 5: #Passive sessions + if len(lick_times) < 5: # Passive sessions time_from_last_lick = np.full(len(flash_times), np.nan) else: time_from_last_lick = time_from_last(flash_times, lick_times) - if len(reward_times) < 1: # Sometimes mice are bad + if len(reward_times) < 1: # Sometimes mice are bad time_from_last_reward = np.full(len(flash_times), np.nan) else: time_from_last_reward = time_from_last(flash_times, reward_times) @@ -101,7 +105,8 @@ def get_extended_stimulus_presentations(session): changes_including_first[0] = True change_indices = np.flatnonzero(changes_including_first) flash_inds = np.arange(len(intermediate_df)) - block_inds = np.searchsorted(a=change_indices, v=flash_inds, side="right") - 1 + block_inds = np.searchsorted(a=change_indices, v=flash_inds, + side="right") - 1 intermediate_df["block_index"] = block_inds @@ -114,18 +119,19 @@ def get_extended_stimulus_presentations(session): for image_name, image_blocks in blocks_per_image.iteritems(): if image_name != "omitted": for ind_block, block_number in enumerate(image_blocks): - # block_rep_number starts as a copy of block_inds, so we can go write over the index number with the rep number + # block_rep_number starts as a copy of block_inds, so we can + # go write over the index number with the rep number block_repetition_number[ - block_repetition_number == block_number - ] = ind_block + block_repetition_number == block_number] = ind_block intermediate_df["image_block_repetition"] = block_repetition_number # Repeat number within a block repeat_number = np.full(len(intermediate_df), np.nan) - assert ( - intermediate_df.iloc[0].name == 0 - ) # Assuming that the row index starts at zero + + # Assuming that the row index starts at zero + assert intermediate_df.iloc[0].name == 0 + for ind_group, group in intermediate_df.groupby("block_index"): repeat = 0 for ind_row, row in group.iterrows(): @@ -138,15 +144,16 @@ def get_extended_stimulus_presentations(session): # Lists of licks/rewards on each flash licks_each_flash = intermediate_df.apply( lambda row: lick_times[ - ((lick_times > row["start_time"]) & (lick_times < row["start_time"] + 0.75)) + ((lick_times > row["start_time"]) & ( + lick_times < row["start_time"] + 0.75)) ], axis=1, ) rewards_each_flash = intermediate_df.apply( lambda row: reward_times[ ( - (reward_times > row["start_time"]) - & (reward_times < row["start_time"] + 0.75) + (reward_times > row["start_time"]) + & (reward_times < row["start_time"] + 0.75) ) ], axis=1, @@ -161,7 +168,7 @@ def get_extended_stimulus_presentations(session): session.running_speed.values, session.running_speed.timestamps, row["start_time"], - row["start_time"]+0.25, + row["start_time"] + 0.25, ), axis=1, ) @@ -193,9 +200,14 @@ def get_extended_stimulus_presentations(session): case = 0 cache_json = { - "manifest_path": "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv", - "nwb_base_dir": "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files", - "analysis_files_base_dir": "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files", + "manifest_path": "/allen/programs/braintv/workgroups/nc-ophys" + "/visual_behavior/SWDB_2019/" + "visual_behavior_data_manifest.csv", + "nwb_base_dir": "/allen/programs/braintv/workgroups/nc-ophys" + "/visual_behavior/SWDB_2019/nwb_files", + "analysis_files_base_dir": + "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior" + "/SWDB_2019/extra_files", } cache = bpc.BehaviorProjectCache(cache_json) @@ -208,27 +220,31 @@ def get_extended_stimulus_presentations(session): api = BehaviorOphysNwbApi(nwb_path) session = BehaviorOphysExperiment(api) - # output_path = "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files_final" - output_path = "/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/corrected_extended_stim" + # output_path = "/allen/programs/braintv/workgroups/nc-ophys + # /visual_behavior/SWDB_2019/extra_files_final" + output_path = "/allen/programs/braintv/workgroups/nc-ophys" \ + "/visual_behavior/SWDB_2019/corrected_extended_stim" - extended_stimulus_presentations_df = get_extended_stimulus_presentations(session) + extended_stimulus_presentations_df = \ + get_extended_stimulus_presentations(session) output_fn = os.path.join( - output_path, "extended_stimulus_presentations_df_{}.h5".format(experiment_id) + output_path, + "extended_stimulus_presentations_df_{}.h5".format(experiment_id) ) - print("Writing extended_stimulus_presentations_df to {}".format(output_fn)) + print("Writing extended_stimulus_presentations_df to {}".format( + output_fn)) extended_stimulus_presentations_df.to_hdf(output_fn, key="df") elif case == 1: - + failed_oeid = 825623170 success_oeid = 826585773 # nwb_path = cache.get_nwb_filepath(success_oeid) nwb_path = cache.get_nwb_filepath(failed_oeid) - api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois = True) + api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True) session = BehaviorOphysExperiment(api) - extended_stimulus_presentations_df = get_extended_stimulus_presentations(session) - - + extended_stimulus_presentations_df = \ + get_extended_stimulus_presentations(session) diff --git a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py index 634276a69..30fe15626 100644 --- a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py +++ b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py @@ -5,30 +5,36 @@ import itertools from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ - BehaviorOphysExperiment + BehaviorOphysExperiment from allensdk.brain_observatory.behavior.session_apis.data_io import ( BehaviorOphysNwbApi) -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysLimsApi) -from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc -from allensdk.brain_observatory.behavior.swdb.analysis_tools import get_nearest_frame, get_trace_around_timepoint, get_mean_in_window +from allensdk.brain_observatory.behavior.swdb import \ + behavior_project_cache as bpc +from allensdk.brain_observatory.behavior.swdb.analysis_tools import \ + get_trace_around_timepoint, get_mean_in_window ''' - This script computes the flash_response_df for a BehaviorOphysExperiment object + This script computes the flash_response_df for a BehaviorOphysExperiment + object ''' + def get_flash_response_df(session, response_analysis_params): ''' Builds the flash response dataframe for INPUTS: - BehaviorOphysExperiment to build the flash response dataframe for - A dictionary with the following keys - 'window_around_timepoint_seconds' is the time window to save out the dff_trace around the flash onset. - 'response_window_duration_seconds' is the length of time after the flash onset to compute the mean_response - 'baseline_window_duration_seconds' is the length of time before the flash onset to compute the baseline response - + BehaviorOphysExperiment to build the flash response + dataframe for + A dictionary with the following keys + 'window_around_timepoint_seconds' is the time window to save + out the dff_trace around the flash onset. + 'response_window_duration_seconds' is the length of time + after the flash onset to compute the mean_response + 'baseline_window_duration_seconds' is the length of time + before the flash onset to compute the baseline response + OUTPUTS: A dataframe with index: (cell_specimen_id, flash_id) and columns: @@ -39,113 +45,154 @@ def get_flash_response_df(session, response_analysis_params): dff_trace_timestamps, the timestamps for the dff_trace ''' - frame_rate = 31. # Shouldn't hard code this here + frame_rate = 31. # Shouldn't hard code this here # get data to analyze dff_traces = session.dff_traces.copy() flashes = session.stimulus_presentations.copy() # get params to define response window, in seconds - window_around_timepoint_seconds = response_analysis_params['window_around_timepoint_seconds'] - response_window_duration_seconds = response_analysis_params['response_window_duration_seconds'] - baseline_window_duration_seconds = response_analysis_params['baseline_window_duration_seconds'] - mean_response_window_seconds = [np.abs(window_around_timepoint_seconds[0]), - np.abs(window_around_timepoint_seconds[0]) + response_window_duration_seconds] - baseline_window_seconds = [np.abs(window_around_timepoint_seconds[0]) - baseline_window_duration_seconds, - np.abs(window_around_timepoint_seconds[0])] - - # Build a dataframe with multiindex defined as product of cell_id X flash_id - cell_flash_combinations = itertools.product(dff_traces.index,flashes.index) - index = pd.MultiIndex.from_tuples(cell_flash_combinations, names=['cell_specimen_id', 'flash_id']) + window_around_timepoint_seconds = response_analysis_params[ + 'window_around_timepoint_seconds'] + response_window_duration_seconds = response_analysis_params[ + 'response_window_duration_seconds'] + baseline_window_duration_seconds = response_analysis_params[ + 'baseline_window_duration_seconds'] + mean_response_window_seconds = [np.abs(window_around_timepoint_seconds[0]), + np.abs(window_around_timepoint_seconds[ + 0]) + + response_window_duration_seconds] + baseline_window_seconds = [np.abs( + window_around_timepoint_seconds[0]) - baseline_window_duration_seconds, + np.abs(window_around_timepoint_seconds[0])] + + # Build a dataframe with multiindex defined as product of cell_id X + # flash_id + cell_flash_combinations = itertools.product(dff_traces.index, + flashes.index) + index = pd.MultiIndex.from_tuples(cell_flash_combinations, + names=['cell_specimen_id', 'flash_id']) df = pd.DataFrame(index=index) traces_list = [] trace_timestamps_list = [] - + # Iterate though cell/flash pairs and build table - for cell_specimen_id, flash_id in itertools.product(dff_traces.index,flashes.index): + for cell_specimen_id, flash_id in itertools.product(dff_traces.index, + flashes.index): timepoint = flashes.loc[flash_id]['start_time'] cell_roi_id = dff_traces.loc[cell_specimen_id]['cell_roi_id'] full_cell_trace = dff_traces.loc[cell_specimen_id, 'dff'] - trace, trace_timestamps = get_trace_around_timepoint(full_cell_trace, timepoint, session.ophys_timestamps, window_around_timepoint_seconds, frame_rate) - mean_response = get_mean_in_window(trace, mean_response_window_seconds, frame_rate) - baseline_response = get_mean_in_window(trace, baseline_window_seconds, frame_rate) + trace, trace_timestamps = get_trace_around_timepoint( + full_cell_trace, + timepoint, + session.ophys_timestamps, + window_around_timepoint_seconds, + frame_rate) + mean_response = get_mean_in_window(trace, mean_response_window_seconds, + frame_rate) + baseline_response = get_mean_in_window(trace, baseline_window_seconds, + frame_rate) traces_list.append(trace) trace_timestamps_list.append(trace_timestamps) df.loc[(cell_specimen_id, flash_id), 'cell_roi_id'] = int(cell_roi_id) df.loc[(cell_specimen_id, flash_id), 'mean_response'] = mean_response - df.loc[(cell_specimen_id, flash_id), 'baseline_response'] = baseline_response + df.loc[(cell_specimen_id, + flash_id), 'baseline_response'] = baseline_response df.insert(loc=1, column='dff_trace', value=traces_list) - df.insert(loc=2, column='dff_trace_timestamps', value=trace_timestamps_list) + df.insert(loc=2, column='dff_trace_timestamps', + value=trace_timestamps_list) return df -def get_p_values_from_shuffled_spontaneous(session, flash_response_df, response_window_duration=0.5,number_of_shuffles=10000): + +def get_p_values_from_shuffled_spontaneous(session, flash_response_df, + response_window_duration=0.5, + number_of_shuffles=10000): ''' - Computes the P values for each cell/flash. The P value is the probability of observing a response of that + Computes the P values for each cell/flash. The P value is the + probability of observing a response of that magnitude in the spontaneous window. The algorithm is copied from VBA INPUTS: a BehaviorOphysExperiment object the flash_response_df for this session - is the duration of the response_window that was used to compute the mean_response in the flash_response_df. This is used here to extract an equivalent duration df/f trace from the spontaneous timepoint - the number of shuffles of spontaneous activity used to compute the pvalue - + is the duration of the + response_window that was used to compute the mean_response in + the flash_response_df. This is used here to extract an + equivalent duration df/f trace from the spontaneous timepoint + the number of shuffles of spontaneous + activity used to compute the pvalue + OUTPUTS: - fdf, a copy of the flash_response_df with a new column appended 'p_value' which is the per-flash X per-cell p-value - + fdf, a copy of the flash_response_df with a new column appended + 'p_value' which is the per-flash X per-cell p-value + ASSERTS: each p value is bounded by 0 and 1, and does not include any NaNs ''' # Organize Data fdf = flash_response_df.copy() - st = session.stimulus_presentations.copy() + st = session.stimulus_presentations.copy() included_flashes = fdf.index.get_level_values(1).unique() - st = st[st.index.isin(included_flashes)] + st = st[st.index.isin(included_flashes)] # Get Sample of Spontaneous Frames spontaneous_frames = get_spontaneous_frames(session) # Compute the number of response_window frames - ophys_frame_rate = 31 # Shouldn't hard code this here - n_mean_response_window_frames = int(np.round(response_window_duration * ophys_frame_rate, 0)) - cell_ids = np.unique(fdf.index.get_level_values(0)) + ophys_frame_rate = 31 # Shouldn't hard code this here + n_mean_response_window_frames = int( + np.round(response_window_duration * ophys_frame_rate, 0)) + cell_ids = np.unique(fdf.index.get_level_values(0)) n_cells = len(cell_ids) # Get Shuffled responses from spontaneous frames # get mean response for shuffles of the spontaneous activity frames # in a window the same size as the stim response window duration - shuffled_responses = np.empty((n_cells, number_of_shuffles, n_mean_response_window_frames)) + shuffled_responses = np.empty( + (n_cells, number_of_shuffles, n_mean_response_window_frames)) idx = np.random.choice(spontaneous_frames, number_of_shuffles) - dff_traces = np.stack(session.dff_traces.to_numpy()[:,1],axis=0) + dff_traces = np.stack(session.dff_traces.to_numpy()[:, 1], axis=0) for i in range(n_mean_response_window_frames): shuffled_responses[:, :, i] = dff_traces[:, idx + i] shuffled_mean = shuffled_responses.mean(axis=2) - # compare flash responses to shuffled values and make a dataframe of p_value for cell_id X flash_id + # compare flash responses to shuffled values and make a dataframe of + # p_value for cell_id X flash_id iterables = [cell_ids, st.index.values] - flash_p_values = pd.DataFrame(index=pd.MultiIndex.from_product(iterables,names= ['cell_specimen_id','flash_id'])) + flash_p_values = pd.DataFrame(index=pd.MultiIndex.from_product( + iterables, + names=[ + 'cell_specimen_id', + 'flash_id'])) for i, cell_index in enumerate(cell_ids): responses = fdf.loc[cell_index].mean_response.values null_dist_mat = np.tile(shuffled_mean[i, :], reps=(len(responses), 1)) actual_is_less = responses.reshape(len(responses), 1) <= null_dist_mat p_values = np.mean(actual_is_less, axis=1) - for j in range(0,len(p_values)): - flash_p_values.at[(cell_index,j),'p_value'] = p_values[j] - fdf = pd.concat([fdf,flash_p_values],axis=1) + for j in range(0, len(p_values)): + flash_p_values.at[(cell_index, j), 'p_value'] = p_values[j] + fdf = pd.concat([fdf, flash_p_values], axis=1) - # Test to ensure p values are bounded between 0 and 1, and dont include NaNs + # Test to ensure p values are bounded between 0 and 1, and dont include + # NaNs assert np.all(fdf['p_value'].values <= 1) assert np.all(fdf['p_value'].values >= 0) assert np.all(~np.isnan(fdf['p_value'].values)) - - return fdf + + return fdf + def get_spontaneous_frames(session): ''' - Returns a list of the frames that occur during the before and after spontaneous windows. This is copied from VBA. Does not use the full spontaneous period because that is what VBA did. It only uses 4 minutes of the before and after spontaneous period. - + Returns a list of the frames that occur during the before and after + spontaneous windows. This is copied from VBA. Does not use the full + spontaneous period because that is what VBA did. It only uses 4 + minutes of the before and after spontaneous period. + INPUTS: - a BehaviorOphysExperiment object to get all the spontaneous frames + a BehaviorOphysExperiment object to get all the + spontaneous frames OUTPUTS: a list of the frames during the spontaneous period ''' @@ -158,22 +205,30 @@ def get_spontaneous_frames(session): behavior_start_time = st.iloc[0].start_time spontaneous_start_time_pre = behavior_start_time - spont_duration spontaneous_end_time_pre = behavior_start_time - spontaneous_start_frame_pre = get_successive_frame_list(spontaneous_start_time_pre, session.ophys_timestamps) - spontaneous_end_frame_pre = get_successive_frame_list(spontaneous_end_time_pre, session.ophys_timestamps) - spontaneous_frames_pre = np.arange(spontaneous_start_frame_pre, spontaneous_end_frame_pre, 1) + spontaneous_start_frame_pre = get_successive_frame_list( + spontaneous_start_time_pre, session.ophys_timestamps) + spontaneous_end_frame_pre = get_successive_frame_list( + spontaneous_end_time_pre, session.ophys_timestamps) + spontaneous_frames_pre = np.arange(spontaneous_start_frame_pre, + spontaneous_end_frame_pre, 1) # for spontaneous epoch at end of session behavior_end_time = st.iloc[-1].stop_time spontaneous_start_time_post = behavior_end_time + 0.5 spontaneous_end_time_post = behavior_end_time + spont_duration - spontaneous_start_frame_post = get_successive_frame_list(spontaneous_start_time_post, session.ophys_timestamps) - spontaneous_end_frame_post = get_successive_frame_list(spontaneous_end_time_post, session.ophys_timestamps) - spontaneous_frames_post = np.arange(spontaneous_start_frame_post, spontaneous_end_frame_post, 1) + spontaneous_start_frame_post = get_successive_frame_list( + spontaneous_start_time_post, session.ophys_timestamps) + spontaneous_end_frame_post = get_successive_frame_list( + spontaneous_end_time_post, session.ophys_timestamps) + spontaneous_frames_post = np.arange(spontaneous_start_frame_post, + spontaneous_end_frame_post, 1) # add them together - spontaneous_frames = list(spontaneous_frames_pre) + (list(spontaneous_frames_post)) + spontaneous_frames = list(spontaneous_frames_pre) + ( + list(spontaneous_frames_post)) return spontaneous_frames + def get_successive_frame_list(timepoints_array, timestamps): ''' Returns the next frame after timestamps in timepoints_array @@ -184,34 +239,43 @@ def get_successive_frame_list(timepoints_array, timestamps): successive_frames = np.searchsorted(timestamps, timepoints_array) return successive_frames -def add_image_name(session,fdf): + +def add_image_name(session, fdf): ''' - Adds a column to flash_response_df with the image_name taken from the stimulus_presentations table - Slow to run, could probably be improved with some more intelligent use of pandas - + Adds a column to flash_response_df with the image_name taken from + the stimulus_presentations table + Slow to run, could probably be improved with some more intelligent + use of pandas + INPUTS: a BehaviorOphysExperiment object a flash_response_df for this session OUTPUTS: - fdf, with a new column appended 'image_name' which gives the image identity (like 'im066') for each flash. + fdf, with a new column appended 'image_name' which gives the image + identity (like 'im066') for each flash. ''' fdf = fdf.reset_index() fdf = fdf.set_index('flash_id') - fdf['image_name']= '' + fdf['image_name'] = '' # So slow!!! for stim_id in np.unique(fdf.index.values): - fdf.loc[stim_id,'image_name'] = session.stimulus_presentations.loc[stim_id].image_name + fdf.loc[stim_id, 'image_name'] = session.stimulus_presentations.loc[ + stim_id].image_name fdf = fdf.reset_index() - fdf = fdf.set_index(['cell_specimen_id','flash_id']) + fdf = fdf.set_index(['cell_specimen_id', 'flash_id']) return fdf + def annotate_flash_response_df_with_pref_stim(fdf): ''' - Adds a column to flash_response_df with a boolean value of whether that flash was that cells pref image. - Computes preferred image by looking for the image that on average evokes the largest response. - Slow to run, could probably be improved with more intelligent pandas use + Adds a column to flash_response_df with a boolean value of whether + that flash was that cells pref image. + Computes preferred image by looking for the image that on average + evokes the largest response. + Slow to run, could probably be improved with more intelligent pandas + use INPUTS: fdf, a flash_response_dataframe @@ -229,113 +293,154 @@ def annotate_flash_response_df_with_pref_stim(fdf): cell_key = 'cell_specimen_id' else: cell_key = 'cell' - + # Set up empty column fdf['pref_stim'] = False - + # Compute average response for each image mean_response = fdf.groupby([cell_key, 'image_name']).apply(get_mean_sem) m = mean_response.unstack() - - # Iterate through each cell and find which image evoked the largest average response + + # Iterate through each cell and find which image evoked the largest + # average response for cell in m.index: - temp = np.where(m.loc[cell]['mean_response'].values == np.nanmax(m.loc[cell]['mean_response'].values))[0] - # If the mean_response was NaN, then temp is empty, so we have this check here + temp = np.where(m.loc[cell]['mean_response'].values == np.nanmax( + m.loc[cell]['mean_response'].values))[0] + # If the mean_response was NaN, then temp is empty, so we have this + # check here if len(temp) > 0: image_index = temp[0] pref_image = m.loc[cell]['mean_response'].index[image_index] - # find all repeats of that cell X pref_image, and set 'pref_stim' to True - cell_flash_pairs = fdf[(fdf[cell_key] == cell) & (fdf.image_name == pref_image)].index + # find all repeats of that cell X pref_image, and set + # 'pref_stim' to True + cell_flash_pairs = fdf[ + (fdf[cell_key] == cell) & (fdf.image_name == pref_image)].index fdf.loc[cell_flash_pairs, 'pref_stim'] = True # Test to ensure preferred stimulus is unique for each cell for cell in fdf['cell_specimen_id'].unique(): - assert len(fdf.set_index('cell_specimen_id').loc[cell].query('pref_stim').image_name.unique()) == 1 + assert len(fdf.set_index('cell_specimen_id').loc[cell].query( + 'pref_stim').image_name.unique()) == 1 # Reset the df index - fdf = fdf.set_index(['cell_specimen_id','flash_id']) + fdf = fdf.set_index(['cell_specimen_id', 'flash_id']) return fdf + def get_mean_sem(group): ''' - Returns the mean and sem of the mean_response values for all entries in the group. Copied from VBA - + Returns the mean and sem of the mean_response values for all entries + in the group. Copied from VBA + INPUTS: group is a pandas group - - Output, a pandas series with the average 'mean_response' from the group, and the sem 'mean_response' from the group + + Output, a pandas series with the average 'mean_response' from the + group, and the sem 'mean_response' from the group ''' mean_response = np.mean(group['mean_response']) - sem_response = np.std(group['mean_response'].values) / np.sqrt(len(group['mean_response'].values)) - return pd.Series({'mean_response': mean_response, 'sem_response': sem_response}) + sem_response = np.std(group['mean_response'].values) / np.sqrt( + len(group['mean_response'].values)) + return pd.Series( + {'mean_response': mean_response, 'sem_response': sem_response}) -if __name__=='__main__': +if __name__ == '__main__': case = 0 - if case==0: - # This is the main usage case. - + if case == 0: + # This is the main usage case. + # Grab the experiment ID experiment_id = sys.argv[1] # Define the cache - cache_json = {'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv', - 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files', - 'analysis_files_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files' - } + cache_json = { + 'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys' + '/visual_behavior/SWDB_2019/' + 'visual_behavior_data_manifest.csv', + 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys' + '/visual_behavior/SWDB_2019/nwb_files', + 'analysis_files_base_dir': + '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior' + '/SWDB_2019/extra_files' + } # load the session cache = bpc.BehaviorProjectCache(cache_json) nwb_path = cache.get_nwb_filepath(experiment_id) - api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois = True) + api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True) session = BehaviorOphysExperiment(api) # Where to save the results - output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/flash_response_500msec_response' + output_path = '/allen/programs/braintv/workgroups/nc-ophys' \ + '/visual_behavior/SWDB_2019/' \ + 'flash_response_500msec_response' # Define parameters for dff_trace, and response_window - response_analysis_params = {'window_around_timepoint_seconds':[-.5,.75], # -500ms, 750ms - 'response_window_duration_seconds': 0.5, - 'baseline_window_duration_seconds': 0.5} + response_analysis_params = { + 'window_around_timepoint_seconds': [-.5, .75], # -500ms, 750ms + 'response_window_duration_seconds': 0.5, + 'baseline_window_duration_seconds': 0.5} # compute the base flash_response_df - flash_response_df = get_flash_response_df(session, response_analysis_params) + flash_response_df = get_flash_response_df(session, + response_analysis_params) # Add p_value, image_name, and pref_stim - flash_response_df = get_p_values_from_shuffled_spontaneous(session,flash_response_df) - flash_response_df = add_image_name(session,flash_response_df) - flash_response_df = annotate_flash_response_df_with_pref_stim(flash_response_df) - + flash_response_df = get_p_values_from_shuffled_spontaneous( + session, + flash_response_df) + flash_response_df = add_image_name(session, flash_response_df) + flash_response_df = annotate_flash_response_df_with_pref_stim( + flash_response_df) + # Test columns in flash_response_df - for new_key in ['cell_roi_id','mean_response','baseline_response','dff_trace','dff_trace_timestamps','p_value','image_name', 'pref_stim']: + for new_key in ['cell_roi_id', 'mean_response', 'baseline_response', + 'dff_trace', 'dff_trace_timestamps', 'p_value', + 'image_name', 'pref_stim']: assert new_key in flash_response_df.keys() # Save the flash_response_df to file - output_fn = os.path.join(output_path, 'flash_response_df_{}.h5'.format(experiment_id)) + output_fn = os.path.join(output_path, 'flash_response_df_{}.h5'.format( + experiment_id)) print('Writing flash response df to {}'.format(output_fn)) - flash_response_df.to_hdf(output_fn, key='df', complib='bzip2', complevel=9) - - elif case==1: - # This case is just for debugging. It computes the flash_response_df on a truncated portion of the data. - nwb_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files/behavior_ophys_session_880961028.nwb' + flash_response_df.to_hdf(output_fn, key='df', complib='bzip2', + complevel=9) + + elif case == 1: + # This case is just for debugging. It computes the flash_response_df + # on a truncated portion of the data. + nwb_path = '/allen/programs/braintv/workgroups/nc-ophys' \ + '/visual_behavior/SWDB_2019/nwb_files' \ + '/behavior_ophys_session_880961028.nwb' api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True) session = BehaviorOphysExperiment(api) - #Small data for testing + # Small data for testing session.__dict__['dff_traces'].value = session.dff_traces.iloc[:5] - session.__dict__['stimulus_presentations'].value = session.stimulus_presentations.iloc[:20] - - response_analysis_params = {'window_around_timepoint_seconds':[-.5,.75], # -500ms, 750ms - 'response_window_duration_seconds': 0.5, - 'baseline_window_duration_seconds': 0.5} - - flash_response_df = get_flash_response_df(session, response_analysis_params) - flash_response_df = get_p_values_from_shuffled_spontaneous(session,flash_response_df) - flash_response_df = add_image_name(session,flash_response_df) - flash_response_df = annotate_flash_response_df_with_pref_stim(flash_response_df) + session.__dict__[ + 'stimulus_presentations'].value = \ + session.stimulus_presentations.iloc[ + :20] + + response_analysis_params = { + 'window_around_timepoint_seconds': [-.5, .75], # -500ms, 750ms + 'response_window_duration_seconds': 0.5, + 'baseline_window_duration_seconds': 0.5} + + flash_response_df = get_flash_response_df(session, + response_analysis_params) + flash_response_df = get_p_values_from_shuffled_spontaneous( + session, + flash_response_df) + flash_response_df = add_image_name(session, flash_response_df) + flash_response_df = annotate_flash_response_df_with_pref_stim( + flash_response_df) # Test columns in flash_response_df - for new_key in ['cell_roi_id','mean_response','baseline_response','dff_trace','dff_trace_timestamps','p_value','image_name', 'pref_stim']: + for new_key in ['cell_roi_id', 'mean_response', 'baseline_response', + 'dff_trace', 'dff_trace_timestamps', 'p_value', + 'image_name', 'pref_stim']: assert new_key in flash_response_df.keys() diff --git a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py index 42cdef385..faa750f7c 100644 --- a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py +++ b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py @@ -5,212 +5,272 @@ from scipy import stats import itertools -from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ - BehaviorOphysExperiment -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi, BehaviorOphysLimsApi) -from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc -from importlib import reload; reload(bpc) -from allensdk.brain_observatory.behavior.swdb.analysis_tools import get_nearest_frame, get_trace_around_timepoint, get_mean_in_window +from allensdk.brain_observatory.behavior.swdb import \ + behavior_project_cache as bpc +from importlib import reload + +from allensdk.brain_observatory.behavior.swdb.analysis_tools import \ + get_trace_around_timepoint, get_mean_in_window + +reload(bpc) ''' - This file contains functions and a script for computing the trial_response_df dataframe. - This file was hastily constructed before friday harbor. Places where there are known issues are flagged with PROBLEM + This file contains functions and a script for computing the + trial_response_df dataframe. + This file was hastily constructed before friday harbor. Places where + there are known issues are flagged with PROBLEM ''' -def add_p_vals_tr(tr,response_window = [4,4.5]): +def add_p_vals_tr(tr, response_window=[4, 4.5]): ''' - Computes the p value for each cell's response on each trial. The p-value is computed using the function 'get_p_val' + Computes the p value for each cell's response on each trial. The + p-value is computed using the function 'get_p_val' INPUT: tr, trial_response_dataframe - response_window, the time points in the dff trace to use for computing the p-value. - PROBLEM: The default value here assumes that the dff_trace starts 4 seconds before the change time. - This should be set up with more care and flexibility. + response_window, the time points in the dff trace to use for + computing the p-value. + PROBLEM: The default value here assumes that the + dff_trace starts 4 seconds before the change time. + This should be set up with more care and flexibility. OUTPUTS: - tr, the same trial_response_dataframe, with a new column 'p_value' appended. - + tr, the same trial_response_dataframe, with a new column 'p_value' + appended. + ASSERTS: - tr['p_value'] is inclusively bounded between 0 and 1, and does not include NaNs + tr['p_value'] is inclusively bounded between 0 and 1, and does not + include NaNs ''' - + # Set up empty column tr['p_value'] = 1. - ophys_frame_rate=31. #Shouldn't hard code this PROBLEM - + ophys_frame_rate = 31. # Shouldn't hard code this PROBLEM + # Iterate over trial/cell pairs, and compute p-value - for index,row in tr.iterrows(): - tr.at[index,'p_value'] = get_p_val(row.dff_trace, response_window, ophys_frame_rate) + for index, row in tr.iterrows(): + tr.at[index, 'p_value'] = get_p_val(row.dff_trace, response_window, + ophys_frame_rate) - # Test to ensure p values are bounded between 0 and 1, and dont include NaNs + # Test to ensure p values are bounded between 0 and 1, and dont include + # NaNs assert np.all(tr['p_value'].values <= 1) assert np.all(tr['p_value'].values >= 0) - assert np.all(~np.isnan(tr['p_value'].values)) + assert np.all(~np.isnan(tr['p_value'].values)) return tr + def get_p_val(trace, response_window, frame_rate): ''' - Computes a p-value for the trace by comparing the dff in the response_window to the same sized trace before the response_window. - PROBLEM: This should be computed by comparing to spontaneous activity to be consistent with the flash_response_df + Computes a p-value for the trace by comparing the dff in the + response_window to the same sized trace before the response_window. + PROBLEM: This should be computed by comparing to spontaneous + activity to be consistent with the flash_response_df INPUTS: - trace, the dff trace for this cell/trial - response_window, [start_time, end_time] the time in seconds from the start of trace to asses whether the activity is significant - frame_rate, the number of samples in trace per second. + trace, the dff trace for this cell/trial + response_window, [start_time, end_time] the time in seconds from the + start of trace to asses whether the activity is significant + frame_rate, the number of samples in trace per second. OUTPUTS: a p-value ''' response_window_duration = response_window[1] - response_window[0] baseline_end = int(response_window[0] * frame_rate) - baseline_start = int((response_window[0] - response_window_duration) * frame_rate) + baseline_start = int( + (response_window[0] - response_window_duration) * frame_rate) stim_start = int(response_window[0] * frame_rate) - stim_end = int((response_window[0] + response_window_duration) * frame_rate) - (_, p) = stats.f_oneway(trace[baseline_start:baseline_end], trace[stim_start:stim_end]) + stim_end = int( + (response_window[0] + response_window_duration) * frame_rate) + (_, p) = stats.f_oneway(trace[baseline_start:baseline_end], + trace[stim_start:stim_end]) return p + def annotate_trial_response_df_with_pref_stim(trial_response_df): ''' - Computes the preferred stimulus for each cell/trial combination. Preferred image is computed by seeing which image evoked the largest average mean_response across all change_images. + Computes the preferred stimulus for each cell/trial combination. + Preferred image is computed by seeing which image evoked the largest + average mean_response across all change_images. INPUTS: trial_response_df, the trial_response_df to be annotated OUTPUTS: - a copy of trial_response_df with a new column appended 'pref_stim' which is a boolean TRUE/FALSE for whether that change_image was that cell's preferred image. - + a copy of trial_response_df with a new column appended 'pref_stim' + which is a boolean TRUE/FALSE for whether that change_image was that + cell's preferred image. + ASSERTS: - Each cell has one unique preferred stimulus + Each cell has one unique preferred stimulus ''' - + # Copy the trial_response_df rdf = trial_response_df.copy() - + # Set up empty column rdf['pref_stim'] = False - + # get average mean_response for each cell X change_image - mean_response = rdf.groupby(['cell_specimen_id', 'change_image_name']).apply(get_mean_sem_trace) + mean_response = rdf.groupby( + ['cell_specimen_id', 'change_image_name']).apply(get_mean_sem_trace) m = mean_response.unstack() - # set index to be cell/image pairs + # set index to be cell/image pairs rdf = rdf.reset_index() - rdf = rdf.set_index(['cell_specimen_id','change_image_name']) + rdf = rdf.set_index(['cell_specimen_id', 'change_image_name']) - # Iterate through cells, and determine which change_image evoked the largest response + # Iterate through cells, and determine which change_image evoked the + # largest response for cell in m.index: - image_index = np.where(m.loc[cell]['mean_response'].values == np.max(m.loc[cell]['mean_response'].values))[0][0] + image_index = np.where(m.loc[cell]['mean_response'].values == np.max( + m.loc[cell]['mean_response'].values))[0][0] pref_image = m.loc[cell]['mean_response'].index[image_index] - - # Update the cell X change_image pairs to have the pref_stim set to True - rdf.at[(cell,pref_image),'pref_stim'] = True + + # Update the cell X change_image pairs to have the pref_stim set to + # True + rdf.at[(cell, pref_image), 'pref_stim'] = True # Test to ensure preferred stimulus is unique for each cell - for cell in rdf.reset_index()['cell_specimen_id'].unique(): - assert len(rdf.reset_index().set_index('cell_specimen_id').loc[cell].query('pref_stim').change_image_name.unique()) == 1 + for cell in rdf.reset_index()['cell_specimen_id'].unique(): + assert len( + rdf.reset_index().set_index('cell_specimen_id').loc[cell].query( + 'pref_stim').change_image_name.unique()) == 1 # Reset index to be cell/trial pairs rdf = rdf.reset_index() - rdf = rdf.set_index(['cell_specimen_id','trial_id']) + rdf = rdf.set_index(['cell_specimen_id', 'trial_id']) return rdf + def get_mean_sem_trace(group): ''' Computes the average and sem of the mean_response column INPUTS: group, a pandas group - + OUTPUT: - a pandas series with the mean_response, sem_response, mean_trace, sem_trace, and mean_responses computed for the group. + a pandas series with the mean_response, sem_response, mean_trace, + sem_trace, and mean_responses computed for the group. ''' mean_response = np.mean(group['mean_response']) mean_responses = group['mean_response'].values - sem_response = np.std(group['mean_response'].values) / np.sqrt(len(group['mean_response'].values)) + sem_response = np.std(group['mean_response'].values) / np.sqrt( + len(group['mean_response'].values)) mean_trace = np.mean(group['dff_trace']) - sem_trace = np.std(group['dff_trace'].values) / np.sqrt(len(group['dff_trace'].values)) - return pd.Series({'mean_response': mean_response, 'sem_response': sem_response, - 'mean_trace': mean_trace, 'sem_trace': sem_trace, - 'mean_responses': mean_responses}) + sem_trace = np.std(group['dff_trace'].values) / np.sqrt( + len(group['dff_trace'].values)) + return pd.Series( + {'mean_response': mean_response, 'sem_response': sem_response, + 'mean_trace': mean_trace, 'sem_trace': sem_trace, + 'mean_responses': mean_responses}) + def get_trial_response_df(session, response_analysis_params): ''' Computes the trial_response_df for the session - PROBLEM: Ignores aborted trials + PROBLEM: Ignores aborted trials INPUTS: session, a behaviorOphysSession object to be analyzed response_analysis_params, a dictionary with keys: - 'window_around_timepoint_seconds' The window around the change_time to use in the dff trace - 'response_window_duration_seconds' The duration after the change time to use in the mean_response - 'baseline_window_duration_seconds' The duration before the change time to use as the baseline_response - + 'window_around_timepoint_seconds' The window around the + change_time to use in the dff trace + 'response_window_duration_seconds' The duration after the + change time to use in the mean_response + 'baseline_window_duration_seconds' The duration before the + change time to use as the baseline_response + OUTPUTS: - trial_response_df, a pandas dataframe with multi-index (cell_specimen_id/trial_id), and columns: + trial_response_df, a pandas dataframe with multi-index ( + cell_specimen_id/trial_id), and columns: cell_roi_id, this sessions roi_id mean_response, the average dff in the response_window baseline_response, the average dff in the baseline window dff_trace, the dff_trace in the window_around_timepoint_seconds dff_trace_timestamps, the timestamps for the dff_trace ''' - frame_rate = 31. #PROBLEM, shouldnt hard code this here + frame_rate = 31. # PROBLEM, shouldnt hard code this here # get data to analyze dff_traces = session.dff_traces.copy() trials = session.trials.copy() - trials = trials[trials.aborted==False] # PROBLEM + trials = trials[~trials.aborted] # PROBLEM # get params to define response window, in seconds - window_around_timepoint_seconds = response_analysis_params['window_around_timepoint_seconds'] - response_window_duration_seconds = response_analysis_params['response_window_duration_seconds'] - baseline_window_duration_seconds = response_analysis_params['baseline_window_duration_seconds'] - mean_response_window_seconds = [np.abs(window_around_timepoint_seconds[0]), - np.abs(window_around_timepoint_seconds[0]) + response_window_duration_seconds] - baseline_window_seconds = [np.abs(window_around_timepoint_seconds[0]) - baseline_window_duration_seconds, - np.abs(window_around_timepoint_seconds[0])] + window_around_timepoint_seconds = response_analysis_params[ + 'window_around_timepoint_seconds'] + response_window_duration_seconds = response_analysis_params[ + 'response_window_duration_seconds'] + baseline_window_duration_seconds = response_analysis_params[ + 'baseline_window_duration_seconds'] + mean_response_window_seconds = [np.abs(window_around_timepoint_seconds[0]), + np.abs(window_around_timepoint_seconds[ + 0]) + + response_window_duration_seconds] + baseline_window_seconds = [np.abs( + window_around_timepoint_seconds[0]) - baseline_window_duration_seconds, + np.abs(window_around_timepoint_seconds[0])] # Set up multi-index dataframe - cell_trial_combinations = itertools.product(dff_traces.index,trials.index) - index = pd.MultiIndex.from_tuples(cell_trial_combinations, names=['cell_specimen_id', 'trial_id']) + cell_trial_combinations = itertools.product(dff_traces.index, trials.index) + index = pd.MultiIndex.from_tuples(cell_trial_combinations, + names=['cell_specimen_id', 'trial_id']) df = pd.DataFrame(index=index) # Iterate through cell/trial pairs, and construct the columns traces_list = [] trace_timestamps_list = [] - for cell_specimen_id, trial_id in itertools.product(dff_traces.index,trials.index): + for cell_specimen_id, trial_id in itertools.product(dff_traces.index, + trials.index): timepoint = trials.loc[trial_id]['change_time'] cell_roi_id = dff_traces.loc[cell_specimen_id]['cell_roi_id'] full_cell_trace = dff_traces.loc[cell_specimen_id, 'dff'] - trace, trace_timestamps = get_trace_around_timepoint(full_cell_trace, timepoint, session.ophys_timestamps, - window_around_timepoint_seconds, frame_rate) - mean_response = get_mean_in_window(trace, mean_response_window_seconds, frame_rate) - baseline_response = get_mean_in_window(trace, baseline_window_seconds, frame_rate) + trace, trace_timestamps = get_trace_around_timepoint( + full_cell_trace, + timepoint, + session.ophys_timestamps, + window_around_timepoint_seconds, + frame_rate) + mean_response = get_mean_in_window(trace, mean_response_window_seconds, + frame_rate) + baseline_response = get_mean_in_window(trace, baseline_window_seconds, + frame_rate) traces_list.append(trace) trace_timestamps_list.append(trace_timestamps) df.loc[(cell_specimen_id, trial_id), 'cell_roi_id'] = int(cell_roi_id) df.loc[(cell_specimen_id, trial_id), 'mean_response'] = mean_response - df.loc[(cell_specimen_id, trial_id), 'baseline_response'] = baseline_response + df.loc[(cell_specimen_id, + trial_id), 'baseline_response'] = baseline_response df.insert(loc=1, column='dff_trace', value=traces_list) - df.insert(loc=2, column='dff_trace_timestamps', value=trace_timestamps_list) + df.insert(loc=2, column='dff_trace_timestamps', + value=trace_timestamps_list) return df -if __name__=='__main__': - # Load cache - cache_json = {'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv', - 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files', - 'analysis_files_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files_final' - } +if __name__ == '__main__': + # Load cache + cache_json = { + 'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys' + '/visual_behavior/SWDB_2019/' + 'visual_behavior_data_manifest.csv', + 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys' + '/visual_behavior/SWDB_2019/nwb_files', + 'analysis_files_base_dir': + '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior' + '/SWDB_2019/extra_files_final' + } cache = bpc.BehaviorProjectCache(cache_json) - case=0 + case = 0 if case == 0: # this is the main use case - experiment_id = sys.argv[1] # get experiment_id to analyze + experiment_id = sys.argv[1] # get experiment_id to analyze # Load session object # experiment_id = cache.manifest.iloc[5]['ophys_experiment_id'] @@ -218,57 +278,61 @@ def get_trial_response_df(session, response_analysis_params): # api = BehaviorOphysNwbApi(nwb_path, filter_invalid_rois=True) # session = BehaviorOphysExperiment(api) - # Get the session using the cache so that the change time fix is applied + # Get the session using the cache so that the change time fix is + # applied session = cache.get_session(experiment_id) - change_times = session.trials['change_time'][~pd.isnull(session.trials['change_time'])].values + change_times = session.trials['change_time'][ + ~pd.isnull(session.trials['change_time'])].values flash_times = session.stimulus_presentations['start_time'].values assert np.all(np.isin(change_times, flash_times)) - output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files_final' + output_path = '/allen/programs/braintv/workgroups/nc-ophys' \ + '/visual_behavior/SWDB_2019/extra_files_final' - response_analysis_params = {'window_around_timepoint_seconds':[-4,8], - 'response_window_duration_seconds': 0.5, - 'baseline_window_duration_seconds': 0.5} + response_analysis_params = {'window_around_timepoint_seconds': [-4, 8], + 'response_window_duration_seconds': 0.5, + 'baseline_window_duration_seconds': 0.5} - trial_response_df = get_trial_response_df(session, response_analysis_params) + trial_response_df = get_trial_response_df(session, + response_analysis_params) trial_metadata = session.trials.copy() trial_metadata.index.names = ['trial_id'] trial_response_df = trial_response_df.join(trial_metadata) trial_response_df = add_p_vals_tr(trial_response_df) - trial_response_df = annotate_trial_response_df_with_pref_stim(trial_response_df) + trial_response_df = annotate_trial_response_df_with_pref_stim( + trial_response_df) - output_fn = os.path.join(output_path, 'trial_response_df_{}.h5'.format(experiment_id)) + output_fn = os.path.join(output_path, 'trial_response_df_{}.h5'.format( + experiment_id)) print('Writing trial response df to {}'.format(output_fn)) - trial_response_df.to_hdf(output_fn, key='df', complib='bzip2', complevel=9) + trial_response_df.to_hdf(output_fn, key='df', complib='bzip2', + complevel=9) elif case == 1: - # This is a debugging case - experiment_id = 846487947 - - # api = BehaviorOphysLimsApi(experiment_id) - # session = BehaviorOphysExperiment(api) - # nwb_path = cache.get_nwb_filepath(experiment_id) - # api = BehaviorOphysNwbApi(nwb_path) - # session = BehaviorOphysExperiment(api) + # This is a debugging case + experiment_id = 846487947 session = cache.get_session(experiment_id) - change_times = session.trials['change_time'][~pd.isnull(session.trials['change_time'])].values + change_times = session.trials['change_time'][ + ~pd.isnull(session.trials['change_time'])].values flash_times = session.stimulus_presentations['start_time'].values assert np.all(np.isin(change_times, flash_times)) - output_path = '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files_final' + output_path = '/allen/programs/braintv/workgroups/nc-ophys' \ + '/visual_behavior/SWDB_2019/extra_files_final' - response_analysis_params = {'window_around_timepoint_seconds':[-4,8], - 'response_window_duration_seconds': 0.5, - 'baseline_window_duration_seconds': 0.5} + response_analysis_params = {'window_around_timepoint_seconds': [-4, 8], + 'response_window_duration_seconds': 0.5, + 'baseline_window_duration_seconds': 0.5} - trial_response_df = get_trial_response_df(session, response_analysis_params) + trial_response_df = get_trial_response_df(session, + response_analysis_params) trial_metadata = session.trials.copy() trial_metadata.index.names = ['trial_id'] trial_response_df = trial_response_df.join(trial_metadata) trial_response_df = add_p_vals_tr(trial_response_df) - trial_response_df = annotate_trial_response_df_with_pref_stim(trial_response_df) - + trial_response_df = annotate_trial_response_df_with_pref_stim( + trial_response_df) diff --git a/allensdk/brain_observatory/behavior/validation.py b/allensdk/brain_observatory/behavior/validation.py deleted file mode 100644 index acb4045d0..000000000 --- a/allensdk/brain_observatory/behavior/validation.py +++ /dev/null @@ -1,110 +0,0 @@ -import h5py -import os - -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysLimsApi) -from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api \ - import OphysLimsApi -from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ - BehaviorOphysExperiment - -class ValidationError(AssertionError): - pass - -def get_raw_ophys_file_shape(raw_filepath): - with h5py.File(raw_filepath, 'r') as raw_file: - raw_data_shape = raw_file['data'].shape - return raw_data_shape - -def validate_ophys_dff_length(ophys_experiment_id, api=None): - api = OphysLimsApi() if api is None else api - - ophys_experiment_dir = api.get_ophys_experiment_dir(ophys_experiment_id) - raw_filepath = os.path.join(ophys_experiment_dir, str(ophys_experiment_id)+'.h5') - raw_data_shape = get_raw_ophys_file_shape(raw_filepath) - - dff_filepath = api.get_dff_file(ophys_experiment_id=ophys_experiment_id) - dff_shape = get_raw_ophys_file_shape(dff_filepath) - - if raw_data_shape[0] != dff_shape[1]: - raise ValidationError('dff length does not match raw data length') - -def validate_ophys_timestamps(ophys_experiment_id, api=None): - api = BehaviorOphysLimsApi() if api is None else api - - ophys_experiment_dir = api.get_ophys_experiment_dir(ophys_experiment_id) - raw_filepath = os.path.join(ophys_experiment_dir, str(ophys_experiment_id)+'.h5') - raw_data_shape = get_raw_ophys_file_shape(raw_filepath) - - ophys_timestamps_shape = api.get_ophys_timestamps(ophys_experiment_id=ophys_experiment_id).shape - - if raw_data_shape[0] != ophys_timestamps_shape[0]: - raise ValidationError('ophys_timestamp length does not match raw data length') - -def validate_last_trial_ends_adjacent_to_flash(ophys_experiment_id, api=None, verbose=False): - # ensure that the last trial ends sometime on the last flash/blank cycle - # i.e, if this is the last flash/blank iteration (high = stimulus present): - # ------- ------- ------- ------- - # | | | | | | | | - # --------- --------- --------- --------- ------------------------------------------- - # ^ ^ - # The last trial has to have ended somewhere between the two carrots where: - # the first carrot represents the time of the last recorded stimulus flash - # the second carrot represents the time at which another flash should have started, after accounting for the possibility of the session ending on an omitted flash - - api = BehaviorOphysLimsApi() if api is None else api - session = BehaviorOphysExperiment(api) - - # get the flash/blank parameters - max_flash_duration = session.stimulus_presentations['duration'].max() - max_blank_duration = session.task_parameters['blank_duration_sec'][1] - - # count number of omitted flashes at the very end of the session - N_final_omitted_flashes = session.stimulus_presentations.index.max() - session.stimulus_presentations.query('omitted == False').index.max() - - # get the start/end time of the last valid (non-omitted) flash - last_flash_start = session.stimulus_presentations.query('omitted == False')['start_time'].iloc[-1] - last_flash_end = session.stimulus_presentations.query('omitted == False')['stop_time'].iloc[-1] - - # calculate when the next stimulus should have flashed, after accounting for any omitted flashes at the end of the session - next_flash_would_have_started = last_flash_end + max_blank_duration + N_final_omitted_flashes*(max_flash_duration + max_blank_duration) - - # get the end time of the last trial - last_trial_end = session.trials.iloc[-1]['stop_time'] - - if verbose: - print('last flash ended at {}'.format(last_flash_end)) - print('another flash should have started by {}'.format(next_flash_would_have_started)) - print('last trial ended at {}'.format(last_trial_end)) - - if not last_flash_start <= last_trial_end <= next_flash_would_have_started: - raise ValidationError('The last trial does not end between the start of the last flash and the expected start time of the next flash') - -if __name__ == "__main__": - - api = BehaviorOphysLimsApi() - ophys_experiment_id_list = [775614751, 778644591, 787461073, 782675436, 783928214, 783927872, - 787501821, 787498309, 788490510, 788488596, 788489531, 789359614, - 790149413, 790709081, 791119849, 791453282, 791980891, 792813858, - 792812544, 792816531, 792815735, 794381992, 794378505, 795076128, - 795073741, 795952471, 795952488, 795953296, 795948257, 796106850, - 796106321, 796108483, 796105823, 796308505, 797255551, 795075034, - 798403387, 798404219, 799366517, 799368904, 799368262, 803736273, - 805100431, 805784331, 805784313, 806456687, 806455766, 806989729, - 807753318, 807752719, 807753334, 807753920, 796105304, 784482326, - 779335436, 782675457, 791974731, 791979236, - 800034837, 802649986, 806990245, 808621958, - 808619526, 808619543, 808621034, 808621015] - - for ophys_experiment_id in ophys_experiment_id_list: - validation_functions_to_run = [ - validate_ophys_timestamps, - validate_ophys_dff_length, - validate_last_trial_ends_adjacent_to_flash, - ] - for validation_function in validation_functions_to_run: - - try: - validation_function(ophys_experiment_id, api=api) - except ValidationError as e: - print(ophys_experiment_id, e) diff --git a/allensdk/brain_observatory/behavior/write_behavior_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_behavior_nwb/__main__.py index ff0263282..9932f64ea 100644 --- a/allensdk/brain_observatory/behavior/write_behavior_nwb/__main__.py +++ b/allensdk/brain_observatory/behavior/write_behavior_nwb/__main__.py @@ -3,11 +3,10 @@ import sys import argschema import marshmallow +from pynwb import NWBHDF5IO from allensdk.brain_observatory.behavior.behavior_session import ( BehaviorSession) -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorNwbApi, BehaviorJsonApi, BehaviorLimsApi) from allensdk.brain_observatory.behavior.write_behavior_nwb._schemas import ( InputSchema, OutputSchema) from allensdk.brain_observatory.argschema_utilities import ( @@ -28,21 +27,22 @@ def write_behavior_nwb(session_data, nwb_filepath): os.remove(filename) try: - json_session = BehaviorSession(api=BehaviorJsonApi(session_data)) - lims_api = BehaviorLimsApi( - behavior_session_id=session_data['behavior_session_id']) - lims_session = BehaviorSession(api=lims_api) + json_session = BehaviorSession.from_json(session_data) + + behavior_session_id = session_data['behavior_session_id'] + lims_session = BehaviorSession.from_lims(behavior_session_id) logging.info("Comparing a BehaviorSession created from JSON " "with a BehaviorSession created from LIMS") assert sessions_are_equal(json_session, lims_session, reraise=True) - BehaviorNwbApi(nwb_filepath_inprogress).save(json_session) + nwbfile = lims_session.to_nwb() + with NWBHDF5IO(nwb_filepath_inprogress, 'w') as nwb_file_writer: + nwb_file_writer.write(nwbfile) logging.info("Comparing a BehaviorSession created from JSON " "with a BehaviorSession created from NWB") - nwb_api = BehaviorNwbApi(nwb_filepath_inprogress) - nwb_session = BehaviorSession(api=nwb_api) + nwb_session = BehaviorSession.from_nwb_path(nwb_filepath_inprogress) assert sessions_are_equal(json_session, nwb_session, reraise=True) os.rename(nwb_filepath_inprogress, nwb_filepath) diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py index fdb66905e..127ddcad8 100644 --- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py +++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py @@ -3,11 +3,10 @@ import sys import argschema import marshmallow +from pynwb import NWBHDF5IO from allensdk.brain_observatory.behavior.behavior_ophys_experiment import ( BehaviorOphysExperiment) -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi, BehaviorOphysJsonApi, BehaviorOphysLimsApi) from allensdk.brain_observatory.behavior.write_nwb._schemas import ( InputSchema, OutputSchema) from allensdk.brain_observatory.argschema_utilities import ( @@ -30,24 +29,24 @@ def write_behavior_ophys_nwb(session_data: dict, os.remove(filename) try: - json_api = BehaviorOphysJsonApi(data=session_data, - skip_eye_tracking=skip_eye_tracking) - json_session = BehaviorOphysExperiment(api=json_api) - lims_api = BehaviorOphysLimsApi( + json_session = BehaviorOphysExperiment.from_json( + session_data=session_data, skip_eye_tracking=skip_eye_tracking) + lims_session = BehaviorOphysExperiment.from_lims( ophys_experiment_id=session_data['ophys_experiment_id'], skip_eye_tracking=skip_eye_tracking) - lims_session = BehaviorOphysExperiment(api=lims_api) logging.info("Comparing a BehaviorOphysExperiment created from JSON " "with a BehaviorOphysExperiment created from LIMS") - assert sessions_are_equal(json_session, lims_session, reraise=True) + assert sessions_are_equal(json_session, lims_session, reraise=True, + ignore_keys={'metadata': {'project_code'}}) - BehaviorOphysNwbApi(nwb_filepath_inprogress).save(json_session) + nwbfile = json_session.to_nwb() + with NWBHDF5IO(nwb_filepath_inprogress, 'w') as nwb_file_writer: + nwb_file_writer.write(nwbfile) logging.info("Comparing a BehaviorOphysExperiment created from JSON " "with a BehaviorOphysExperiment created from NWB") - nwb_api = BehaviorOphysNwbApi(nwb_filepath_inprogress) - nwb_session = BehaviorOphysExperiment(api=nwb_api) + nwb_session = BehaviorOphysExperiment.from_nwb(nwbfile=nwbfile) assert sessions_are_equal(json_session, nwb_session, reraise=True) os.rename(nwb_filepath_inprogress, nwb_filepath) diff --git a/allensdk/brain_observatory/comparison_utils.py b/allensdk/brain_observatory/comparison_utils.py new file mode 100644 index 000000000..a9b6b1ab2 --- /dev/null +++ b/allensdk/brain_observatory/comparison_utils.py @@ -0,0 +1,70 @@ +import datetime +import math +from typing import Any, Optional, Set + +import SimpleITK as sitk +import numpy as np +import pandas as pd +import xarray as xr +from pandas.util.testing import assert_frame_equal + + +def compare_fields(x1: Any, x2: Any, err_msg="", + ignore_keys: Optional[Set[str]] = None): + """Helper function to compare if two fields (attributes) + are equal to one another. + + Parameters + ---------- + x1 : Any + The first field + x2 : Any + The other field + err_msg : str, optional + The error message to display if two compared fields do not equal + one another, by default "" (an empty string) + ignore_keys + For dictionary comparison, ignore these keys + """ + if ignore_keys is None: + ignore_keys = set() + + if isinstance(x1, pd.DataFrame): + try: + assert_frame_equal(x1, x2, check_like=True) + except Exception: + print(err_msg) + raise + elif isinstance(x1, np.ndarray): + np.testing.assert_array_almost_equal(x1, x2, err_msg=err_msg) + elif isinstance(x1, xr.DataArray): + xr.testing.assert_allclose(x1, x2) + elif isinstance(x1, (list, tuple)): + assert len(x1) == len(x2) + for i in range(len(x1)): + compare_fields(x1=x1[i], x2=x2[i]) + elif isinstance(x1, (sitk.Image,)): + assert x1.GetSize() == x2.GetSize(), err_msg + assert x1 == x2, err_msg + elif isinstance(x1, (datetime.datetime, pd.Timestamp)): + if isinstance(x1, pd.Timestamp): + x1 = x1.to_pydatetime() + if isinstance(x2, pd.Timestamp): + x2 = x2.to_pydatetime() + time_delta = (x1 - x2).total_seconds() + # Timestamp differences should be less than 60 seconds + assert abs(time_delta) < 60 + elif isinstance(x1, (float,)): + if math.isnan(x1) or math.isnan(x2): + both_nan = (math.isnan(x1) and math.isnan(x2)) + assert both_nan, err_msg + else: + assert x1 == x2, err_msg + elif isinstance(x1, (dict,)): + for key in set(x1.keys()).union(set(x2.keys())): + if key in ignore_keys: + continue + key_err_msg = f"Mismatch when checking key {key}. {err_msg}" + compare_fields(x1[key], x2[key], err_msg=key_err_msg) + else: + assert x1 == x2, err_msg diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py index c63664365..30e4fc847 100644 --- a/allensdk/brain_observatory/nwb/__init__.py +++ b/allensdk/brain_observatory/nwb/__init__.py @@ -17,7 +17,8 @@ from pynwb.ophys import ( DfOverF, ImageSegmentation, OpticalChannel, Fluorescence) -from allensdk.brain_observatory.behavior.stimulus_processing.stimulus_templates import StimulusTemplate # noqa: E501 +from allensdk.brain_observatory.behavior.data_objects.stimuli\ + .stimulus_templates import StimulusTemplate from allensdk.brain_observatory.behavior.write_nwb.extensions.stimulus_template.ndx_stimulus_template import StimulusTemplateExtension # noqa: E501 from allensdk.brain_observatory.nwb.nwb_utils import (get_column_name) from allensdk.brain_observatory import dict_to_indexed_array diff --git a/allensdk/brain_observatory/nwb/nwb_utils.py b/allensdk/brain_observatory/nwb/nwb_utils.py index 591364a25..5807eba46 100644 --- a/allensdk/brain_observatory/nwb/nwb_utils.py +++ b/allensdk/brain_observatory/nwb/nwb_utils.py @@ -1,7 +1,11 @@ -import pandas as pd # All of the omitted stimuli have a duration of 250ms as defined # by the Visual Behavior team. For questions about duration contact that # team. +from pynwb import NWBFile, ProcessingModule +from pynwb.base import Images +from pynwb.image import GrayscaleImage + +from allensdk.brain_observatory.behavior.image_api import ImageApi, Image def get_column_name(table_cols: list, @@ -25,28 +29,57 @@ def get_column_name(table_cols: list, return column_names[0] -def set_omitted_stop_time(stimulus_table: pd.DataFrame, - omitted_time_duration: float=0.25) -> None: +def get_image(nwbfile: NWBFile, name: str, module: str) -> Image: + nwb_img = nwbfile.processing[module].get_data_interface('images')[name] + data = nwb_img.data + resolution = nwb_img.resolution # px/cm + spacing = [resolution * 10, resolution * 10] + + img = ImageApi.serialize(data, spacing, 'mm') + img = ImageApi.deserialize(img=img) + return img + + +def add_image_to_nwb(nwbfile: NWBFile, image_data: Image, image_name: str): """ - This function sets the stop time for a row that of a stimuli table that - is a omitted stimuli. A omitted stimuli is a stimuli where a mouse is - shown only a grey screen and these last for 250 milliseconds. These do not - include a stop_time or end_frame as other stimuli in the stimulus table due - to design choices. For these stimuli to be added they must have the - stop_time calculated and put into the row as data before writing to NWB. - :param stimulus_table: pd.DataFrame that contains the stimuli presented to - an experiment subject - :param omitted_time_duration: The duration in seconds of the expected length - of the omitted stimuli - :return: - stimulus_table_row: returns the same dictionary as inputted but with - an additional entry for stop_time. + Adds image given by image_data with name image_name to nwbfile + + Parameters + ---------- + nwbfile + nwbfile to add image to + image_data + The image data + image_name + Image name + + Returns + ------- + None """ - omitted_row_indexs = stimulus_table.index[stimulus_table['omitted']].tolist() - for omitted_row_idx in omitted_row_indexs: - row = stimulus_table.iloc[omitted_row_idx] - start_time = row['start_time'] - end_time = start_time + omitted_time_duration - row['stop_time'] = end_time - row['duration'] = omitted_time_duration - stimulus_table.iloc[omitted_row_idx] = row \ No newline at end of file + module_name = 'ophys' + description = '{} image at pixels/cm resolution'.format(image_name) + + data, spacing, unit = image_data + + assert spacing[0] == spacing[1] and len( + spacing) == 2 and unit == 'mm' + + if module_name not in nwbfile.processing: + ophys_mod = ProcessingModule(module_name, + 'Ophys processing module') + nwbfile.add_processing_module(ophys_mod) + else: + ophys_mod = nwbfile.processing[module_name] + + image = GrayscaleImage(image_name, + data, + resolution=spacing[0] / 10, + description=description) + + if 'images' not in ophys_mod.containers: + images = Images(name='images') + ophys_mod.add_data_interface(images) + else: + images = ophys_mod['images'] + images.add_image(image) diff --git a/allensdk/brain_observatory/session_api_utils.py b/allensdk/brain_observatory/session_api_utils.py index 5e1689421..2b07882ca 100644 --- a/allensdk/brain_observatory/session_api_utils.py +++ b/allensdk/brain_observatory/session_api_utils.py @@ -1,20 +1,17 @@ import inspect import logging -import math import warnings -import datetime +from collections import Callable from itertools import zip_longest -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional, Set, Iterable import numpy as np import pandas as pd -import xarray as xr -import SimpleITK as sitk -from pandas.util.testing import assert_frame_equal +from allensdk.brain_observatory.comparison_utils import compare_fields +from allensdk.brain_observatory.behavior.data_objects import DataObject -from allensdk.core.lazy_property import LazyProperty logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -153,9 +150,13 @@ def clear_updated_params(self, data_params: set): self._updated_params -= data_params -def sessions_are_equal(A, B, reraise=False) -> bool: - """Check if two Session objects are equal (have same methods and - attributes). +def sessions_are_equal(A, B, reraise=False, + ignore_keys: Optional[Dict[str, Set[str]]] = None, + skip_fields: Optional[Iterable] = None, + test_methods=False) \ + -> bool: + """Check if two Session objects are equal (have same property and + get method values). Parameters ---------- @@ -166,30 +167,53 @@ def sessions_are_equal(A, B, reraise=False) -> bool: reraise : bool, optional Whether to reraise when encountering an Assertion or AttributeError, by default False + ignore_keys + Set of keys to ignore for property/method. Should be given as + {property/method name: {field_to_ignore, ...}, ...} + test_methods + Whether to test get methods + skip_fields + Do not compare these fields Returns ------- bool Whether the two sessions are equal to one another. """ + if ignore_keys is None: + ignore_keys = dict() + if skip_fields is None: - field_set = set() - for key, val in A.__dict__.items(): - if isinstance(val, LazyProperty): - field_set.add(key) - for key, val in B.__dict__.items(): - if isinstance(val, LazyProperty): - field_set.add(key) + skip_fields = set() + + A_data_attrs_and_methods = A.list_data_attributes_and_methods() + B_data_attrs_and_methods = B.list_data_attributes_and_methods() + field_set = set(A_data_attrs_and_methods).union(B_data_attrs_and_methods) logger.info(f"Comparing the following fields: {field_set}") for field in sorted(field_set): + if field in skip_fields: + continue + try: logger.info(f"Comparing field: {field}") x1, x2 = getattr(A, field), getattr(B, field) + if test_methods: + if isinstance(x1, Callable): + x1 = x1() + x2 = x2() + else: + continue + err_msg = (f"{field} on {A} did not equal {field} " f"on {B} (\n{x1} vs\n{x2}\n)") - compare_session_fields(x1, x2, err_msg) + if isinstance(x1, DataObject): + x1 = x1.value + if isinstance(x2, DataObject): + x2 = x2.value + compare_fields(x1, x2, err_msg, + ignore_keys=ignore_keys.get(field, None)) except NotImplementedError: A_implements_get_field = hasattr( @@ -205,54 +229,3 @@ def sessions_are_equal(A, B, reraise=False) -> bool: return False return True - - -def compare_session_fields(x1: Any, x2: Any, err_msg=""): - """Helper function to compare if two fields (attributes) from a - Session object are equal to one another. - - Parameters - ---------- - x1 : Any - The field from the first session to compare - x2 : Any - The corresponding field from the second session to compare - err_msg : str, optional - The error message to display if two compared fields do not equal - one another, by default "" (an empty string) - """ - if isinstance(x1, pd.DataFrame): - try: - assert_frame_equal(x1, x2, check_like=True) - except Exception: - print(err_msg) - raise - elif isinstance(x1, np.ndarray): - np.testing.assert_array_almost_equal(x1, x2, err_msg=err_msg) - elif isinstance(x1, xr.DataArray): - xr.testing.assert_allclose(x1, x2) - elif isinstance(x1, (list,)): - assert x1 == x2, err_msg - elif isinstance(x1, (sitk.Image,)): - assert x1.GetSize() == x2.GetSize(), err_msg - assert x1 == x2, err_msg - elif isinstance(x1, (datetime.datetime, pd.Timestamp)): - if isinstance(x1, pd.Timestamp): - x1 = x1.to_pydatetime() - if isinstance(x2, pd.Timestamp): - x2 = x2.to_pydatetime() - time_delta = (x1 - x2).total_seconds() - # Timestamp differences should be less than 60 seconds - assert abs(time_delta) < 60 - elif isinstance(x1, (float,)): - if math.isnan(x1) or math.isnan(x2): - both_nan = (math.isnan(x1) and math.isnan(x2)) - assert both_nan, err_msg - else: - assert x1 == x2, err_msg - elif isinstance(x1, (dict,)): - for key in set(x1.keys()).union(set(x2.keys())): - key_err_msg = f"Mismatch when checking key {key}. {err_msg}" - compare_session_fields(x1[key], x2[key], err_msg=key_err_msg) - else: - assert x1 == x2, err_msg diff --git a/allensdk/internal/api/mtrain_api.py b/allensdk/internal/api/mtrain_api.py index d1c6aa665..1c096ac86 100644 --- a/allensdk/internal/api/mtrain_api.py +++ b/allensdk/internal/api/mtrain_api.py @@ -6,12 +6,15 @@ import json import uuid -from . import PostgresQueryMixin -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsApi) +from . import PostgresQueryMixin, db_connection_creator from allensdk.brain_observatory.behavior.trials_processing import EDF_COLUMNS -from allensdk.core.auth_config import MTRAIN_DB_CREDENTIAL_MAP +from allensdk.core.auth_config import MTRAIN_DB_CREDENTIAL_MAP, \ + LIMS_DB_CREDENTIAL_MAP from allensdk.core.authentication import credential_injector +from allensdk.brain_observatory.behavior.data_objects \ + import BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata.\ + behavior_metadata.behavior_metadata import BehaviorMetadata class MtrainApi: @@ -59,22 +62,29 @@ def get_session(self, behavior_session_uuid=None, behavior_session_id]), 'must enter either a ' \ 'behavior_session_uuid or a ' \ 'behavior_session_id' - - if behavior_session_uuid is not None and behavior_session_id is not \ - None: - # if both a behavior session uuid and a lims id are entered, - # ensure that they match - behavior_api = BehaviorLimsApi(behavior_session_id) - assert behavior_session_uuid == \ - str(behavior_api.get_metadata().behavior_session_uuid), \ - 'behavior_session {} does not match ' \ - 'behavior_session_id {}'.format(behavior_session_uuid, - behavior_session_id) - if behavior_session_uuid is None and behavior_session_id is not None: - # get a behavior session uuid if a lims ID was entered - behavior_api = BehaviorLimsApi(behavior_session_id) - behavior_session_uuid = str(behavior_api.get_metadata(). - behavior_session_uuid) + if behavior_session_id is not None: + def _get_behavior_metadata(): + lims_db = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP + ) + behavior_session_id_ = BehaviorSessionId( + behavior_session_id=behavior_session_id) + bm = BehaviorMetadata.from_lims( + behavior_session_id=behavior_session_id_, lims_db=lims_db) + return bm + bm = _get_behavior_metadata() + + if behavior_session_uuid is not None: + # if both a behavior session uuid and a lims id are entered, + # ensure that they match + assert behavior_session_uuid == \ + str(bm.behavior_session_uuid), \ + 'behavior_session {} does not match ' \ + 'behavior_session_id {}'.format( + behavior_session_uuid, bm.behavior_session_uuid) + else: + # get a behavior session uuid if a lims ID was entered + behavior_session_uuid = str(bm.behavior_session_uuid) filters = [{"name": "id", "op": "eq", "val": behavior_session_uuid}] behavior_df = self.get_df('behavior_sessions', filters=filters).rename( diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py index a0aa8427c..11763ddef 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py @@ -1,4 +1,11 @@ +from unittest.mock import create_autospec + import pytest + +from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ + BehaviorOphysExperiment +from allensdk.brain_observatory.behavior.behavior_session import \ + BehaviorSession from .utils import create_bucket, load_dataset import boto3 from moto import mock_s3 @@ -58,7 +65,7 @@ def test_manifest_methods(tmpdir, s3_cloud_cache_data): @mock_s3 -def test_local_cache_construction(tmpdir, s3_cloud_cache_data): +def test_local_cache_construction(tmpdir, s3_cloud_cache_data, monkeypatch): data, versions = s3_cloud_cache_data cache_dir = pathlib.Path(tmpdir) / "test_construction" @@ -75,7 +82,12 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data): v_names = [f'{project_name}_manifest_v{i}.json' for i in versions] cache.load_manifest(v_names[0]) - cache.get_behavior_ophys_experiment(ophys_experiment_id=5111) + + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorOphysExperiment, 'from_nwb_path', + lambda path: create_autospec( + BehaviorOphysExperiment, instance=True)) + cache.get_behavior_ophys_experiment(ophys_experiment_id=5111) assert cache.fetch_api.cache._downloaded_data_path.is_file() cache.fetch_api.cache._downloaded_data_path.unlink() assert not cache.fetch_api.cache._downloaded_data_path.is_file() @@ -106,7 +118,7 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data): @mock_s3 -def test_load_out_of_date_manifest(tmpdir, s3_cloud_cache_data): +def test_load_out_of_date_manifest(tmpdir, s3_cloud_cache_data, monkeypatch): """ Test that VisualBehaviorOphysProjectCache can load a manifest other than the latest and download files @@ -129,9 +141,17 @@ def test_load_out_of_date_manifest(tmpdir, s3_cloud_cache_data): v_names = [f'{project_name}_manifest_v{i}.json' for i in versions] cache.load_manifest(v_names[0]) for sess_id in (333, 444): - cache.get_behavior_session(behavior_session_id=sess_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorSession, 'from_nwb_path', + lambda path: create_autospec( + BehaviorSession, instance=True)) + cache.get_behavior_session(behavior_session_id=sess_id) for exp_id in (5111, 5222): - cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorOphysExperiment, 'from_nwb_path', + lambda path: create_autospec( + BehaviorOphysExperiment, instance=True)) + cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) v1_dir = cache_dir / f'{project_name}-{versions[0]}/data' @@ -160,7 +180,7 @@ def test_load_out_of_date_manifest(tmpdir, s3_cloud_cache_data): @mock_s3 @pytest.mark.parametrize("delete_cache", [True, False]) -def test_file_linkage(tmpdir, s3_cloud_cache_data, delete_cache): +def test_file_linkage(tmpdir, s3_cloud_cache_data, delete_cache, monkeypatch): """ Test that symlinks are used where appropriate @@ -193,9 +213,17 @@ def test_file_linkage(tmpdir, s3_cloud_cache_data, delete_cache): assert cache.list_all_downloaded_manifests() == v_names for sess_id in (333, 444): - cache.get_behavior_session(behavior_session_id=sess_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorSession, 'from_nwb_path', + lambda path: create_autospec( + BehaviorSession, instance=True)) + cache.get_behavior_session(behavior_session_id=sess_id) for exp_id in (5111, 5222): - cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorOphysExperiment, 'from_nwb_path', + lambda path: create_autospec( + BehaviorOphysExperiment, instance=True)) + cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) v1_glob = v_dirs[0].glob('*') v1_paths = {} @@ -217,9 +245,17 @@ def test_file_linkage(tmpdir, s3_cloud_cache_data, delete_cache): cache.load_manifest(v_names[-1]) assert cache.current_manifest() == v_names[-1] for sess_id in (777, 888): - cache.get_behavior_session(behavior_session_id=sess_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorSession, 'from_nwb_path', + lambda path: create_autospec( + BehaviorSession, instance=True)) + cache.get_behavior_session(behavior_session_id=sess_id) for exp_id in (5444, 5666, 5777): - cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorOphysExperiment, 'from_nwb_path', + lambda path: create_autospec( + BehaviorOphysExperiment, instance=True)) + cache.get_behavior_ophys_experiment(ophys_experiment_id=exp_id) v2_glob = v_dirs[-1].glob('*') v2_paths = {} diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache_data_model/conftest.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache_data_model/conftest.py index 69d44519b..b9fbd4808 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache_data_model/conftest.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache_data_model/conftest.py @@ -14,14 +14,17 @@ add_passive_flag_to_ophys_experiment_table, add_image_set_to_experiment_table) -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata - from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ .util.prior_exposure_processing import \ get_prior_exposures_to_session_type, \ get_prior_exposures_to_image_set, \ get_prior_exposures_to_omissions +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.full_genotype import \ + FullGenotype +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.reporter_line import \ + ReporterLine @pytest.fixture(scope='session') @@ -417,11 +420,11 @@ def intermediate_behavior_table(behavior_session_table, df = behavior_session_table.copy(deep=True) df['reporter_line'] = df['reporter_line'].apply( - BehaviorMetadata.parse_reporter_line) + ReporterLine.parse) df['cre_line'] = df['full_genotype'].apply( - BehaviorMetadata.parse_cre_line) + lambda x: FullGenotype(full_genotype=x).parse_cre_line()) df['indicator'] = df['reporter_line'].apply( - BehaviorMetadata.parse_indicator) + lambda x: ReporterLine(reporter_line=x).parse_indicator()) df['prior_exposures_to_session_type'] = \ get_prior_exposures_to_session_type(df=df) diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py index ff30c0c4d..1409bf402 100644 --- a/allensdk/test/brain_observatory/behavior/conftest.py +++ b/allensdk/test/brain_observatory/behavior/conftest.py @@ -8,9 +8,64 @@ import pytest import pytz -from allensdk.brain_observatory.behavior.image_api import ImageApi -from allensdk.brain_observatory.behavior.session_apis.data_transforms import \ - BehaviorOphysDataTransforms +from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_metadata import \ + BehaviorMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_session_uuid import \ + BehaviorSessionUUID +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.equipment import \ + Equipment +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.session_type import \ + SessionType +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.stimulus_frame_rate import \ + StimulusFrameRate +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_ophys_metadata import \ + BehaviorOphysMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.experiment_container_id import \ + ExperimentContainerId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.field_of_view_shape import \ + FieldOfViewShape +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.imaging_depth import \ + ImagingDepth +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.imaging_plane import \ + ImagingPlane +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_experiment_metadata import \ + OphysExperimentMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_session_id import \ + OphysSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.age import \ + Age +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.driver_line import \ + DriverLine +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.full_genotype import \ + FullGenotype +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.mouse_id import \ + MouseId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.reporter_line import \ + ReporterLine +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.sex import \ + Sex +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.subject_metadata import \ + SubjectMetadata from allensdk.brain_observatory.behavior.stimulus_processing import \ StimulusTemplateFactory from allensdk.test_utilities.custom_comparators import WhitespaceStrippedString @@ -143,22 +198,28 @@ def stimulus_presentations_behavior(stimulus_templates, @pytest.fixture def behavior_only_metadata_fixture(): """Fixture that provides mock behavior only session metadata""" - return { - "behavior_session_id": 4242, - "session_type": 'Unknown', - "date_of_acquisition": pytz.utc.localize(datetime.datetime.now()), - "reporter_line": "Ai93(TITL-GCaMP6f)", - "driver_line": ["Camk2a-tTA", "Slc17a7-IRES2-Cre"], - "cre_line": "Slc17a7-IRES2-Cre", - "mouse_id": 416369, - "full_genotype": "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;" - "Ai93(TITL-GCaMP6f)/wt", - "behavior_session_uuid": uuid.uuid4(), - "stimulus_frame_rate": 60.0, - "equipment_name": 'my_device', - "sex": 'M', - "age_in_days": 139 - } + subject_meta = SubjectMetadata( + sex=Sex(sex='M'), + age=Age(age=139), + reporter_line=ReporterLine(reporter_line="Ai93(TITL-GCaMP6f)"), + full_genotype=FullGenotype( + full_genotype="Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;" + "Ai93(TITL-GCaMP6f)/wt"), + driver_line=DriverLine( + driver_line=["Camk2a-tTA", "Slc17a7-IRES2-Cre"]), + mouse_id=MouseId(mouse_id=416369) + + ) + behavior_meta = BehaviorMetadata( + subject_metadata=subject_meta, + behavior_session_id=BehaviorSessionId(behavior_session_id=4242), + equipment=Equipment(equipment_name='my_device'), + stimulus_frame_rate=StimulusFrameRate(stimulus_frame_rate=60.0), + session_type=SessionType(session_type='Unknown'), + behavior_session_uuid=BehaviorSessionUUID( + behavior_session_uuid=uuid.uuid4()) + ) + return behavior_meta @pytest.fixture @@ -196,33 +257,25 @@ def metadata_fixture(): @pytest.fixture -def partial_metadata_fixture(): - """Fixture that passes only metadata that will be saved in - custom pyNWB extension fields""" - return { - "behavior_session_id": 777, - "ophys_session_id": 999, - "ophys_experiment_id": 1234, - "experiment_container_id": 5678, - "stimulus_frame_rate": 60.0, - "imaging_depth": 375, - "session_type": 'Unknown', - "date_of_acquisition": pytz.utc.localize(datetime.datetime.now()), - "reporter_line": "Ai93(TITL-GCaMP6f)", - "driver_line": ["Camk2a-tTA", "Slc17a7-IRES2-Cre"], - "cre_line": "Slc17a7-IRES2-Cre", - "mouse_id": 416369, - "full_genotype": "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;" - "Ai93(TITL-GCaMP6f)/wt", - "behavior_session_uuid": uuid.uuid4(), - "field_of_view_width": 4, - "field_of_view_height": 4, - "equipment_name": 'my_device', - "sex": 'M', - "age_in_days": 139, - "imaging_plane_group": None, - "imaging_plane_group_count": 0 - } +def behavior_ophys_metadata_fixture( + behavior_only_metadata_fixture) -> BehaviorOphysMetadata: + ophys_meta = OphysExperimentMetadata( + ophys_experiment_id=1234, + ophys_session_id=OphysSessionId(session_id=999), + experiment_container_id=ExperimentContainerId( + experiment_container_id=5678), + imaging_plane=ImagingPlane( + ophys_frame_rate=31.0, + targeted_structure='VISp', + excitation_lambda=1.0 + ), + field_of_view_shape=FieldOfViewShape(width=4, height=4), + imaging_depth=ImagingDepth(imaging_depth=375) + ) + return BehaviorOphysMetadata( + behavior_metadata=behavior_only_metadata_fixture, + ophys_metadata=ophys_meta + ) @pytest.fixture @@ -409,46 +462,3 @@ def behavior_stimuli_data_fixture(request): data["items"]["behavior"]["stimuli"]["grating"] = grating_data return data - - -@pytest.fixture -def cell_specimen_table_api(): - - roi_1 = np.array([ - [1, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ]) - - roi_2 = np.array([ - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ]) - - # Must implement at least the get_cell_specimen_table - # and get_max_projection methods from BehaviorOphysBase - class CellSpecimenTableApi(BehaviorOphysDataTransforms): - - def __init__(self): - pass - - def get_cell_specimen_table(self): - return pd.DataFrame( - { - "cell_roi_id": [1, 2], - "y": [1, 1], - "x": [2, 1], - "roi_mask": [roi_1, roi_2] - }, index=pd.Index(data=[10, 11], name="cell_specimen_id") - ) - - def get_max_projection(self): - return ImageApi.serialize(roi_1 + roi_2, - [0.5, 1.], 'mm') - - return CellSpecimenTableApi() diff --git a/allensdk/test/brain_observatory/behavior/data_files/test_stimulus_file.py b/allensdk/test/brain_observatory/behavior/data_files/test_stimulus_file.py new file mode 100644 index 000000000..7fded55c6 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_files/test_stimulus_file.py @@ -0,0 +1,82 @@ +from typing import Tuple +from pathlib import Path +import pickle +from unittest.mock import create_autospec + +import pytest + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_files.stimulus_file import ( + STIMULUS_FILE_QUERY_TEMPLATE +) + + +@pytest.fixture +def stimulus_file_fixture(request, tmp_path) -> Tuple[Path, dict]: + default_stim_pkl_data = {"a": 1, "b": 2, "c": 3} + stim_pkl_data = request.param.get("pkl_data", default_stim_pkl_data) + stim_pkl_filename = request.param.get("filename", "test_stimulus_file.pkl") + + stim_pkl_path = tmp_path / stim_pkl_filename + with stim_pkl_path.open('wb') as f: + pickle.dump(stim_pkl_data, f) + + return (stim_pkl_path, stim_pkl_data) + + +@pytest.mark.parametrize("stimulus_file_fixture", [ + ({"pkl_data": {"a": 42, "b": 7}}), + ({"pkl_data": {"slightly_more_complex": [1, 2, 3, 4]}}) +], indirect=["stimulus_file_fixture"]) +def test_stimulus_file_from_json(stimulus_file_fixture): + stim_pkl_path, stim_pkl_data = stimulus_file_fixture + + # Basic test case + input_json_dict = {"behavior_stimulus_file": str(stim_pkl_path)} + stimulus_file = StimulusFile.from_json(input_json_dict) + assert stimulus_file.data == stim_pkl_data + + # Now test caching by deleting the stimulus_file + stim_pkl_path.unlink() + stimulus_file_cached = StimulusFile.from_json(input_json_dict) + assert stimulus_file_cached.data == stim_pkl_data + + +@pytest.mark.parametrize("stimulus_file_fixture, behavior_session_id", [ + ({"pkl_data": {"a": 42, "b": 7}}, 12), + ({"pkl_data": {"slightly_more_complex": [1, 2, 3, 4]}}, 8) +], indirect=["stimulus_file_fixture"]) +def test_stimulus_file_from_lims(stimulus_file_fixture, behavior_session_id): + stim_pkl_path, stim_pkl_data = stimulus_file_fixture + + mock_db_conn = create_autospec(PostgresQueryMixin, instance=True) + + # Basic test case + mock_db_conn.fetchone.return_value = str(stim_pkl_path) + stimulus_file = StimulusFile.from_lims(mock_db_conn, behavior_session_id) + assert stimulus_file.data == stim_pkl_data + + # Now test caching by deleting stimulus_file and also asserting db + # `fetchone` called only once + stim_pkl_path.unlink() + stimfile_cached = StimulusFile.from_lims(mock_db_conn, behavior_session_id) + assert stimfile_cached.data == stim_pkl_data + + query = STIMULUS_FILE_QUERY_TEMPLATE.format( + behavior_session_id=behavior_session_id + ) + + mock_db_conn.fetchone.assert_called_once_with(query, strict=True) + + +@pytest.mark.parametrize("stimulus_file_fixture", [ + ({"filename": "test_stim_file_1.pkl"}), + ({"filename": "mock_stim_pkl_2.pkl"}) +], indirect=["stimulus_file_fixture"]) +def test_stimulus_file_to_json(stimulus_file_fixture): + stim_pkl_path, stim_pkl_data = stimulus_file_fixture + + stimulus_file = StimulusFile(filepath=stim_pkl_path) + obt_json = stimulus_file.to_json() + assert obt_json == {"behavior_stimulus_file": str(stim_pkl_path)} diff --git a/allensdk/test/brain_observatory/behavior/data_files/test_sync_file.py b/allensdk/test/brain_observatory/behavior/data_files/test_sync_file.py new file mode 100644 index 000000000..81b5631df --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_files/test_sync_file.py @@ -0,0 +1,112 @@ +from typing import Tuple +from pathlib import Path +import h5py +from unittest.mock import create_autospec +import numpy as np + +import pytest + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_files.sync_file import ( + SYNC_FILE_QUERY_TEMPLATE +) + + +@pytest.fixture +def sync_file_fixture(request, tmp_path) -> Tuple[Path, dict]: + default_sync_data = [1, 2, 3, 4, 5] + sync_data = request.param.get("sync_data", default_sync_data) + sync_filename = request.param.get("filename", "test_sync_file.h5") + + sync_path = tmp_path / sync_filename + with h5py.File(sync_path, "w") as f: + f.create_dataset("data", data=sync_data) + + return (sync_path, sync_data) + + +def mock_get_sync_data(sync_path): + with h5py.File(sync_path, "r") as f: + data = f["data"][:] + return data + + +@pytest.mark.parametrize("sync_file_fixture", [ + ({"sync_data": [2, 3, 4, 5]}), +], indirect=["sync_file_fixture"]) +def test_sync_file_from_json(monkeypatch, sync_file_fixture): + sync_path, sync_data = sync_file_fixture + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_files" + ".sync_file.get_sync_data", + mock_get_sync_data + ) + + # Basic test case + input_json_dict = {"sync_file": str(sync_path)} + sync_file = SyncFile.from_json(input_json_dict) + assert np.allclose(sync_file.data, sync_data) + + # Now test caching by deleting the sync_file + sync_path.unlink() + sync_file_cached = SyncFile.from_json(input_json_dict) + assert np.allclose(sync_file_cached.data, sync_data) + + +@pytest.mark.parametrize("sync_file_fixture, ophys_experiment_id", [ + ({"sync_data": [2, 3, 4, 5]}, 12), + ({"sync_data": [2, 3, 4, 5]}, 8) +], indirect=["sync_file_fixture"]) +def test_sync_file_from_lims( + monkeypatch, + sync_file_fixture, + ophys_experiment_id +): + sync_path, sync_data = sync_file_fixture + + mock_db_conn = create_autospec(PostgresQueryMixin, instance=True) + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_files" + ".sync_file.get_sync_data", + mock_get_sync_data + ) + + # Basic test case + mock_db_conn.fetchone.return_value = str(sync_path) + sync_file = SyncFile.from_lims(mock_db_conn, ophys_experiment_id) + np.allclose(sync_file.data, sync_data) + + # Now test caching by deleting sync_file and also asserting db + # `fetchone` called only once + sync_path.unlink() + stimfile_cached = SyncFile.from_lims(mock_db_conn, ophys_experiment_id) + np.allclose(stimfile_cached.data, sync_data) + + query = SYNC_FILE_QUERY_TEMPLATE.format( + ophys_experiment_id=ophys_experiment_id + ) + + mock_db_conn.fetchone.assert_called_once_with(query, strict=True) + + +@pytest.mark.parametrize("sync_file_fixture", [ + ({"filename": "test_sync_file_1.h5"}), + ({"filename": "mock_sync_file_2.h5"}) +], indirect=["sync_file_fixture"]) +def test_sync_file_to_json(monkeypatch, sync_file_fixture): + sync_path, sync_data = sync_file_fixture + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_files" + ".sync_file.get_sync_data", + mock_get_sync_data + ) + sync_file = SyncFile(filepath=sync_path) + obt_json = sync_file.to_json() + assert obt_json == {"sync_file": str(sync_path)} diff --git a/allensdk/test/brain_observatory/behavior/data_objects/base/test_data_object.py b/allensdk/test/brain_observatory/behavior/data_objects/base/test_data_object.py new file mode 100644 index 000000000..1ebdc7953 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/base/test_data_object.py @@ -0,0 +1,81 @@ +import pytest + +from allensdk.brain_observatory.behavior.data_objects import DataObject + + +class TestDataObject: + def test_to_dict_simple(self): + class Simple(DataObject): + def __init__(self): + super().__init__(name='simple', value=1) + s = Simple() + assert s.to_dict() == {'simple': 1} + + def test_to_dict_nested(self): + class B(DataObject): + def __init__(self): + super().__init__(name='b', value='!') + + class A(DataObject): + def __init__(self, b: B): + super().__init__(name='a', value=self) + self._b = b + + @property + def prop1(self): + return self._b + + @property + def prop2(self): + return '@' + a = A(b=B()) + assert a.to_dict() == {'a': {'b': '!', 'prop2': '@'}} + + def test_to_dict_double_nested(self): + class C(DataObject): + def __init__(self): + super().__init__(name='c', value='!!!') + + class B(DataObject): + def __init__(self, c: C): + super().__init__(name='b', value=self) + self._c = c + + @property + def prop1(self): + return self._c + + @property + def prop2(self): + return '!!' + + class A(DataObject): + def __init__(self, b: B): + super().__init__(name='a', value=self) + self._b = b + + @property + def prop1(self): + return self._b + + @property + def prop2(self): + return '@' + + a = A(b=B(c=C())) + assert a.to_dict() == {'a': {'b': {'c': '!!!', 'prop2': '!!'}, + 'prop2': '@'}} + + def test_not_equals(self): + s1 = DataObject(name='s1', value=1) + s2 = DataObject(name='s1', value='1') + assert s1 != s2 + + def test_exclude_equals(self): + s1 = DataObject(name='s1', value=1, exclude_from_equals={'s1'}) + s2 = DataObject(name='s1', value='1') + assert s1 == s2 + + def test_cannot_compare(self): + with pytest.raises(NotImplementedError): + assert DataObject(name='foo', value=1) == 1 diff --git a/allensdk/test/brain_observatory/behavior/data_objects/conftest.py b/allensdk/test/brain_observatory/behavior/data_objects/conftest.py new file mode 100644 index 000000000..33a068bf7 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/conftest.py @@ -0,0 +1,24 @@ +import pynwb +import pytest + + +@pytest.fixture +def data_object_roundtrip_fixture(tmp_path): + def f(nwbfile, data_object_cls, **data_object_cls_kwargs): + tmp_dir = tmp_path / "data_object_nwb_roundtrip_tests" + tmp_dir.mkdir() + nwb_path = tmp_dir / "data_object_roundtrip_nwbfile.nwb" + + with pynwb.NWBHDF5IO(str(nwb_path), 'w') as write_io: + write_io.write(nwbfile) + + with pynwb.NWBHDF5IO(str(nwb_path), 'r') as read_io: + roundtripped_nwbfile = read_io.read() + + data_object_instance = data_object_cls.from_nwb( + roundtripped_nwbfile, **data_object_cls_kwargs + ) + + return data_object_instance + + return f diff --git a/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_eye_tracking_table.py b/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_eye_tracking_table.py new file mode 100644 index 000000000..6c54fba93 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_eye_tracking_table.py @@ -0,0 +1,85 @@ +from datetime import datetime +from pathlib import Path + +import numpy as np +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import \ + SyncFile +from allensdk.brain_observatory.behavior.data_files.eye_tracking_file import \ + EyeTrackingFile +from allensdk.brain_observatory.behavior.data_objects.eye_tracking \ + .eye_tracking_table import \ + EyeTrackingTable +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest +from allensdk.test.brain_observatory.behavior.test_eye_tracking_processing \ + import \ + create_refined_eye_tracking_df + + +class TestFromDataFile(LimsTest): + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + + df = pd.read_pickle(str(test_data_dir / 'eye_tracking_table.pkl')) + cls.expected = EyeTrackingTable(eye_tracking=df) + + @pytest.mark.requires_bamboo + def test_from_data_file(self): + etf = EyeTrackingFile.from_lims( + ophys_experiment_id=self.ophys_experiment_id, db=self.dbconn) + sync_file = SyncFile.from_lims( + ophys_experiment_id=self.ophys_experiment_id, db=self.dbconn) + ett = EyeTrackingTable.from_data_file(data_file=etf, + sync_file=sync_file) + + # filter to first 100 values for testing + ett = EyeTrackingTable(eye_tracking=ett.value.iloc[:100]) + assert ett == self.expected + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.parent.resolve() + cls.test_data_dir = dir / 'test_data' + + df = create_refined_eye_tracking_df( + np.array([[0.1, 12 * np.pi, 72 * np.pi, 196 * np.pi, False, + 196 * np.pi, 12 * np.pi, 72 * np.pi, + 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., + 13., 14., 15.], + [0.2, 20 * np.pi, 90 * np.pi, 225 * np.pi, False, + 225 * np.pi, 20 * np.pi, 90 * np.pi, + 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., + 14., 15., 16.]]) + ) + cls.eye_tracking_table = EyeTrackingTable(eye_tracking=df) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.eye_tracking_table.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=EyeTrackingTable) + else: + obt = EyeTrackingTable.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.eye_tracking_table diff --git a/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_rig_geometry.py b/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_rig_geometry.py new file mode 100644 index 000000000..f04db161a --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/eye_tracking/test_rig_geometry.py @@ -0,0 +1,127 @@ +import json +import pandas as pd + +from datetime import datetime +from pathlib import Path + +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_objects.eye_tracking \ + .rig_geometry import \ + RigGeometry, Coordinates +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromLims(LimsTest): + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + + with open(test_data_dir / 'eye_tracking_rig_geometry.json') as f: + x = json.load(f) + x = x['rig_geometry'] + x = {'eye_tracking_rig_geometry': x} + cls.expected = RigGeometry.from_json(dict_repr=x) + + @pytest.mark.requires_bamboo + def test_from_lims(self): + rg = RigGeometry.from_lims( + ophys_experiment_id=self.ophys_experiment_id, lims_db=self.dbconn) + assert rg == self.expected + + @pytest.mark.requires_bamboo + def test_rig_geometry_newer_than_experiment(self): + """ + This test ensures that if the experiment date_of_acquisition + is before a rig activate_date that it is not returned as the rig + used for the experiment + """ + # This experiment has rig config more recent than the + # experiment date_of_acquisition + ophys_experiment_id = 521405260 + + rg = RigGeometry.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=self.dbconn) + expected = RigGeometry( + camera_position_mm=Coordinates(x=130.0, y=0.0, z=0.0), + led_position=Coordinates(x=265.1, y=-39.3, z=1.0), + monitor_position_mm=Coordinates(x=170.0, y=0.0, z=0.0), + camera_rotation_deg=Coordinates(x=0.0, y=0.0, z=13.1), + monitor_rotation_deg=Coordinates(x=0.0, y=0.0, z=0.0), + equipment='CAM2P.1' + ) + assert rg == expected + + def test_only_single_geometry_returned(self): + """Tests that when a rig contains multiple geometries, that only 1 is + returned""" + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + + # This example contains multiple geometries per config + df = pd.read_pickle( + str(test_data_dir / 'raw_eye_tracking_rig_geometry.pkl')) + + obtained = RigGeometry._select_most_recent_geometry(rig_geometry=df) + assert (obtained.groupby(obtained.index).size() == 1).all() + + +class TestFromJson(LimsTest): + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + + with open(test_data_dir / 'eye_tracking_rig_geometry.json') as f: + x = json.load(f) + x = x['rig_geometry'] + x = {'eye_tracking_rig_geometry': x} + cls.expected = RigGeometry.from_json(dict_repr=x) + + @pytest.mark.requires_bamboo + def test_from_json(self): + dict_repr = {'eye_tracking_rig_geometry': + self.expected.to_dict()['rig_geometry']} + rg = RigGeometry.from_json(dict_repr=dict_repr) + assert rg == self.expected + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.parent.resolve() + cls.test_data_dir = dir / 'test_data' + + with open(cls.test_data_dir / 'eye_tracking_rig_geometry.json') as f: + x = json.load(f) + x = x['rig_geometry'] + x = {'eye_tracking_rig_geometry': x} + cls.rig_geometry = RigGeometry.from_json(dict_repr=x) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.rig_geometry.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=RigGeometry) + else: + obt = RigGeometry.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.rig_geometry diff --git a/allensdk/test/brain_observatory/behavior/data_objects/lims_util.py b/allensdk/test/brain_observatory/behavior/data_objects/lims_util.py new file mode 100644 index 000000000..0af3615f0 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/lims_util.py @@ -0,0 +1,16 @@ +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator + + +class LimsTest: + """Helper class for testing LIMS. For each test, checks whether + bamboo is required and if so sets up a connection""" + def setup_method(self, method): + marks = getattr(method, 'pytestmark', None) + if marks: + marks = [m.name for m in marks] + + # Will only create a dbconn if the test requires_bamboo + if 'requires_bamboo' in marks: + self.dbconn = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py b/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py new file mode 100644 index 000000000..a4ecc0b45 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py @@ -0,0 +1,307 @@ +import datetime +import pickle +import uuid +from pathlib import Path + +import pynwb +import pytest +import pytz +from uuid import UUID + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.behavior_metadata import \ + BehaviorMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.behavior_session_uuid import \ + BehaviorSessionUUID +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.date_of_acquisition import \ + DateOfAcquisition +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.equipment import \ + Equipment +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.session_type import \ + SessionType +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.stimulus_frame_rate import \ + StimulusFrameRate +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.age import \ + Age +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.driver_line import \ + DriverLine +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.full_genotype import \ + FullGenotype +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.mouse_id import \ + MouseId +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.reporter_line import \ + ReporterLine +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.sex import \ + Sex +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .subject_metadata.subject_metadata import \ + SubjectMetadata +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class BehaviorMetaTestCase: + @classmethod + def setup_class(cls): + cls.meta = cls._get_meta() + + @staticmethod + def _get_meta(): + subject_meta = SubjectMetadata( + sex=Sex(sex='M'), + age=Age(age=139), + reporter_line=ReporterLine(reporter_line="Ai93(TITL-GCaMP6f)"), + full_genotype=FullGenotype( + full_genotype="Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;" + "Ai93(TITL-GCaMP6f)/wt"), + driver_line=DriverLine( + driver_line=["Camk2a-tTA", "Slc17a7-IRES2-Cre"]), + mouse_id=MouseId(mouse_id=416369) + + ) + behavior_meta = BehaviorMetadata( + subject_metadata=subject_meta, + behavior_session_id=BehaviorSessionId(behavior_session_id=4242), + equipment=Equipment(equipment_name='my_device'), + stimulus_frame_rate=StimulusFrameRate(stimulus_frame_rate=60.0), + session_type=SessionType(session_type='Unknown'), + behavior_session_uuid=BehaviorSessionUUID( + behavior_session_uuid=uuid.uuid4()) + ) + return behavior_meta + + +class TestLims(LimsTest): + @pytest.mark.requires_bamboo + def test_behavior_session_uuid(self): + behavior_session_id = 823847007 + meta = BehaviorMetadata.from_lims( + behavior_session_id=BehaviorSessionId( + behavior_session_id=behavior_session_id), + lims_db=self.dbconn + ) + assert meta.behavior_session_uuid == \ + uuid.UUID('394a910e-94c7-4472-9838-5345aff59ed8') + + +class TestBehaviorMetadata(BehaviorMetaTestCase): + def test_cre_line(self): + """Tests that cre_line properly parsed from driver_line""" + fg = FullGenotype( + full_genotype='Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt') + assert fg.parse_cre_line() == 'Sst-IRES-Cre' + + def test_cre_line_bad_full_genotype(self): + """Test that cre_line is None and no error raised""" + fg = FullGenotype(full_genotype='foo') + + with pytest.warns(UserWarning) as record: + cre_line = fg.parse_cre_line(warn=True) + assert cre_line is None + assert str(record[0].message) == 'Unable to parse cre_line from ' \ + 'full_genotype' + + def test_reporter_line(self): + """Test that reporter line properly parsed from list""" + reporter_line = ReporterLine.parse(reporter_line=['foo']) + assert reporter_line == 'foo' + + def test_reporter_line_str(self): + """Test that reporter line returns itself if str""" + reporter_line = ReporterLine.parse(reporter_line='foo') + assert reporter_line == 'foo' + + @pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( + (('foo', 'bar'), 'More than 1 reporter line. ' + 'Returning the first one', 'foo'), + (None, 'Error parsing reporter line. It is null.', None), + ([], 'Error parsing reporter line. The array is empty', None) + ) + ) + def test_reporter_edge_cases(self, input_reporter_line, warning_msg, + expected): + """Test reporter line edge cases""" + with pytest.warns(UserWarning) as record: + reporter_line = ReporterLine.parse( + reporter_line=input_reporter_line, + warn=True) + assert reporter_line == expected + assert str(record[0].message) == warning_msg + + def test_age_in_days(self): + """Test that age_in_days properly parsed from age""" + age = Age._age_code_to_days(age='P123') + assert age == 123 + + @pytest.mark.parametrize("input_age, warning_msg, expected", ( + ('unkown', 'Could not parse numeric age from age code ' + '(age code does not start with "P")', None), + ('P', 'Could not parse numeric age from age code ' + '(no numeric values found in age code)', None) + ) + ) + def test_age_in_days_edge_cases(self, monkeypatch, input_age, warning_msg, + expected): + """Test age in days edge cases""" + with pytest.warns(UserWarning) as record: + age_in_days = Age._age_code_to_days(age=input_age, warn=True) + + assert age_in_days is None + assert str(record[0].message) == warning_msg + + @pytest.mark.parametrize("test_params, expected_warn_msg", [ + # Vanilla test case + ({ + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", + "%Y-%m-%d %H:%M:%S"), + "pkl_expt_date": datetime.datetime.strptime("2021-03-14 03:14:15", + "%Y-%m-%d %H:%M:%S"), + "behavior_session_id": 1 + }, None), + + # pkl expt date stored in unix format + ({ + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", + "%Y-%m-%d %H:%M:%S"), + "pkl_expt_date": 1615716855.0, + "behavior_session_id": 2 + }, None), + + # Extractor and pkl dates differ significantly + ({ + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", + "%Y-%m-%d %H:%M:%S"), + "pkl_expt_date": datetime.datetime.strptime("2021-03-14 20:14:15", + "%Y-%m-%d %H:%M:%S"), + "behavior_session_id": 3 + }, + "The `date_of_acquisition` field in LIMS *"), + + # pkl file contains an unparseable datetime + ({ + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", + "%Y-%m-%d %H:%M:%S"), + "pkl_expt_date": None, + "behavior_session_id": 4 + }, + "Could not parse the acquisition datetime *"), + ]) + def test_get_date_of_acquisition(self, tmp_path, test_params, + expected_warn_msg): + mock_session_id = test_params["behavior_session_id"] + + pkl_save_path = tmp_path / f"mock_pkl_{mock_session_id}.pkl" + with open(pkl_save_path, 'wb') as handle: + pickle.dump({"start_time": test_params['pkl_expt_date']}, handle) + + tz = pytz.timezone("America/Los_Angeles") + extractor_expt_date = tz.localize( + test_params['extractor_expt_date']).astimezone(pytz.utc) + + stimulus_file = StimulusFile(filepath=pkl_save_path) + obt_date = DateOfAcquisition( + date_of_acquisition=extractor_expt_date) + + if expected_warn_msg: + with pytest.warns(Warning, match=expected_warn_msg): + obt_date.validate( + stimulus_file=stimulus_file, + behavior_session_id=test_params['behavior_session_id']) + + assert obt_date.value == extractor_expt_date + + def test_indicator(self): + """Test that indicator is parsed from full_genotype""" + reporter_line = ReporterLine( + reporter_line='Ai148(TIT2L-GC6f-ICL-tTA2)') + assert reporter_line.parse_indicator() == 'GCaMP6f' + + @pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( + (None, + 'Could not parse indicator from reporter because there is no ' + 'reporter', None), + ('foo', 'Could not parse indicator from reporter because none' + 'of the expected substrings were found in the reporter', + None) + ) + ) + def test_indicator_edge_cases(self, input_reporter_line, warning_msg, + expected): + """Test indicator parsing edge cases""" + with pytest.warns(UserWarning) as record: + reporter_line = ReporterLine(reporter_line=input_reporter_line) + indicator = reporter_line.parse_indicator(warn=True) + assert indicator is expected + assert str(record[0].message) == warning_msg + + +class TestStimulusFile: + """Tests properties read from stimulus file""" + def setup_class(cls): + dir = Path(__file__).parent.parent.parent.resolve() + test_data_dir = dir / 'test_data' + sf_path = test_data_dir / 'stimulus_file.pkl' + cls.stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(sf_path)}) + + def test_session_uuid(self): + uuid = BehaviorSessionUUID.from_stimulus_file( + stimulus_file=self.stimulus_file) + expected = UUID('138531ab-fe59-4523-9154-07c8d97bbe03') + assert expected == uuid.value + + def test_get_stimulus_frame_rate(self): + rate = StimulusFrameRate.from_stimulus_file( + stimulus_file=self.stimulus_file) + assert 62.0 == rate.value + + +def test_date_of_acquisition_utc(): + """Tests that when read from json (in Pacific time), that + date of acquisition is converted to utc""" + expected = DateOfAcquisition( + date_of_acquisition=datetime.datetime(2019, 9, 26, 16, + tzinfo=pytz.UTC)) + actual = DateOfAcquisition.from_json( + dict_repr={'date_of_acquisition': '2019-09-26 09:00:00'}) + assert expected == actual + + +class TestNWB(BehaviorMetaTestCase): + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='afile', + session_start_time=datetime.datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_add_behavior_only_metadata(self, roundtrip, + data_object_roundtrip_fixture): + self.meta.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + meta_obt = data_object_roundtrip_fixture( + self.nwbfile, BehaviorMetadata + ) + else: + meta_obt = BehaviorMetadata.from_nwb(nwbfile=self.nwbfile) + + assert self.meta == meta_obt diff --git a/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py b/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py new file mode 100644 index 000000000..7783b98fd --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py @@ -0,0 +1,198 @@ +import datetime +import json +from pathlib import Path +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.equipment import \ + Equipment +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_ophys_metadata import \ + BehaviorOphysMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.experiment_container_id import \ + ExperimentContainerId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.field_of_view_shape import \ + FieldOfViewShape +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.imaging_depth import \ + ImagingDepth +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata\ + .imaging_plane_group import \ + ImagingPlaneGroup +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata\ + .multi_plane_metadata import \ + MultiplaneMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_experiment_metadata import \ + OphysExperimentMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.ophys_session_id import \ + OphysSessionId +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator +from allensdk.test.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.test_behavior_metadata import \ + TestBehaviorMetadata + + +class TestBOM: + @classmethod + def setup_class(cls): + cls.meta = cls._get_meta() + + def setup_method(self, method): + self.meta = self._get_meta() + + @staticmethod + def _get_meta(): + ophys_meta = OphysExperimentMetadata( + ophys_experiment_id=1234, + ophys_session_id=OphysSessionId(session_id=999), + experiment_container_id=ExperimentContainerId( + experiment_container_id=5678), + field_of_view_shape=FieldOfViewShape(width=4, height=4), + imaging_depth=ImagingDepth(imaging_depth=375) + ) + + behavior_metadata = TestBehaviorMetadata() + behavior_metadata.setup_class() + return BehaviorOphysMetadata( + behavior_metadata=behavior_metadata.meta, + ophys_metadata=ophys_meta + ) + + def _get_multiplane_meta(self): + bo_meta = self.meta + bo_meta.behavior_metadata._equipment = \ + Equipment(equipment_name='MESO.1') + ophys_experiment_metadata = bo_meta.ophys_metadata + + imaging_plane_group = ImagingPlaneGroup(plane_group_count=5, + plane_group=0) + multiplane_meta = MultiplaneMetadata( + ophys_experiment_id=ophys_experiment_metadata.ophys_experiment_id, + ophys_session_id=ophys_experiment_metadata._ophys_session_id, + experiment_container_id=ophys_experiment_metadata._experiment_container_id, # noqa E501 + field_of_view_shape=ophys_experiment_metadata._field_of_view_shape, + imaging_depth=ophys_experiment_metadata._imaging_depth, + project_code=ophys_experiment_metadata._project_code, + imaging_plane_group=imaging_plane_group + ) + return BehaviorOphysMetadata( + behavior_metadata=bo_meta.behavior_metadata, + ophys_metadata=multiplane_meta + ) + + +class TestInternal(TestBOM): + @classmethod + def setup_method(self, method): + marks = getattr(method, 'pytestmark', None) + if marks: + marks = [m.name for m in marks] + + # Will only create a dbconn if the test requires_bamboo + if 'requires_bamboo' in marks: + self.dbconn = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + + @pytest.mark.requires_bamboo + @pytest.mark.parametrize('meso', [True, False]) + def test_from_lims(self, meso): + if meso: + ophys_experiment_id = 951980471 + else: + ophys_experiment_id = 994278291 + bom = BehaviorOphysMetadata.from_lims( + ophys_experiment_id=ophys_experiment_id, lims_db=self.dbconn, + is_multiplane=meso) + + if meso: + assert isinstance(bom.ophys_metadata, + MultiplaneMetadata) + assert bom.ophys_metadata.imaging_depth == 150 + assert bom.behavior_metadata.session_type == 'OPHYS_1_images_A' + assert bom.behavior_metadata.subject_metadata.reporter_line == \ + 'Ai148(TIT2L-GC6f-ICL-tTA2)' + assert bom.behavior_metadata.subject_metadata.driver_line == \ + ['Sst-IRES-Cre'] + assert bom.behavior_metadata.subject_metadata.mouse_id == 457841 + assert bom.behavior_metadata.subject_metadata.full_genotype == \ + 'Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt' + assert bom.behavior_metadata.subject_metadata.age_in_days == 233 + assert bom.behavior_metadata.subject_metadata.sex == 'F' + else: + assert isinstance(bom.ophys_metadata, OphysExperimentMetadata) + assert bom.ophys_metadata.imaging_depth == 175 + assert bom.behavior_metadata.session_type == 'OPHYS_4_images_A' + assert bom.behavior_metadata.subject_metadata.reporter_line == \ + 'Ai93(TITL-GCaMP6f)' + assert bom.behavior_metadata.subject_metadata.driver_line == \ + ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'] + assert bom.behavior_metadata.subject_metadata.mouse_id == 491060 + assert bom.behavior_metadata.subject_metadata.full_genotype == \ + 'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt' + assert bom.behavior_metadata.subject_metadata.age_in_days == 130 + assert bom.behavior_metadata.subject_metadata.sex == 'M' + + +class TestJson(TestBOM): + @classmethod + def setup_method(self, method): + dir = Path(__file__).parent.resolve() + test_data_dir = dir.parent / 'test_data' + with open(test_data_dir / 'test_input.json') as f: + dict_repr = json.load(f) + dict_repr = dict_repr['session_data'] + dict_repr['sync_file'] = str(test_data_dir / 'sync.h5') + dict_repr['behavior_stimulus_file'] = str(test_data_dir / + 'behavior_stimulus_file.pkl') + dict_repr['dff_file'] = str(test_data_dir / 'demix_file.h5') + self.dict_repr = dict_repr + + @pytest.mark.parametrize('meso', [True, False]) + def test_from_json(self, meso): + if meso: + self.dict_repr['rig_name'] = 'MESO.1' + bom = BehaviorOphysMetadata.from_json(dict_repr=self.dict_repr, + is_multiplane=meso) + + if meso: + assert isinstance(bom.ophys_metadata, MultiplaneMetadata) + else: + assert isinstance(bom.ophys_metadata, OphysExperimentMetadata) + + +class TestNWB(TestBOM): + def setup_method(self, method): + self.meta = self._get_meta() + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier=str(self.meta.ophys_metadata.ophys_experiment_id), + session_start_time=datetime.datetime.now() + ) + + @pytest.mark.parametrize('meso', [True, False]) + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture, meso): + if meso: + self.meta = self._get_multiplane_meta() + + self.meta.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=BehaviorOphysMetadata, + is_multiplane=meso) + else: + obt = self.meta.from_nwb(nwbfile=self.nwbfile, + is_multiplane=meso) + + assert obt == self.meta diff --git a/allensdk/test/brain_observatory/behavior/data_objects/nwb_input_json.py b/allensdk/test/brain_observatory/behavior/data_objects/nwb_input_json.py new file mode 100644 index 000000000..ba5058f3d --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/nwb_input_json.py @@ -0,0 +1,23 @@ +import json +from pathlib import Path + + +class NwbInputJson: + def __init__(self): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + with open(test_data_dir / 'test_input.json') as f: + dict_repr = json.load(f) + dict_repr = dict_repr['session_data'] + dict_repr['sync_file'] = str(test_data_dir / 'sync.h5') + dict_repr['behavior_stimulus_file'] = str(test_data_dir / + 'behavior_stimulus_file.pkl') + dict_repr['dff_file'] = str(test_data_dir / 'demix_file.h5') + dict_repr['demix_file'] = str(test_data_dir / 'demix_file.h5') + dict_repr['events_file'] = str(test_data_dir / 'events.h5') + + self._dict_repr = dict_repr + + @property + def dict_repr(self): + return self._dict_repr diff --git a/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_acquisition.py b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_acquisition.py new file mode 100644 index 000000000..fc93f1b9f --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_acquisition.py @@ -0,0 +1,312 @@ +import pytest +from unittest.mock import create_autospec + +import pandas as pd + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing import ( # noqa: E501 + get_running_df +) +from allensdk.brain_observatory.behavior.data_objects import ( + RunningAcquisition, StimulusTimestamps +) + + +@pytest.mark.parametrize( + "dict_repr, returned_running_acq_df, expected_running_acq_df", + [ + ( + # dict_repr + { + "behavior_stimulus_file": "mock_stimulus_file.pkl" + }, + # returned_running_acq_df + pd.DataFrame( + { + "timestamps": [1, 2], + "speed": [3, 4], + "dx": [5, 6], + "v_sig": [7, 8], + "v_in": [9, 10] + } + ).set_index("timestamps"), + # expected_running_acq_df + pd.DataFrame( + { + "timestamps": [1, 2], + "dx": [5, 6], + "v_sig": [7, 8], + "v_in": [9, 10] + } + ).set_index("timestamps") + ), + ] +) +def test_running_acquisition_from_json( + monkeypatch, dict_repr, returned_running_acq_df, expected_running_acq_df +): + mock_stimulus_file = create_autospec(StimulusFile) + mock_stimulus_timestamps = create_autospec(StimulusTimestamps) + mock_get_running_df = create_autospec(get_running_df) + + mock_get_running_df.return_value = returned_running_acq_df + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.StimulusTimestamps", + mock_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.get_running_df", + mock_get_running_df + ) + obt = RunningAcquisition.from_json(dict_repr) + + mock_stimulus_file.from_json.assert_called_once_with(dict_repr) + mock_stimulus_file_instance = mock_stimulus_file.from_json(dict_repr) + assert obt._stimulus_file == mock_stimulus_file_instance + + mock_stimulus_timestamps.from_json.assert_called_once_with(dict_repr) + mock_stimulus_timestamps_instance = mock_stimulus_timestamps.from_json( + dict_repr + ) + assert obt._stimulus_timestamps == mock_stimulus_timestamps_instance + + mock_get_running_df.assert_called_once_with( + data=mock_stimulus_file_instance.data, + time=mock_stimulus_timestamps_instance.value, + ) + + pd.testing.assert_frame_equal(obt.value, expected_running_acq_df) + + +@pytest.mark.parametrize( + "stimulus_file, stimulus_file_to_json_ret, " + "stimulus_timestamps, stimulus_timestamps_to_json_ret, raises, expected", + [ + # Test to_json with both stimulus_file and sync_file + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # stimulus_timestamps + create_autospec(StimulusTimestamps, instance=True), + # stimulus_timestamps_to_json_ret + {"sync_file": "sync.h5"}, + # raises + False, + # expected + {"behavior_stimulus_file": "stim.pkl", "sync_file": "sync.h5"} + ), + # Test to_json without stimulus_file + ( + # stimulus_file + None, + # stimulus_file_to_json_ret + None, + # stimulus_timestamps + create_autospec(StimulusTimestamps, instance=True), + # stimulus_timestamps_to_json_ret + {"sync_file": "sync.h5"}, + # raises + "RunningAcquisition DataObject lacks information about", + # expected + None + ), + # Test to_json without stimulus_timestamps + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # stimulus_timestamps_to_json_ret + None, + # sync_file_to_json_ret + None, + # raises + "RunningAcquisition DataObject lacks information about", + # expected + None + ), + ] +) +def test_running_acquisition_to_json( + stimulus_file, stimulus_file_to_json_ret, + stimulus_timestamps, stimulus_timestamps_to_json_ret, raises, expected +): + if stimulus_file is not None: + stimulus_file.to_json.return_value = stimulus_file_to_json_ret + if stimulus_timestamps is not None: + stimulus_timestamps.to_json.return_value = ( + stimulus_timestamps_to_json_ret + ) + + running_acq = RunningAcquisition( + running_acquisition=None, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps + ) + + if raises: + with pytest.raises(RuntimeError, match=raises): + _ = running_acq.to_json() + else: + obt = running_acq.to_json() + assert obt == expected + + +@pytest.mark.parametrize( + "behavior_session_id, ophys_experiment_id, " + "returned_running_acq_df, expected_running_acq_df", + [ + ( + # behavior_session_id + 12345, + # ophys_experiment_id + None, + # returned_running_acq_df + pd.DataFrame( + { + "timestamps": [1, 2], + "speed": [3, 4], + "dx": [5, 6], + "v_sig": [7, 8], + "v_in": [9, 10] + } + ).set_index("timestamps"), + # expected_running_acq_df + pd.DataFrame( + { + "timestamps": [1, 2], + "dx": [5, 6], + "v_sig": [7, 8], + "v_in": [9, 10] + } + ).set_index("timestamps") + ), + ( + # behavior_session_id + 1234, + # ophys_experiment_id + 5678, + # returned_running_acq_df + pd.DataFrame( + { + "timestamps": [2, 4], + "speed": [6, 8], + "dx": [10, 12], + "v_sig": [14, 16], + "v_in": [18, 20] + } + ).set_index("timestamps"), + # expected_running_acq_df + pd.DataFrame( + { + "timestamps": [2, 4], + "dx": [10, 12], + "v_sig": [14, 16], + "v_in": [18, 20] + } + ).set_index("timestamps") + ) + ] +) +def test_running_acquisition_from_lims( + monkeypatch, behavior_session_id, ophys_experiment_id, + returned_running_acq_df, expected_running_acq_df +): + mock_db_conn = create_autospec(PostgresQueryMixin, instance=True) + + mock_stimulus_file = create_autospec(StimulusFile) + mock_stimulus_timestamps = create_autospec(StimulusTimestamps) + mock_get_running_df = create_autospec(get_running_df) + + mock_get_running_df.return_value = returned_running_acq_df + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.StimulusTimestamps", + mock_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_acquisition.get_running_df", + mock_get_running_df + ) + obt = RunningAcquisition.from_lims( + mock_db_conn, behavior_session_id, ophys_experiment_id + ) + + mock_stimulus_file.from_lims.assert_called_once_with( + mock_db_conn, behavior_session_id + ) + mock_stimulus_file_instance = mock_stimulus_file.from_lims( + mock_db_conn, behavior_session_id + ) + assert obt._stimulus_file == mock_stimulus_file_instance + + mock_stimulus_timestamps.from_stimulus_file.assert_called_once_with( + mock_stimulus_file_instance + ) + mock_stimulus_timestamps_instance = mock_stimulus_timestamps.\ + from_stimulus_file(stimulus_file=mock_stimulus_file) + assert obt._stimulus_timestamps == mock_stimulus_timestamps_instance + + mock_get_running_df.assert_called_once_with( + data=mock_stimulus_file_instance.data, + time=mock_stimulus_timestamps_instance.value, + ) + + pd.testing.assert_frame_equal( + obt.value, expected_running_acq_df, check_like=True + ) + + +# Fixtures: +# nwbfile: +# test/brain_observatory/behavior/conftest.py +# data_object_roundtrip_fixture: +# test/brain_observatory/behavior/data_objects/conftest.py +@pytest.mark.parametrize("roundtrip", [True, False]) +@pytest.mark.parametrize("running_acq_data", [ + ( + # expected_running_acq_df + pd.DataFrame( + { + "timestamps": [2.0, 4.0], + "dx": [10.0, 12.0], + "v_sig": [14.0, 16.0], + "v_in": [18.0, 20.0] + } + ).set_index("timestamps") + ), +]) +def test_running_acquisition_nwb_roundtrip( + nwbfile, data_object_roundtrip_fixture, roundtrip, running_acq_data +): + running_acq = RunningAcquisition(running_acquisition=running_acq_data) + nwbfile = running_acq.to_nwb(nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture(nwbfile, RunningAcquisition) + else: + obt = RunningAcquisition.from_nwb(nwbfile) + + pd.testing.assert_frame_equal( + obt.value, running_acq_data, check_like=True + ) diff --git a/allensdk/test/brain_observatory/behavior/test_running_processing.py b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_processing.py similarity index 95% rename from allensdk/test/brain_observatory/behavior/test_running_processing.py rename to allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_processing.py index 3b2e3712f..b06b263d7 100644 --- a/allensdk/test/brain_observatory/behavior/test_running_processing.py +++ b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_processing.py @@ -1,13 +1,12 @@ import numpy as np -import pandas as pd import pytest -from allensdk.brain_observatory.behavior.running_processing import ( +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing import ( # noqa: E501 get_running_df, calc_deriv, deg_to_dist, _shift, _identify_wraps, _unwrap_voltage_signal, _angular_change, _zscore_threshold_1d, _clip_speed_wraps) -import allensdk.brain_observatory.behavior.running_processing as rp +import allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing as rp # noqa: E501 @pytest.fixture @@ -110,8 +109,14 @@ def test_get_running_df_one_fewer_timestamp_check_truncation(running_data, # Check that the output is actually trimmed, and the values are the same assert len(output) == len(timestamps) - 1 - np.testing.assert_equal(output["v_sig"], running_data["items"]["behavior"]["encoders"][0]["vsig"][:-1]) - np.testing.assert_equal(output["v_in"], running_data["items"]["behavior"]["encoders"][0]["vin"][:-1]) + np.testing.assert_equal( + output["v_sig"], + running_data["items"]["behavior"]["encoders"][0]["vsig"][:-1] + ) + np.testing.assert_equal( + output["v_in"], + running_data["items"]["behavior"]["encoders"][0]["vin"][:-1] + ) @pytest.mark.parametrize( diff --git a/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_speed.py b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_speed.py new file mode 100644 index 000000000..9e3bd110f --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/running_speed/test_running_speed.py @@ -0,0 +1,349 @@ +import pytest +from unittest.mock import create_autospec + +import pandas as pd + +from allensdk.core.exceptions import DataFrameIndexError +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects.running_speed.running_processing import ( # noqa: E501 + get_running_df +) +from allensdk.brain_observatory.behavior.data_objects import ( + RunningSpeed, StimulusTimestamps +) + + +@pytest.mark.parametrize("filtered", [True, False]) +@pytest.mark.parametrize("zscore_threshold", [1.0, 4.2]) +@pytest.mark.parametrize("returned_running_df, expected_running_df, raises", [ + # Test basic case + ( + # returned_running_df + pd.DataFrame({ + "timestamps": [2, 4, 6, 8], + "speed": [1, 2, 3, 4] + }).set_index("timestamps"), + # expected_running_df + pd.DataFrame({ + "timestamps": [2, 4, 6, 8], + "speed": [1, 2, 3, 4] + }), + # raises + False + ), + # Test when returned dataframe lacks "timestamps" as index + ( + # returned_running_df + pd.DataFrame({ + "timestamps": [2, 4, 6, 8], + "speed": [1, 2, 3, 4] + }).set_index("speed"), + # expected_running_df + None, + # raises + "Expected running_data_df index to be named 'timestamps'" + ), +]) +def test_get_running_speed_df( + monkeypatch, returned_running_df, filtered, zscore_threshold, + expected_running_df, raises +): + + mock_stimulus_file_instance = create_autospec(StimulusFile, instance=True) + mock_stimulus_timestamps_instance = create_autospec( + StimulusTimestamps, instance=True + ) + mock_get_running_speed_df = create_autospec(get_running_df) + mock_get_running_speed_df.return_value = returned_running_df + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.get_running_df", + mock_get_running_speed_df + ) + + if raises: + with pytest.raises(DataFrameIndexError, match=raises): + _ = RunningSpeed._get_running_speed_df( + mock_stimulus_file_instance, + mock_stimulus_timestamps_instance, + filtered, zscore_threshold + ) + else: + obt = RunningSpeed._get_running_speed_df( + mock_stimulus_file_instance, + mock_stimulus_timestamps_instance, + filtered, zscore_threshold + ) + + pd.testing.assert_frame_equal(obt, expected_running_df) + + mock_get_running_speed_df.assert_called_once_with( + data=mock_stimulus_file_instance.data, + time=mock_stimulus_timestamps_instance.value, + lowpass=filtered, + zscore_threshold=zscore_threshold + ) + + +@pytest.mark.parametrize("filtered", [True, False]) +@pytest.mark.parametrize("zscore_threshold", [1.0, 4.2]) +@pytest.mark.parametrize( + "dict_repr, returned_running_df, expected_running_df", + [ + ( + # dict_repr + { + "behavior_stimulus_file": "mock_stimulus_file.pkl" + }, + # returned_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ).set_index("timestamps"), + # expected_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ), + ), + ] +) +def test_running_speed_from_json( + monkeypatch, dict_repr, returned_running_df, expected_running_df, + filtered, zscore_threshold +): + mock_stimulus_file = create_autospec(StimulusFile) + mock_stimulus_timestamps = create_autospec(StimulusTimestamps) + mock_get_running_speed_df = create_autospec(get_running_df) + + mock_get_running_speed_df.return_value = returned_running_df + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.StimulusTimestamps", + mock_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.get_running_df", + mock_get_running_speed_df + ) + obt = RunningSpeed.from_json(dict_repr, filtered, zscore_threshold) + + mock_stimulus_file.from_json.assert_called_once_with(dict_repr) + mock_stimulus_file_instance = mock_stimulus_file.from_json(dict_repr) + assert obt._stimulus_file == mock_stimulus_file_instance + + mock_stimulus_timestamps.from_json.assert_called_once_with(dict_repr) + mock_stimulus_timestamps_instance = mock_stimulus_timestamps.from_json( + dict_repr + ) + assert obt._stimulus_timestamps == mock_stimulus_timestamps_instance + + mock_get_running_speed_df.assert_called_once_with( + data=mock_stimulus_file_instance.data, + time=mock_stimulus_timestamps_instance.value, + lowpass=filtered, + zscore_threshold=zscore_threshold + ) + + assert obt._filtered == filtered + pd.testing.assert_frame_equal(obt.value, expected_running_df) + + +@pytest.mark.parametrize( + "stimulus_file, stimulus_file_to_json_ret, " + "stimulus_timestamps, stimulus_timestamps_to_json_ret, raises, expected", + [ + # Test to_json with both stimulus_file and sync_file + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # stimulus_timestamps + create_autospec(StimulusTimestamps, instance=True), + # stimulus_timestamps_to_json_ret + {"sync_file": "sync.h5"}, + # raises + False, + # expected + {"behavior_stimulus_file": "stim.pkl", "sync_file": "sync.h5"} + ), + # Test to_json without stimulus_file + ( + # stimulus_file + None, + # stimulus_file_to_json_ret + None, + # stimulus_timestamps + create_autospec(StimulusTimestamps, instance=True), + # stimulus_timestamps_to_json_ret + {"sync_file": "sync.h5"}, + # raises + "RunningSpeed DataObject lacks information about", + # expected + None + ), + # Test to_json without stimulus_timestamps + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # stimulus_timestamps_to_json_ret + None, + # sync_file_to_json_ret + None, + # raises + "RunningSpeed DataObject lacks information about", + # expected + None + ), + ] +) +def test_running_speed_to_json( + stimulus_file, stimulus_file_to_json_ret, + stimulus_timestamps, stimulus_timestamps_to_json_ret, raises, expected +): + if stimulus_file is not None: + stimulus_file.to_json.return_value = stimulus_file_to_json_ret + if stimulus_timestamps is not None: + stimulus_timestamps.to_json.return_value = ( + stimulus_timestamps_to_json_ret + ) + + running_speed = RunningSpeed( + running_speed=None, + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps + ) + + if raises: + with pytest.raises(RuntimeError, match=raises): + _ = running_speed.to_json() + else: + obt = running_speed.to_json() + assert obt == expected + + +@pytest.mark.parametrize("behavior_session_id", [12345, 1234]) +@pytest.mark.parametrize("filtered", [True, False]) +@pytest.mark.parametrize("zscore_threshold", [1.0, 4.2]) +@pytest.mark.parametrize( + "returned_running_df, expected_running_df", + [ + ( + # returned_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ).set_index("timestamps"), + # expected_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ), + ), + ( + # returned_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ).set_index("timestamps"), + # expected_running_df + pd.DataFrame( + {"timestamps": [1, 2], "speed": [3, 4]} + ), + ) + ] +) +def test_running_speed_from_lims( + monkeypatch, behavior_session_id, returned_running_df, + expected_running_df, filtered, zscore_threshold +): + mock_db_conn = create_autospec(PostgresQueryMixin, instance=True) + + mock_stimulus_file = create_autospec(StimulusFile) + mock_stimulus_timestamps = create_autospec(StimulusTimestamps) + mock_get_running_speed_df = create_autospec(get_running_df) + mock_get_running_speed_df.return_value = returned_running_df + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.StimulusTimestamps", + mock_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".running_speed.running_speed.get_running_df", + mock_get_running_speed_df + ) + obt = RunningSpeed.from_lims( + mock_db_conn, behavior_session_id, filtered, + zscore_threshold + ) + + mock_stimulus_file.from_lims.assert_called_once_with( + mock_db_conn, behavior_session_id + ) + mock_stimulus_file_instance = mock_stimulus_file.from_lims( + mock_db_conn, behavior_session_id + ) + assert obt._stimulus_file == mock_stimulus_file_instance + + mock_stimulus_timestamps.from_stimulus_file.assert_called_once_with( + mock_stimulus_file_instance + ) + mock_stimulus_timestamps_instance = mock_stimulus_timestamps.\ + from_stimulus_file(stimulus_file=mock_stimulus_file) + assert obt._stimulus_timestamps == mock_stimulus_timestamps_instance + + mock_get_running_speed_df.assert_called_once_with( + data=mock_stimulus_file_instance.data, + time=mock_stimulus_timestamps_instance.value, + lowpass=filtered, + zscore_threshold=zscore_threshold + ) + + assert obt._filtered == filtered + pd.testing.assert_frame_equal(obt.value, expected_running_df) + + +# Fixtures: +# nwbfile: +# test/brain_observatory/behavior/conftest.py +# data_object_roundtrip_fixture: +# test/brain_observatory/behavior/data_objects/conftest.py +@pytest.mark.parametrize("roundtrip", [True, False]) +@pytest.mark.parametrize("filtered", [True, False]) +@pytest.mark.parametrize("running_speed_data", [ + (pd.DataFrame({"timestamps": [3.0, 4.0], "speed": [5.0, 6.0]})), +]) +def test_running_speed_nwb_roundtrip( + nwbfile, data_object_roundtrip_fixture, roundtrip, running_speed_data, + filtered +): + running_speed = RunningSpeed( + running_speed=running_speed_data, filtered=filtered + ) + nwbfile = running_speed.to_nwb(nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile, RunningSpeed, filtered=filtered + ) + else: + obt = RunningSpeed.from_nwb(nwbfile, filtered=filtered) + + pd.testing.assert_frame_equal(obt.value, running_speed_data) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_stimulus_timestamps.py b/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_stimulus_timestamps.py new file mode 100644 index 000000000..9eb01fdaf --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_stimulus_timestamps.py @@ -0,0 +1,312 @@ +from pathlib import Path + +import pytest +from unittest.mock import create_autospec + +import numpy as np + +from allensdk.internal.api import PostgresQueryMixin +from allensdk.brain_observatory.behavior.data_files import ( + StimulusFile, SyncFile +) +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .stimulus_timestamps.timestamps_processing import ( + get_behavior_stimulus_timestamps, get_ophys_stimulus_timestamps) +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps + + +@pytest.mark.parametrize("dict_repr, has_pkl, has_sync", [ + # Test where input json only has "behavior_stimulus_file" + ( + # dict_repr + { + "behavior_stimulus_file": "mock_stimulus_file.pkl" + }, + # has_pkl + True, + # has_sync + False + ), + # Test where input json has both "behavior_stimulus_file" and "sync_file" + ( + # dict_repr + { + "behavior_stimulus_file": "mock_stimulus_file.pkl", + "sync_file": "mock_sync_file.h5" + }, + # has_pkl + True, + # has_sync + True + ), +]) +def test_stimulus_timestamps_from_json( + monkeypatch, dict_repr, has_pkl, has_sync +): + mock_stimulus_file = create_autospec(StimulusFile) + mock_sync_file = create_autospec(SyncFile) + + mock_get_behavior_stimulus_timestamps = create_autospec( + get_behavior_stimulus_timestamps + ) + mock_get_ophys_stimulus_timestamps = create_autospec( + get_ophys_stimulus_timestamps + ) + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps.SyncFile", + mock_sync_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps" + ".get_behavior_stimulus_timestamps", + mock_get_behavior_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps" + ".get_ophys_stimulus_timestamps", + mock_get_ophys_stimulus_timestamps + ) + mock_stimulus_file_instance = mock_stimulus_file.from_json(dict_repr) + ts_from_stim = StimulusTimestamps.from_stimulus_file( + stimulus_file=mock_stimulus_file_instance) + + if has_pkl and has_sync: + mock_sync_file_instance = mock_sync_file.from_json(dict_repr) + ts_from_sync = StimulusTimestamps.from_sync_file( + sync_file=mock_sync_file_instance) + + if has_pkl and has_sync: + mock_get_ophys_stimulus_timestamps.assert_called_once_with( + sync_path=mock_sync_file_instance.filepath + ) + assert ts_from_sync._sync_file == mock_sync_file_instance + else: + assert ts_from_stim._stimulus_file == mock_stimulus_file_instance + mock_get_behavior_stimulus_timestamps.assert_called_once_with( + stimulus_pkl=mock_stimulus_file_instance.data + ) + + +def test_stimulus_timestamps_from_json2(): + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + sf_path = test_data_dir / 'stimulus_file.pkl' + + sf = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(sf_path)}) + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=sf) + expected = np.array([0.016 * i for i in range(11)]) + assert np.allclose(expected, stimulus_timestamps.value) + + +def test_stimulus_timestamps_from_json3(): + """ + Test that StimulusTimestamps.from_stimulus_file + just returns the sum of the intervalsms field in the + behavior stimulus pickle file, padded with a zero at the + first timestamp. + """ + dir = Path(__file__).parent.parent.resolve() + test_data_dir = dir / 'test_data' + sf_path = test_data_dir / 'stimulus_file.pkl' + + sf = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(sf_path)}) + + sf._data['items']['behavior']['intervalsms'] = [0.1, 0.2, 0.3, 0.4] + + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=sf) + + expected = np.array([0., 0.0001, 0.0003, 0.0006, 0.001]) + np.testing.assert_array_almost_equal(stimulus_timestamps.value, + expected, + decimal=10) + + +@pytest.mark.parametrize( + "stimulus_file, stimulus_file_to_json_ret, " + "sync_file, sync_file_to_json_ret, raises, expected", + [ + # Test to_json with both stimulus_file and sync_file + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # sync_file + create_autospec(SyncFile, instance=True), + # sync_file_to_json_ret + {"sync_file": "sync.h5"}, + # raises + False, + # expected + {"behavior_stimulus_file": "stim.pkl", "sync_file": "sync.h5"} + ), + # Test to_json with only stimulus_file + ( + # stimulus_file + create_autospec(StimulusFile, instance=True), + # stimulus_file_to_json_ret + {"behavior_stimulus_file": "stim.pkl"}, + # sync_file + None, + # sync_file_to_json_ret + None, + # raises + False, + # expected + {"behavior_stimulus_file": "stim.pkl"} + ), + # Test to_json without stimulus_file nor sync_file + ( + # stimulus_file + None, + # stimulus_file_to_json_ret + None, + # sync_file + None, + # sync_file_to_json_ret + None, + # raises + "StimulusTimestamps DataObject lacks information about", + # expected + None + ), + ] +) +def test_stimulus_timestamps_to_json( + stimulus_file, stimulus_file_to_json_ret, + sync_file, sync_file_to_json_ret, raises, expected +): + if stimulus_file is not None: + stimulus_file.to_json.return_value = stimulus_file_to_json_ret + if sync_file is not None: + sync_file.to_json.return_value = sync_file_to_json_ret + + stimulus_timestamps = StimulusTimestamps( + timestamps=None, + stimulus_file=stimulus_file, + sync_file=sync_file + ) + + if raises: + with pytest.raises(RuntimeError, match=raises): + _ = stimulus_timestamps.to_json() + else: + obt = stimulus_timestamps.to_json() + assert obt == expected + + +@pytest.mark.parametrize("behavior_session_id, ophys_experiment_id", [ + ( + 12345, + None + ), + ( + 1234, + 5678 + ) +]) +def test_stimulus_timestamps_from_lims( + monkeypatch, behavior_session_id, ophys_experiment_id +): + mock_db_conn = create_autospec(PostgresQueryMixin, instance=True) + + mock_stimulus_file = create_autospec(StimulusFile) + mock_sync_file = create_autospec(SyncFile) + + mock_get_behavior_stimulus_timestamps = create_autospec( + get_behavior_stimulus_timestamps + ) + mock_get_ophys_stimulus_timestamps = create_autospec( + get_ophys_stimulus_timestamps + ) + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps.StimulusFile", + mock_stimulus_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps.SyncFile", + mock_sync_file + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps" + ".get_behavior_stimulus_timestamps", + mock_get_behavior_stimulus_timestamps + ) + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.stimulus_timestamps" + ".get_ophys_stimulus_timestamps", + mock_get_ophys_stimulus_timestamps + ) + mock_stimulus_file_instance = mock_stimulus_file.from_lims( + mock_db_conn, behavior_session_id + ) + ts_from_stim = StimulusTimestamps.from_stimulus_file( + stimulus_file=mock_stimulus_file_instance) + assert ts_from_stim._stimulus_file == mock_stimulus_file_instance + + if behavior_session_id is not None and ophys_experiment_id is not None: + mock_sync_file_instance = mock_sync_file.from_lims( + mock_db_conn, ophys_experiment_id + ) + ts_from_sync = StimulusTimestamps.from_sync_file( + sync_file=mock_sync_file_instance) + + if behavior_session_id is not None and ophys_experiment_id is not None: + mock_get_ophys_stimulus_timestamps.assert_called_once_with( + sync_path=mock_sync_file_instance.filepath + ) + assert ts_from_sync._sync_file == mock_sync_file_instance + else: + mock_stimulus_file.from_lims.assert_called_with( + mock_db_conn, behavior_session_id + ) + mock_get_behavior_stimulus_timestamps.assert_called_once_with( + stimulus_pkl=mock_stimulus_file_instance.data + ) + + +# Fixtures: +# nwbfile: +# test/brain_observatory/behavior/conftest.py +# data_object_roundtrip_fixture: +# test/brain_observatory/behavior/data_objects/conftest.py +@pytest.mark.parametrize('roundtrip, stimulus_timestamps_data', [ + (True, np.array([1, 2, 3, 4, 5])), + (True, np.array([6, 7, 8, 9, 10])), + (False, np.array([11, 12, 13, 14, 15])), + (False, np.array([16, 17, 18, 19, 20])) +]) +def test_stimulus_timestamps_nwb_roundtrip( + nwbfile, data_object_roundtrip_fixture, roundtrip, stimulus_timestamps_data +): + stimulus_timestamps = StimulusTimestamps( + timestamps=stimulus_timestamps_data + ) + nwbfile = stimulus_timestamps.to_nwb(nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture(nwbfile, StimulusTimestamps) + else: + obt = StimulusTimestamps.from_nwb(nwbfile) + + assert np.allclose(obt.value, stimulus_timestamps_data) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_timestamps_processing.py b/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_timestamps_processing.py new file mode 100644 index 000000000..4e6d53dca --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/stimulus_timestamps/test_timestamps_processing.py @@ -0,0 +1,80 @@ +from unittest.mock import create_autospec, PropertyMock + +import numpy as np +import pytest + +from allensdk.brain_observatory.behavior.data_objects.timestamps \ + .stimulus_timestamps.timestamps_processing import ( + get_behavior_stimulus_timestamps, get_ophys_stimulus_timestamps) +from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner + + +@pytest.mark.parametrize("pkl_data, expected", [ + # Extremely basic test case + ( + # pkl_data + { + "items": { + "behavior": { + "intervalsms": np.array([ + 1000, 1001, 1002, 1003, 1004, 1005 + ]) + } + } + }, + # expected + np.array([ + 0.0, 1.0, 2.001, 3.003, 4.006, 5.01, 6.015 + ]) + ), + # More realistic test case + ( + # pkl_data + { + "items": { + "behavior": { + "intervalsms": np.array([ + 16.5429, 16.6685, 16.66580001, 16.70569999, + 16.6668, + 16.69619999, 16.655, 16.6805, 16.75940001, 16.6831 + ]) + } + } + }, + # expected + np.array([ + 0.0, 0.0165429, 0.0332114, 0.0498772, 0.0665829, 0.0832497, + 0.0999459, 0.1166009, 0.1332814, 0.1500408, 0.1667239 + ]) + ) +]) +def test_get_behavior_stimulus_timestamps(pkl_data, expected): + obt = get_behavior_stimulus_timestamps(pkl_data) + assert np.allclose(obt, expected) + + +@pytest.mark.parametrize("sync_path, expected_timestamps", [ + ("/tmp/mock_sync_file.h5", [1, 2, 3]), +]) +def test_get_ophys_stimulus_timestamps( + monkeypatch, sync_path, expected_timestamps +): + mock_ophys_time_aligner = create_autospec(OphysTimeAligner) + mock_aligner_instance = mock_ophys_time_aligner.return_value + property_mock = PropertyMock( + return_value=(expected_timestamps, "ignored_return_val") + ) + type(mock_aligner_instance).clipped_stim_timestamps = property_mock + + with monkeypatch.context() as m: + m.setattr( + "allensdk.brain_observatory.behavior.data_objects" + ".timestamps.stimulus_timestamps.timestamps_processing" + ".OphysTimeAligner", + mock_ophys_time_aligner + ) + obt = get_ophys_stimulus_timestamps(sync_path) + + mock_ophys_time_aligner.assert_called_with(sync_file=sync_path) + property_mock.assert_called_once() + assert np.allclose(obt, expected_timestamps) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_cell_specimens.py b/allensdk/test/brain_observatory/behavior/data_objects/test_cell_specimens.py new file mode 100644 index 000000000..41da57a8b --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_cell_specimens.py @@ -0,0 +1,273 @@ +import json +from datetime import datetime +from pathlib import Path +import numpy as np +import pynwb +import pandas as pd + +import pytest + +from allensdk.brain_observatory.behavior.data_objects import DataObject +from allensdk.brain_observatory.behavior.data_objects.cell_specimens.\ + cell_specimens import CellSpecimens, CellSpecimenMeta +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .rois_mixin import \ + RoisMixin +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.imaging_plane import \ + ImagingPlane +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .ophys_timestamps import \ + OphysTimestamps +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator +from allensdk.test.brain_observatory.behavior.data_objects.metadata\ + .test_behavior_ophys_metadata import \ + TestBOM + + +class TestLims: + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + cls.expected_meta = CellSpecimenMeta( + emission_lambda=520.0, + imaging_plane=ImagingPlane( + excitation_lambda=910.0, + indicator='GCaMP6f', + ophys_frame_rate=10.0, + targeted_structure='VISp' + ) + ) + + def setup_method(self, method): + marks = getattr(method, 'pytestmark', None) + if marks: + marks = [m.name for m in marks] + + # Will only create a dbconn if the test requires_bamboo + if 'requires_bamboo' in marks: + self.dbconn = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + + @pytest.mark.requires_bamboo + def test_from_lims(self): + number_of_frames = 140296 + ots = OphysTimestamps(timestamps=np.linspace(start=.1, + stop=.1*number_of_frames, + num=number_of_frames)) + csp = CellSpecimens.from_lims( + ophys_experiment_id=self.ophys_experiment_id, lims_db=self.dbconn, + ophys_timestamps=ots, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + assert not csp.table.empty + assert not csp.events.empty + assert not csp.dff_traces.empty + assert not csp.corrected_fluorescence_traces.empty + assert csp.meta == self.expected_meta + + +class TestJson: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + with open(test_data_dir / 'test_input.json') as f: + dict_repr = json.load(f) + dict_repr = dict_repr['session_data'] + dict_repr['sync_file'] = str(test_data_dir / 'sync.h5') + dict_repr['behavior_stimulus_file'] = str(test_data_dir / + 'behavior_stimulus_file.pkl') + dict_repr['dff_file'] = str(test_data_dir / 'demix_file.h5') + dict_repr['demix_file'] = str(test_data_dir / 'demix_file.h5') + dict_repr['events_file'] = str(test_data_dir / 'events.h5') + + cls.dict_repr = dict_repr + cls.expected_meta = CellSpecimenMeta( + emission_lambda=520.0, + imaging_plane=ImagingPlane( + excitation_lambda=910.0, + indicator='GCaMP6f', + ophys_frame_rate=10.0, + targeted_structure='VISp' + ) + ) + cls.ophys_timestamps = OphysTimestamps( + timestamps=np.array([.1, .2, .3])) + + def test_from_json(self): + csp = CellSpecimens.from_json( + dict_repr=self.dict_repr, + ophys_timestamps=self.ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + assert not csp.table.empty + assert not csp.events.empty + assert not csp.dff_traces.empty + assert not csp.corrected_fluorescence_traces.empty + assert csp.meta == self.expected_meta + + @pytest.mark.parametrize('data', + ('dff_traces', + 'corrected_fluorescence_traces', + 'events')) + def test_roi_data_same_order_as_cell_specimen_table(self, data): + """tests that roi data are in same order as cell specimen table""" + csp = CellSpecimens.from_json( + dict_repr=self.dict_repr, + ophys_timestamps=self.ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + private_attr = getattr(csp, f'_{data}') + public_attr = getattr(csp, data) + + # Events stores cell_roi_id as column whereas traces is index + data_cell_roi_ids = getattr( + private_attr.value, + 'cell_roi_id' if data == 'events' else 'index').values + + current_order = np.where(data_cell_roi_ids == + csp._cell_specimen_table['cell_roi_id'])[0] + + # make sure same order + private_attr._value = private_attr.value\ + .iloc[current_order] + + # rearrange + private_attr._value = private_attr._value.iloc[[1, 0]] + + # make sure same order + np.testing.assert_array_equal(public_attr.index, csp.table.index) + + @pytest.mark.parametrize('extra_in_trace', (True, False)) + @pytest.mark.parametrize('trace_type', + ('dff_traces', + 'corrected_fluorescence_traces')) + def test_trace_rois_different_than_cell_specimen_table(self, trace_type, + extra_in_trace): + """check that an exception is raised if there is a mismatch in rois + between cell specimen table and traces""" + csp = CellSpecimens.from_json( + dict_repr=self.dict_repr, + ophys_timestamps=self.ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + private_trace_attr = getattr(csp, f'_{trace_type}') + + if extra_in_trace: + # Drop an roi from cell specimen table that is in trace + trace_rois = private_trace_attr.value.index + csp._cell_specimen_table = csp._cell_specimen_table[ + csp._cell_specimen_table['cell_roi_id'] != trace_rois[0]] + else: + # Drop an roi from trace that is in cell specimen table + csp_rois = csp._cell_specimen_table['cell_roi_id'] + private_trace_attr._value = private_trace_attr._value[ + private_trace_attr._value.index != csp_rois.iloc[0]] + + if trace_type == 'dff_traces': + trace_args = { + 'dff_traces': private_trace_attr, + 'corrected_fluorescence_traces': + csp._corrected_fluorescence_traces + } + else: + trace_args = { + 'dff_traces': csp._dff_traces, + 'corrected_fluorescence_traces': private_trace_attr + } + with pytest.raises(RuntimeError): + # construct it again using trace/table combo with different rois + CellSpecimens( + cell_specimen_table=csp._cell_specimen_table, + meta=csp._meta, + events=csp._events, + ophys_timestamps=self.ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3), + exclude_invalid_rois=False, + **trace_args + ) + + +class TestNWB: + @classmethod + def setup_class(cls): + cls.ophys_timestamps = OphysTimestamps( + timestamps=np.array([.1, .2, .3])) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + tj = TestJson() + tj.setup_class() + self.dict_repr = tj.dict_repr + + # Write metadata, since csp requires other metdata + tbom = TestBOM() + tbom.setup_class() + bom = tbom.meta + bom.to_nwb(nwbfile=self.nwbfile) + + @pytest.mark.parametrize('exclude_invalid_rois', [True, False]) + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture, + exclude_invalid_rois): + cell_specimens = CellSpecimens.from_json( + dict_repr=self.dict_repr, ophys_timestamps=self.ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3), + exclude_invalid_rois=exclude_invalid_rois) + + csp = cell_specimens._cell_specimen_table + + valid_roi_id = csp[csp['valid_roi']]['cell_roi_id'] + + cell_specimens.to_nwb(nwbfile=self.nwbfile, + ophys_timestamps=self.ophys_timestamps) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=CellSpecimens, + exclude_invalid_rois=exclude_invalid_rois, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + else: + obt = cell_specimens.from_nwb( + nwbfile=self.nwbfile, + exclude_invalid_rois=exclude_invalid_rois, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + + if exclude_invalid_rois: + cell_specimens._cell_specimen_table = \ + cell_specimens._cell_specimen_table[ + cell_specimens._cell_specimen_table['cell_roi_id'] + .isin(valid_roi_id)] + + assert obt == cell_specimens + + +class TestFilterAndReorder: + @pytest.mark.parametrize('raise_if_rois_missing', (True, False)) + def test_missing_rois(self, raise_if_rois_missing): + """Tests that when dataframe missing rois, that they are ignored""" + roi_ids = np.array([1, 2]) + df = pd.DataFrame({'cell_roi_id': [1], 'foo': [2]}) + + class Rois(DataObject, RoisMixin): + def __init__(self): + super().__init__(name='test', value=df) + + rois = Rois() + if raise_if_rois_missing: + with pytest.raises(RuntimeError): + rois.filter_and_reorder( + roi_ids=roi_ids, + raise_if_rois_missing=raise_if_rois_missing) + else: + rois.filter_and_reorder( + roi_ids=roi_ids, + raise_if_rois_missing=raise_if_rois_missing) + expected = pd.DataFrame({'cell_roi_id': [1], + 'foo': [2]}) + pd.testing.assert_frame_equal(rois._value, expected) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/avg_projection.png b/allensdk/test/brain_observatory/behavior/data_objects/test_data/avg_projection.png new file mode 100644 index 000000000..e6d627dc9 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/avg_projection.png differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/behavior_stimulus_file.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/behavior_stimulus_file.pkl new file mode 100644 index 000000000..20d6103b2 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/behavior_stimulus_file.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/demix_file.h5 b/allensdk/test/brain_observatory/behavior/data_objects/test_data/demix_file.h5 new file mode 100644 index 000000000..7ed6ecf0b Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/demix_file.h5 differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/events.h5 b/allensdk/test/brain_observatory/behavior/data_objects/test_data/events.h5 new file mode 100644 index 000000000..6b2e461e7 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/events.h5 differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_rig_geometry.json b/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_rig_geometry.json new file mode 100644 index 000000000..ea748f3bb --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_rig_geometry.json @@ -0,0 +1 @@ +{"rig_geometry": {"camera_position_mm": [102.8, 74.7, 31.6], "camera_rotation_deg": [0.0, 0.0, 2.8], "equipment": "CAM2P.3", "led_position": [246.0, 92.3, 52.6], "monitor_position_mm": [118.6, 86.2, 31.6], "monitor_rotation_deg": [0.0, 0.0, 0.0]}} \ No newline at end of file diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_table.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_table.pkl new file mode 100644 index 000000000..4814d9450 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/eye_tracking_table.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/licks.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/licks.pkl new file mode 100644 index 000000000..91cd01df9 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/licks.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/max_projection.png b/allensdk/test/brain_observatory/behavior/data_objects/test_data/max_projection.png new file mode 100644 index 000000000..142d20fe3 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/max_projection.png differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl new file mode 100644 index 000000000..a432f1b89 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/raw_eye_tracking_rig_geometry.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/raw_eye_tracking_rig_geometry.pkl new file mode 100644 index 000000000..2f14c2acc Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/raw_eye_tracking_rig_geometry.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/rewards.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/rewards.pkl new file mode 100644 index 000000000..1b1876db5 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/rewards.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/rigid_motion_transform_file.csv b/allensdk/test/brain_observatory/behavior/data_objects/test_data/rigid_motion_transform_file.csv new file mode 100644 index 000000000..98647b066 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_data/rigid_motion_transform_file.csv @@ -0,0 +1,4 @@ +,x,y +0,2,-3 +1,3,-4 +2,2,-4 diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/stimulus_file.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/stimulus_file.pkl new file mode 100644 index 000000000..e2757a53c Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/stimulus_file.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/sync.h5 b/allensdk/test/brain_observatory/behavior/data_objects/test_data/sync.h5 new file mode 100644 index 000000000..af448dd04 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/sync.h5 differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/task_parameters.json b/allensdk/test/brain_observatory/behavior/data_objects/test_data/task_parameters.json new file mode 100644 index 000000000..cd7caa9ed --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_data/task_parameters.json @@ -0,0 +1,13 @@ +{ + "auto_reward_volume": 0.005, + "blank_duration_sec": [0.5, 0.5], + "n_stimulus_frames": 69195, + "omitted_flash_fraction": 0.05, + "response_window_sec": [0.15, 0.75], + "reward_volume": 0.007, + "session_type": "OPHYS_4_images_A", + "stimulus": "images", + "stimulus_distribution": "geometric", + "stimulus_duration_sec": 0.25, + "task_type": "change detection" +} \ No newline at end of file diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/templates.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/templates.pkl new file mode 100644 index 000000000..d52342cf4 Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/templates.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/test_input.json b/allensdk/test/brain_observatory/behavior/data_objects/test_data/test_input.json new file mode 100755 index 000000000..c9de2233a --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_data/test_input.json @@ -0,0 +1,144 @@ +{ + "log_level": "DEBUG", + "session_data": { + "behavior_stimulus_file": "stimulus_file.pkl", + "ophys_experiment_id": 1234, + "ophys_session_id": 999, + "behavior_session_id": 1071270468, + "foraging_id": "968ff5ae-e0d5-4661-bb6d-242bee28c7ac", + "full_genotype": "Vip-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt", + "reporter_line": [ + "Ai148(TIT2L-GC6f-ICL-tTA2)" + ], + "driver_line": [ + "Vip-IRES-Cre" + ], + "ophys_cell_segmentation_run_id": 1080628312, + "rig_name": "CAM2P.4", + "movie_height": 512, + "movie_width": 451, + "container_id": 5678, + "surface_2p_pixel_size_um": 0.78125, + "max_projection_file": "max_projection.png", + "demix_file": "demix_file.h5", + "average_intensity_projection_image_file": "avg_projection.png", + "date_of_acquisition": "2020-12-17 10:01:12", + "external_specimen_name": 544261, + "targeted_structure": "VISp", + "targeted_depth": 175, + "stimulus_name": "OPHYS_6_images_B", + "sex": "F", + "age": "P156", + "eye_tracking_rig_geometry": { + "monitor_position_mm": [ + 118.6, + 86.2, + 31.6 + ], + "monitor_rotation_deg": [ + 0.0, + 0.0, + 0.0 + ], + "camera_position_mm": [ + 102.8, + 74.7, + 31.6 + ], + "camera_rotation_deg": [ + 0.0, + 0.0, + 2.8 + ], + "led_position": [ + 246.0, + 92.3, + 52.6 + ], + "equipment": "CAM2P.4" + }, + "eye_tracking_filepath": "/allen/programs/braintv/production/visualbehavior/prod4/specimen_1050612336/ophys_session_1071202230/eye_tracking/1071202230_ellipse.h5", + "events_file": "/allen/programs/braintv/production/visualbehavior/prod4/specimen_1050612336/ophys_session_1071202230/ophys_experiment_1071440875/1071440875_event.h5", + "imaging_plane_group": null, + "plane_group_count": 0, + "cell_specimen_table_dict": { + "cell_roi_id": { + "0": 1080639771, + "1": 1080639729, + "2": 1080639652 + }, + "cell_specimen_id": { + "0": 1086633380, + "1": 1086633339, + "2": 1086633332 + }, + "x": { + "0": 422, + "1": 185, + "2": 2 + }, + "y": { + "0": 147, + "1": 2, + "2": 336 + }, + "max_correction_up": { + "0": 20.0, + "1": 20.0, + "2": 20.0 + }, + "max_correction_right": { + "0": 10.0, + "1": 10.0, + "2": 10.0 + }, + "max_correction_down": { + "0": 10.0, + "1": 10.0, + "2": 10.0 + }, + "max_correction_left": { + "0": 0.0, + "1": 0.0, + "2": 0.0 + }, + "valid_roi": { + "0": true, + "1": true, + "2": false + }, + "height": { + "0": 1, + "1": 1, + "2": 1 + }, + "width": { + "0": 1, + "1": 1, + "2": 1 + }, + "mask_image_plane": { + "0": 0, + "1": 0, + "2": 0 + }, + "roi_mask": { + "0": [ + [ + true + ] + ], + "1": [ + [ + true + ] + ], + "2": [ + [ + true + ] + ] + } + } + } +} \ No newline at end of file diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/trials.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/trials.pkl new file mode 100644 index 000000000..8a87d8a2d Binary files /dev/null and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/trials.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_licks.py b/allensdk/test/brain_observatory/behavior/data_objects/test_licks.py new file mode 100644 index 000000000..cc97a936b --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_licks.py @@ -0,0 +1,182 @@ +import pickle +from datetime import datetime +from pathlib import Path +import numpy as np +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.licks import Licks + + +class TestFromStimulusFile: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + cls.stimulus_file = StimulusFile( + filepath=test_data_dir / 'behavior_stimulus_file.pkl') + expected = pd.read_pickle(str(test_data_dir / 'licks.pkl')) + cls.expected = Licks(licks=expected) + + def test_from_stimulus_file(self): + st = StimulusTimestamps.from_stimulus_file( + stimulus_file=self.stimulus_file) + licks = Licks.from_stimulus_file(stimulus_file=self.stimulus_file, + stimulus_timestamps=st) + assert licks == self.expected + + def test_from_stimulus_file2(self, tmpdir): + """ + Test that Licks.from_stimulus_file returns a dataframe + of licks whose timestamps are based on their frame number + with respect to the stimulus_timestamps + """ + stimulus_filepath = self._create_test_stimulus_file( + lick_events=[12, 15, 90, 136], tmpdir=tmpdir) + stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(stimulus_filepath)}) + timestamps = StimulusTimestamps(timestamps=np.arange(0, 2.0, 0.01)) + licks = Licks.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + + expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], + 'frame': [12, 15, 90, 136]} + expected_df = pd.DataFrame(expected_dict) + assert expected_df.columns.equals(licks.value.columns) + np.testing.assert_array_almost_equal( + expected_df.timestamps.to_numpy(), + licks.value['timestamps'].to_numpy(), + decimal=10) + np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), + licks.value['frame'].to_numpy(), + decimal=10) + + def test_empty_licks(self, tmpdir): + """ + Test that Licks.from_stimulus_file in the case where + there are no licks + """ + + stimulus_filepath = self._create_test_stimulus_file( + lick_events=[], tmpdir=tmpdir) + stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(stimulus_filepath)}) + timestamps = StimulusTimestamps(timestamps=np.arange(0, 2.0, 0.01)) + licks = Licks.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + + expected_dict = {'timestamps': [], + 'frame': []} + expected_df = pd.DataFrame(expected_dict) + assert expected_df.columns.equals(licks.value.columns) + np.testing.assert_array_equal(expected_df.timestamps.to_numpy(), + licks.value['timestamps'].to_numpy()) + np.testing.assert_array_equal(expected_df.frame.to_numpy(), + licks.value['frame'].to_numpy()) + + def test_get_licks_excess(self, tmpdir): + """ + Test that Licks.from_stimulus_file + in the case where + there is an extra frame at the end of the trial log and the mouse + licked on that frame + + https://github.com/AllenInstitute/visual_behavior_analysis/blob + /master/visual_behavior/translator/foraging2/extract.py#L640-L647 + """ + stimulus_filepath = self._create_test_stimulus_file( + lick_events=[12, 15, 90, 136, 200], # len(timestamps) == 200, + tmpdir=tmpdir) + stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(stimulus_filepath)}) + timestamps = StimulusTimestamps(timestamps=np.arange(0, 2.0, 0.01)) + licks = Licks.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + + expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], + 'frame': [12, 15, 90, 136]} + expected_df = pd.DataFrame(expected_dict) + assert expected_df.columns.equals(licks.value.columns) + np.testing.assert_array_almost_equal( + expected_df.timestamps.to_numpy(), + licks.value['timestamps'].to_numpy(), + decimal=10) + np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), + licks.value['frame'].to_numpy(), + decimal=10) + + def test_get_licks_failure(self, tmpdir): + stimulus_filepath = self._create_test_stimulus_file( + lick_events=[12, 15, 90, 136, 201], # len(timestamps) == 200, + tmpdir=tmpdir) + stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(stimulus_filepath)}) + timestamps = StimulusTimestamps(timestamps=np.arange(0, 2.0, 0.01)) + + with pytest.raises(IndexError): + Licks.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + + @staticmethod + def _create_test_stimulus_file(lick_events, tmpdir): + trial_log = [ + {'licks': [(-1.0, 100), (-1.0, 200)]}, + {'licks': [(-1.0, 300), (-1.0, 400)]}, + {'licks': [(-1.0, 500), (-1.0, 600)]} + ] + + lick_events = [{'lick_events': lick_events}] + + data = { + 'items': { + 'behavior': { + 'trial_log': trial_log, + 'lick_sensors': lick_events + } + }, + } + tmp_path = tmpdir / 'stimulus_file.pkl' + with open(tmp_path, 'wb') as f: + pickle.dump(data, f) + f.seek(0) + + return tmp_path + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + stimulus_file = StimulusFile( + filepath=test_data_dir / 'behavior_stimulus_file.pkl') + ts = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + cls.licks = Licks.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=ts) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.licks.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=Licks) + else: + obt = self.licks.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.licks diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_motion_correction.py b/allensdk/test/brain_observatory/behavior/data_objects/test_motion_correction.py new file mode 100644 index 000000000..57994bc7f --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_motion_correction.py @@ -0,0 +1,116 @@ +import json +from datetime import datetime +from pathlib import Path + +import numpy as np +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files\ + .rigid_motion_transform_file import \ + RigidMotionTransformFile +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .cell_specimens import \ + CellSpecimens +from allensdk.brain_observatory.behavior.data_objects.motion_correction \ + import \ + MotionCorrection +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .ophys_timestamps import \ + OphysTimestamps +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest +from allensdk.test.brain_observatory.behavior.data_objects.metadata\ + .test_behavior_ophys_metadata import \ + TestBOM +from allensdk.test.brain_observatory.behavior.data_objects.nwb_input_json \ + import \ + NwbInputJson + + +class TestFromDataFile(LimsTest): + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + + @pytest.mark.requires_bamboo + def test_from_data_file(self): + motion_correction_file = RigidMotionTransformFile.from_lims( + ophys_experiment_id=self.ophys_experiment_id, db=self.dbconn) + mc = MotionCorrection.from_data_file( + rigid_motion_transform_file=motion_correction_file) + assert not mc.value.empty + expected_cols = ['x', 'y'] + assert len(mc.value.columns) == 2 + for c in expected_cols: + assert c in mc.value.columns + + +class TestJson: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + with open(test_data_dir / 'test_input.json') as f: + dict_repr = json.load(f) + dict_repr = dict_repr['session_data'] + dict_repr['rigid_motion_transform_file'] = \ + str(test_data_dir / 'rigid_motion_transform_file.csv') + cls.dict_repr = dict_repr + cls.motion_correction_file = \ + RigidMotionTransformFile.from_json(dict_repr=dict_repr) + expected = pd.DataFrame({'x': [2, 3, 2], 'y': [-3, -4, -4]}) + cls.expected = MotionCorrection(motion_correction=expected) + + def test_from_json(self): + mc = MotionCorrection.from_data_file( + rigid_motion_transform_file=self.motion_correction_file) + assert mc == self.expected + + +class TestNWB: + @classmethod + def setup_class(cls): + df = pd.DataFrame({'x': [2, 3, 2], 'y': [-3, -4, -4]}) + cls.motion_correction = MotionCorrection(motion_correction=df) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + def _write_cell_specimen(): + # write metadata + tbom = TestBOM() + tbom.setup_class() + bom = tbom.meta + bom.to_nwb(nwbfile=self.nwbfile) + + # write cell specimen + ij = NwbInputJson() + ophys_timestamps = OphysTimestamps( + timestamps=np.array([.1, .2, .3])) + csp = CellSpecimens.from_json( + dict_repr=ij.dict_repr, ophys_timestamps=ophys_timestamps, + segmentation_mask_image_spacing=(.78125e-3, .78125e-3)) + csp.to_nwb(nwbfile=self.nwbfile, ophys_timestamps=ophys_timestamps) + + # need to write cell specimen, since it is a dependency + _write_cell_specimen() + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.motion_correction.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=MotionCorrection) + else: + obt = self.motion_correction.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.motion_correction diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_ophys_timestamps.py b/allensdk/test/brain_observatory/behavior/data_objects/test_ophys_timestamps.py new file mode 100644 index 000000000..bd1281e9b --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_ophys_timestamps.py @@ -0,0 +1,91 @@ +from pathlib import Path + +import numpy as np +import pytest + +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_objects.timestamps \ + .ophys_timestamps import \ + OphysTimestamps, OphysTimestampsMultiplane +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromSyncFile(LimsTest): + def setup_method(self, method): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + self.sync_file = SyncFile(filepath=str(test_data_dir / 'sync.h5')) + + def test_from_sync_file(self): + self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])} + ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file)\ + .validate(number_of_frames=3) + expected = np.array([.1, .2, .3]) + np.testing.assert_equal(ts.value, expected) + + def test_too_long_single_plane(self): + """test that timestamps are truncated for single plane data""" + self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])} + ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file)\ + .validate(number_of_frames=2) + expected = np.array([.1, .2]) + np.testing.assert_equal(ts.value, expected) + + def test_too_long_multi_plane(self): + """test that exception raised when timestamps longer than # frames + for multiplane data""" + self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])} + with pytest.raises(RuntimeError): + OphysTimestampsMultiplane.from_sync_file(sync_file=self.sync_file, + group_count=2, + plane_group=0)\ + .validate(number_of_frames=1) + + def test_too_short(self): + """test when timestamps shorter than # frames""" + self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3])} + with pytest.raises(RuntimeError): + OphysTimestamps.from_sync_file(sync_file=self.sync_file)\ + .validate(number_of_frames=4) + + def test_multiplane(self): + """test timestamps properly extracted when multiplane""" + self.sync_file._data = {'ophys_frames': np.array([.1, .2, .3, .4])} + ts = OphysTimestampsMultiplane.from_sync_file(sync_file=self.sync_file, + group_count=2, + plane_group=0)\ + .validate(number_of_frames=2) + expected = np.array([.1, .3]) + np.testing.assert_equal(ts.value, expected) + + @pytest.mark.parametrize( + "timestamps,plane_group,group_count,expected", + [ + (np.ones(10), 1, 0, np.ones(10)), + (np.ones(10), 1, 0, np.ones(10)), + # middle + (np.array([0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0]), 1, 3, np.ones(4)), + # first + (np.array([1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0]), 0, 4, np.ones(3)), + # last + (np.array([0, 1, 0, 1, 0, 1, 0, 1]), 1, 2, np.ones(4)), + # only one group + (np.ones(10), 0, 1, np.ones(10)) + ] + ) + def test_process_ophys_plane_timestamps( + self, timestamps, plane_group, group_count, expected): + """Various test cases""" + self.sync_file._data = {'ophys_frames': timestamps} + number_of_frames = len(timestamps) if group_count == 0 else \ + len(timestamps) / group_count + if group_count == 0: + ts = OphysTimestamps.from_sync_file(sync_file=self.sync_file) + else: + ts = OphysTimestampsMultiplane.from_sync_file( + sync_file=self.sync_file, group_count=group_count, + plane_group=plane_group) + ts = ts.validate(number_of_frames=number_of_frames) + np.testing.assert_array_equal(expected, ts.value) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_projections.py b/allensdk/test/brain_observatory/behavior/data_objects/test_projections.py new file mode 100644 index 000000000..e9a2cf6ae --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_projections.py @@ -0,0 +1,107 @@ +import json +from datetime import datetime +from pathlib import Path + +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_objects.projections import \ + Projections +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator + + +class TestLims: + @classmethod + def setup_class(cls): + cls.ophys_experiment_id = 994278291 + + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + cls.expected_max = Projections._from_filepath( + filepath=str(test_data_dir / 'max_projection.png'), + pixel_size=.78125) + + cls.expected_avg = Projections._from_filepath( + filepath=str(test_data_dir / 'avg_projection.png'), + pixel_size=.78125) + + def setup_method(self, method): + marks = getattr(method, 'pytestmark', None) + if marks: + marks = [m.name for m in marks] + + # Will only create a dbconn if the test requires_bamboo + if 'requires_bamboo' in marks: + self.dbconn = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + + @pytest.mark.requires_bamboo + def test_from_lims(self): + projections = Projections.from_lims( + ophys_experiment_id=self.ophys_experiment_id, lims_db=self.dbconn) + + assert projections.max_projection == self.expected_max + assert projections.avg_projection == self.expected_avg + + +class TestJson: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + with open(test_data_dir / 'test_input.json') as f: + dict_repr = json.load(f) + dict_repr = dict_repr['session_data'] + dict_repr['max_projection_file'] = test_data_dir / \ + dict_repr['max_projection_file'] + dict_repr['average_intensity_projection_image_file'] = \ + test_data_dir / \ + dict_repr['average_intensity_projection_image_file'] + + cls.expected_max = Projections._from_filepath( + filepath=str(test_data_dir / 'max_projection.png'), + pixel_size=.78125) + + cls.expected_avg = Projections._from_filepath( + filepath=str(test_data_dir / 'avg_projection.png'), + pixel_size=.78125) + + cls.dict_repr = dict_repr + + def test_from_json(self): + projections = Projections.from_json(dict_repr=self.dict_repr) + + assert projections.max_projection == self.expected_max + assert projections.avg_projection == self.expected_avg + + +class TestNWB: + @classmethod + def setup_class(cls): + tj = TestJson() + tj.setup_class() + cls.projections = Projections.from_json( + dict_repr=tj.dict_repr) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.projections.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=Projections) + else: + obt = self.projections.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.projections diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_rewards.py b/allensdk/test/brain_observatory/behavior/data_objects/test_rewards.py new file mode 100644 index 000000000..08737eb20 --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_rewards.py @@ -0,0 +1,110 @@ +import pickle +from datetime import datetime +from pathlib import Path +import numpy as np +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromStimulusFile(LimsTest): + @classmethod + def setup_class(cls): + cls.behavior_session_id = 994174745 + + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + expected = pd.read_pickle(str(test_data_dir / 'rewards.pkl')) + cls.expected = Rewards(rewards=expected) + + @pytest.mark.requires_bamboo + def test_from_stimulus_file(self): + stimulus_file = StimulusFile.from_lims( + behavior_session_id=self.behavior_session_id, db=self.dbconn) + timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + rewards = Rewards.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + assert rewards == self.expected + + def test_from_stimulus_file2(self, tmpdir): + """ + Test that Rewards.from_stimulus_file returns + expected results (main nuance is that timestamps should be + determined by applying the reward frame as an index to + stimulus_timestamps) + """ + + def _create_dummy_stimulus_file(): + trial_log = [ + {'rewards': [(0.001, -1.0, 4)], + 'trial_params': {'auto_reward': True}}, + {'rewards': []}, + {'rewards': [(0.002, -1.0, 10)], + 'trial_params': {'auto_reward': False}} + ] + data = { + 'items': { + 'behavior': { + 'trial_log': trial_log + } + }, + } + tmp_path = tmpdir / 'stimulus_file.pkl' + with open(tmp_path, 'wb') as f: + pickle.dump(data, f) + f.seek(0) + + return tmp_path + + stimulus_filepath = _create_dummy_stimulus_file() + stimulus_file = StimulusFile.from_json( + dict_repr={'behavior_stimulus_file': str(stimulus_filepath)}) + timestamps = StimulusTimestamps(timestamps=np.arange(0, 2.0, 0.01)) + rewards = Rewards.from_stimulus_file(stimulus_file=stimulus_file, + stimulus_timestamps=timestamps) + + expected_dict = {'volume': [0.001, 0.002], + 'timestamps': [0.04, 0.1], + 'autorewarded': [True, False]} + expected_df = pd.DataFrame(expected_dict) + expected_df = expected_df + assert expected_df.equals(rewards.value) + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + rewards = pd.read_pickle(str(test_data_dir / 'rewards.pkl')) + cls.rewards = Rewards(rewards=rewards) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.rewards.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=Rewards) + else: + obt = self.rewards.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.rewards diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py b/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py new file mode 100644 index 000000000..a094ec6ad --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py @@ -0,0 +1,125 @@ +from datetime import datetime +from pathlib import Path + +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.stimuli.presentations \ + import \ + Presentations as StimulusPresentations +from allensdk.brain_observatory.behavior.data_objects.stimuli.stimuli import \ + Stimuli +from allensdk.brain_observatory.behavior.data_objects.stimuli.templates \ + import \ + Templates +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromStimulusFile(LimsTest): + @classmethod + def setup_class(cls): + cls.behavior_session_id = 994174745 + + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + presentations = \ + pd.read_pickle(str(test_data_dir / 'presentations.pkl')) + templates = \ + pd.read_pickle(str(test_data_dir / 'templates.pkl')) + cls.expected_presentations = StimulusPresentations( + presentations=presentations) + cls.expected_templates = Templates(templates=templates) + + @pytest.mark.requires_bamboo + def test_from_stimulus_file(self): + stimulus_file = StimulusFile.from_lims( + behavior_session_id=self.behavior_session_id, db=self.dbconn) + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + stimuli = Stimuli.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + limit_to_images=['im065']) + assert stimuli.presentations == self.expected_presentations + assert stimuli.templates == self.expected_templates + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + cls.test_data_dir = dir / 'test_data' + + presentations = \ + pd.read_pickle(str(cls.test_data_dir / 'presentations.pkl')) + templates = \ + pd.read_pickle(str(cls.test_data_dir / 'templates.pkl')) + presentations = presentations.drop('is_change', axis=1) + p = StimulusPresentations(presentations=presentations) + t = Templates(templates=templates) + cls.stimuli = Stimuli(presentations=p, templates=t) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + # Need to write stimulus timestamps first + bsf = StimulusFile( + filepath=self.test_data_dir / 'behavior_stimulus_file.pkl') + ts = StimulusTimestamps.from_stimulus_file(stimulus_file=bsf) + ts.to_nwb(nwbfile=self.nwbfile) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.stimuli.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=Stimuli) + else: + obt = Stimuli.from_nwb(nwbfile=self.nwbfile) + + # is_change different due to limit_to_images + obt.presentations.value.drop('is_change', axis=1, inplace=True) + + assert obt == self.stimuli + + +@pytest.mark.parametrize("stimulus_table, expected_table_data", [ + ({'image_index': [8, 9], + 'image_name': ['omitted', 'not_omitted'], + 'image_set': ['omitted', 'not_omitted'], + 'index': [201, 202], + 'omitted': [True, False], + 'start_frame': [231060, 232340], + 'start_time': [0, 250], + 'stop_time': [None, 1340509], + 'duration': [None, 1340259]}, + {'image_index': [8, 9], + 'image_name': ['omitted', 'not_omitted'], + 'image_set': ['omitted', 'not_omitted'], + 'index': [201, 202], + 'omitted': [True, False], + 'start_frame': [231060, 232340], + 'start_time': [0, 250], + 'stop_time': [0.25, 1340509], + 'duration': [0.25, 1340259]} + ) +]) +def test_set_omitted_stop_time(stimulus_table, expected_table_data): + stimulus_table = pd.DataFrame.from_dict(data=stimulus_table) + expected_table = pd.DataFrame.from_dict(data=expected_table_data) + stimulus_table = \ + StimulusPresentations._fill_missing_values_for_omitted_flashes( + df=stimulus_table) + assert stimulus_table.equals(expected_table) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_task_parameters.py b/allensdk/test/brain_observatory/behavior/data_objects/test_task_parameters.py new file mode 100644 index 000000000..f73f827ea --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_task_parameters.py @@ -0,0 +1,67 @@ +import json +from datetime import datetime +from pathlib import Path +import numpy as np +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects.task_parameters import \ + TaskParameters +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromStimulusFile(LimsTest): + @classmethod + def setup_class(cls): + cls.behavior_session_id = 994174745 + + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + with open(test_data_dir / 'task_parameters.json') as f: + tp = json.load(f) + cls.expected = TaskParameters(**tp) + + @pytest.mark.requires_bamboo + def test_from_stimulus_file(self): + stimulus_file = StimulusFile.from_lims( + behavior_session_id=self.behavior_session_id, db=self.dbconn) + tp = TaskParameters.from_stimulus_file(stimulus_file=stimulus_file) + assert tp == self.expected + + +class TestNWB: + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + dir = Path(__file__).parent.resolve() + self.test_data_dir = dir / 'test_data' + + with open(self.test_data_dir / 'task_parameters.json') as f: + tp = json.load(f) + self.task_parameters = TaskParameters(**tp) + + @pytest.mark.parametrize('is_stimulus_duration_sec_nan', [True, False]) + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture, + is_stimulus_duration_sec_nan): + if is_stimulus_duration_sec_nan: + self.task_parameters._stimulus_duration_sec = np.nan + + self.task_parameters.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=TaskParameters) + else: + obt = TaskParameters.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.task_parameters diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py b/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py new file mode 100644 index 000000000..c82fdb37c --- /dev/null +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_trial_table.py @@ -0,0 +1,181 @@ +from datetime import datetime +from pathlib import Path +from typing import Optional + +import pandas as pd +import pynwb +import pytest + +from allensdk.brain_observatory.behavior.data_files import StimulusFile, \ + SyncFile +from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.licks import Licks +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.equipment import \ + Equipment +from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards +from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \ + calculate_monitor_delay +from allensdk.brain_observatory.behavior.data_objects.trials.trial_table \ + import TrialTable +from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ + LimsTest + + +class TestFromStimulusFile(LimsTest): + @classmethod + def setup_class(cls): + cls.behavior_session_id = 994174745 + cls.ophys_experiment_id = 994278291 + + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + expected = pd.read_pickle(str(test_data_dir / 'trials.pkl')) + cls.expected = TrialTable(trials=expected) + + @pytest.mark.requires_bamboo + def test_from_stimulus_file(self): + stimulus_file, stimulus_timestamps, licks, rewards = \ + self._get_trial_table_data() + sync_file = SyncFile.from_lims( + db=self.dbconn, ophys_experiment_id=self.ophys_experiment_id) + equipment = Equipment.from_lims( + behavior_session_id=self.behavior_session_id, lims_db=self.dbconn) + monitor_delay = calculate_monitor_delay(sync_file=sync_file, + equipment=equipment) + trials = TrialTable.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + licks=licks, + rewards=rewards, + monitor_delay=monitor_delay + ) + assert trials == self.expected + + def test_from_stimulus_file2(self): + dir = Path(__file__).parent.parent.resolve() + stimulus_filepath = dir / 'resources' / 'example_stimulus.pkl.gz' + stimulus_file = StimulusFile(filepath=stimulus_filepath) + stimulus_file, stimulus_timestamps, licks, rewards = \ + self._get_trial_table_data(stimulus_file=stimulus_file) + TrialTable.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps, + monitor_delay=0.02115, + licks=licks, + rewards=rewards + ) + + def _get_trial_table_data(self, + stimulus_file: Optional[StimulusFile] = None): + """returns data required to instantiate a TrialTable""" + if stimulus_file is None: + stimulus_file = StimulusFile.from_lims( + behavior_session_id=self.behavior_session_id, db=self.dbconn) + stimulus_timestamps = StimulusTimestamps.from_stimulus_file( + stimulus_file=stimulus_file) + licks = Licks.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps) + rewards = Rewards.from_stimulus_file( + stimulus_file=stimulus_file, + stimulus_timestamps=stimulus_timestamps) + return stimulus_file, stimulus_timestamps, licks, rewards + + +class TestMonitorDelay: + @classmethod + def setup_class(cls): + cls.lookup_table_expected_values = { + 'CAM2P.1': 0.020842, + 'CAM2P.2': 0.037566, + 'CAM2P.3': 0.021390, + 'CAM2P.4': 0.021102, + 'CAM2P.5': 0.021192, + 'MESO.1': 0.03613 + } + + def setup_method(self, method): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + trials = pd.read_pickle(str(test_data_dir / 'trials.pkl')) + self.sync_file = SyncFile(filepath=str(test_data_dir / 'sync.h5')) + self.trials = TrialTable(trials=trials) + + def test_monitor_delay(self, monkeypatch): + equipment = Equipment(equipment_name='CAM2P.1') + + def dummy_delay(self): + return 1.12 + + with monkeypatch.context() as ctx: + ctx.setattr(OphysTimeAligner, + '_get_monitor_delay', + dummy_delay) + md = calculate_monitor_delay(sync_file=self.sync_file, + equipment=equipment) + assert abs(md - 1.12) < 1.0e-6 + + def test_monitor_delay_lookup(self, monkeypatch): + def dummy_delay(self): + """force monitor delay calculation to fail""" + raise ValueError("that did not work") + + with monkeypatch.context() as ctx: + ctx.setattr(OphysTimeAligner, + '_get_monitor_delay', + dummy_delay) + for equipment, expected in \ + self.lookup_table_expected_values.items(): + equipment = Equipment(equipment_name=equipment) + md = calculate_monitor_delay( + sync_file=self.sync_file, equipment=equipment) + assert abs(md - expected) < 1e-6 + + def test_unkown_rig_name(self, monkeypatch): + def dummy_delay(self): + """force monitor delay calculation to fail""" + raise ValueError("that did not work") + + with monkeypatch.context() as ctx: + ctx.setattr(OphysTimeAligner, + '_get_monitor_delay', + dummy_delay) + equipment = Equipment(equipment_name='spam') + with pytest.raises(RuntimeError): + calculate_monitor_delay(sync_file=self.sync_file, + equipment=equipment) + + +class TestNWB: + @classmethod + def setup_class(cls): + dir = Path(__file__).parent.resolve() + test_data_dir = dir / 'test_data' + + trials = pd.read_pickle(str(test_data_dir / 'trials.pkl')) + cls.trials = TrialTable(trials=trials) + + def setup_method(self, method): + self.nwbfile = pynwb.NWBFile( + session_description='asession', + identifier='1234', + session_start_time=datetime.now() + ) + + @pytest.mark.parametrize('roundtrip', [True, False]) + def test_read_write_nwb(self, roundtrip, + data_object_roundtrip_fixture): + self.trials.to_nwb(nwbfile=self.nwbfile) + + if roundtrip: + obt = data_object_roundtrip_fixture( + nwbfile=self.nwbfile, + data_object_cls=TrialTable) + else: + obt = self.trials.from_nwb(nwbfile=self.nwbfile) + + assert obt == self.trials diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_data_xforms.py b/allensdk/test/brain_observatory/behavior/test_behavior_data_xforms.py deleted file mode 100644 index 650a94127..000000000 --- a/allensdk/test/brain_observatory/behavior/test_behavior_data_xforms.py +++ /dev/null @@ -1,332 +0,0 @@ -import pytest -import logging -import numpy as np -import pandas as pd -from allensdk.brain_observatory.behavior.session_apis.data_transforms import BehaviorDataTransforms # noqa: E501 - - -def test_get_stimulus_timestamps(monkeypatch): - """ - Test that BehaviorDataTransforms.get_stimulus_timestamps() - just returns the sum of the intervalsms field in the - behavior stimulus pickle file, padded with a zero at the - first timestamp. - """ - - expected = np.array([0., 0.0001, 0.0003, 0.0006, 0.001]) - - def dummy_init(self): - pass - - def dummy_stimulus_file(self): - intervalsms = [0.1, 0.2, 0.3, 0.4] - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['intervalsms'] = intervalsms - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xform = BehaviorDataTransforms() - timestamps = xform.get_stimulus_timestamps() - np.testing.assert_array_almost_equal(timestamps, - expected, - decimal=10) - - -def test_get_rewards(monkeypatch): - """ - Test that BehaviorDataTransforms.get_rewards() returns - expected results (main nuance is that timestamps should be - determined by applying the reward frame as an index to - stimulus_timestamps) - """ - - def dummy_init(self): - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - trial_log = [] - trial_log.append({'rewards': [(0.001, -1.0, 4)], - 'trial_params': {'auto_reward': True}}) - trial_log.append({'rewards': []}) - trial_log.append({'rewards': [(0.002, -1.0, 10)], - 'trial_params': {'auto_reward': False}}) - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorDataTransforms() - - rewards = xforms.get_rewards() - - expected_dict = {'volume': [0.001, 0.002], - 'timestamps': [0.04, 0.1], - 'autorewarded': [True, False]} - expected_df = pd.DataFrame(expected_dict) - expected_df = expected_df - assert expected_df.equals(rewards) - - -def test_get_licks(monkeypatch): - """ - Test that BehaviorDataTransforms.get_licks() a dataframe - of licks whose timestamps are based on their frame number - with respect to the stimulus_timestamps - """ - - def dummy_init(self): - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136] - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], - 'frame': [12, 15, 90, 136]} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_almost_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy(), - decimal=10) - np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy(), - decimal=10) - - -def test_empty_licks(monkeypatch): - """ - Test that BehaviorDataTransforms.get_licks() in the case where - there are no licks - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [] - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [], - 'frame': []} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy()) - np.testing.assert_array_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy()) - - -def test_get_licks_excess(monkeypatch): - """ - Test that BehaviorDataTransforms.get_licks() in the case where - there is an extra frame at the end of the trial log and the mouse - licked on that frame - - https://github.com/AllenInstitute/visual_behavior_analysis/blob/master/visual_behavior/translator/foraging2/extract.py#L640-L647 - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136, 200] # len(timestamps) == 200 - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], - 'frame': [12, 15, 90, 136]} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_almost_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy(), - decimal=10) - np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy(), - decimal=10) - - -def test_get_licks_failure(monkeypatch): - """ - Test that BehaviorDataTransforms.get_licks() fails if the last lick - is more than one frame beyond the end of the timestamps - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136, 201] # len(timestamps) == 200 - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorDataTransforms() - with pytest.raises(IndexError): - xforms.get_licks() diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py deleted file mode 100644 index 8ea3e45ab..000000000 --- a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py +++ /dev/null @@ -1,417 +0,0 @@ -import math -from datetime import datetime -from uuid import UUID - -import numpy as np -import pandas as pd -import pytest -import pytz - -from allensdk import OneResultExpectedError -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.brain_observatory.behavior.mtrain import ExtendedTrialSchema -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsApi, BehaviorLimsExtractor, BehaviorOphysLimsApi) -from allensdk.core.authentication import DbCredentials -from allensdk.core.exceptions import DataFrameIndexError -from marshmallow.schema import ValidationError - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('behavior_experiment_id, compare_val', [ - pytest.param(880293569, - ("/allen/programs/braintv/production/neuralcoding/prod0" - "/specimen_703198163/behavior_session_880293569/" - "880289456.pkl")), - pytest.param(0, None) -]) -def test_get_behavior_stimulus_file(behavior_experiment_id, compare_val): - - if compare_val is None: - expected_fail = False - try: - api = BehaviorLimsApi(behavior_experiment_id) - api.extractor.get_behavior_stimulus_file() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - api = BehaviorLimsApi(behavior_experiment_id) - assert api.extractor.get_behavior_stimulus_file() == compare_val - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('behavior_session_uuid', [ - pytest.param('394a910e-94c7-4472-9838-5345aff59ed8'), -]) -def test_foraging_id_to_behavior_session_id(behavior_session_uuid): - session = BehaviorLimsExtractor.from_foraging_id(behavior_session_uuid) - assert session.behavior_session_id == 823847007 - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('behavior_session_id', [ - pytest.param(823847007), -]) -def test_behavior_session_id_to_foraging_id(behavior_session_id): - session = BehaviorLimsApi(behavior_session_id=behavior_session_id) - behavior_session_uuid = session.get_metadata().behavior_session_uuid - expected = UUID('394a910e-94c7-4472-9838-5345aff59ed8') - assert behavior_session_uuid == expected - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize( - 'behavior_experiment_id', [ - 880293569, # stage: TRAINING_0_gratings_autorewards_15min - 881236782, # stage: TRAINING_1_gratings - 881236761, # stage: TRAINING_2_gratings_flashed - ] -) -def test_get_extended_trials(behavior_experiment_id): - api = BehaviorLimsApi(behavior_experiment_id) - df = api.get_extended_trials() - ets = ExtendedTrialSchema(partial=False, many=True) - data_list_cs = df.to_dict('records') - data_list_cs_sc = ets.dump(data_list_cs) - ets.load(data_list_cs_sc) - - df_fail = df.drop(['behavior_session_uuid'], axis=1) - ets = ExtendedTrialSchema(partial=False, many=True) - data_list_cs = df_fail.to_dict('records') - data_list_cs_sc = ets.dump(data_list_cs) - try: - ets.load(data_list_cs_sc) - raise RuntimeError("This should have failed with " - "marshmallow.schema.ValidationError") - except ValidationError: - pass - - -mock_db_credentials = DbCredentials(dbname='mock_db', user='mock_user', - host='mock_host', port='mock_port', - password='mock') - - -@pytest.fixture -def MockBehaviorLimsApi(): - - class MockBehaviorLimsRawApi(BehaviorLimsExtractor): - """ - Mock class that overrides some functions to provide test data and - initialize without calls to db. - """ - def __init__(self): - super().__init__(behavior_session_id=8675309, - lims_credentials=mock_db_credentials, - mtrain_credentials=mock_db_credentials) - self.foraging_id = '138531ab-fe59-4523-9154-07c8d97bbe03' - - def _get_ids(self): - return {} - - def get_date_of_acquisition(self): - return datetime(2019, 9, 26, 16, tzinfo=pytz.UTC) - - def get_behavior_stimulus_file(self): - return "dummy_stimulus_file.pkl" - - def get_foraging_id(self) -> str: - return self.foraging_id - - class MockBehaviorLimsApi(BehaviorLimsApi): - - def _behavior_stimulus_file(self): - data = { - "items": { - "behavior": { - "lick_sensors": [{ - "lick_events": [2, 6, 9], - }], - "intervalsms": np.array([16.0]*10), - "trial_log": [ - { - "events": - [ - ["trial_start", 2, 2], - ["trial_end", 3, 3], - ], - }, - { - "events": - [ - ["trial_start", 4, 4], - ["trial_start", 5, 5], - ], - }, - ], - }, - }, - "session_uuid": '138531ab-fe59-4523-9154-07c8d97bbe03', - "start_time": datetime(2019, 9, 26, 9), - } - return data - - def get_running_acquisition_df(self, lowpass=True): - return pd.DataFrame( - {"timestamps": [0.0, 0.1, 0.2], - "speed": [8.0, 15.0, 16.0]}).set_index("timestamps") - - api = MockBehaviorLimsApi(extractor=MockBehaviorLimsRawApi()) - yield api - api.cache_clear() - - -@pytest.fixture -def MockApiRunSpeedExpectedError(): - class MockApiRunSpeedExpectedError(BehaviorLimsExtractor): - """ - Mock class that overrides some functions to provide test data and - initialize without calls to db. - """ - def __init__(self): - super().__init__(behavior_session_id=8675309, - mtrain_credentials=mock_db_credentials, - lims_credentials=mock_db_credentials) - - def _get_ids(self): - return {} - - class MockBehaviorLimsApiRunSpeedExpectedError(BehaviorLimsApi): - - def get_running_acquisition_df(self, lowpass=True): - return pd.DataFrame( - {"timestamps": [0.0, 0.1, 0.2], - "speed": [8.0, 15.0, 16.0]}) - - return MockBehaviorLimsApiRunSpeedExpectedError( - extractor=MockApiRunSpeedExpectedError()) - - -# Test the non-sql-query functions -# Does not include tests for the following functions, as they are just calls to -# static methods provided for convenience (and should be covered with their own -# unit tests): -# get_rewards -# get_running_data_df -# get_stimulus_templates -# get_task_parameters -# get_trials -# Does not include test for get_metadata since it just collects data from -# methods covered in other unit tests, or data derived from sql queries. -def test_get_stimulus_timestamps(MockBehaviorLimsApi): - api = MockBehaviorLimsApi - expected = np.array([0.016 * i for i in range(11)]) - assert np.allclose(expected, api.get_stimulus_timestamps()) - - -def test_get_licks(MockBehaviorLimsApi): - api = MockBehaviorLimsApi - expected = pd.DataFrame({"timestamps": [0.016 * i for i in [2., 6., 9.]], - "frame": [2, 6, 9]}) - pd.testing.assert_frame_equal(expected, api.get_licks()) - - -def test_get_behavior_session_uuid(MockBehaviorLimsApi, monkeypatch): - with monkeypatch.context() as ctx: - def dummy_init(self, extractor, behavior_stimulus_file): - self._extractor = extractor - self._behavior_stimulus_file = behavior_stimulus_file - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - stimulus_file = MockBehaviorLimsApi._behavior_stimulus_file() - metadata = BehaviorMetadata( - extractor=MockBehaviorLimsApi.extractor, - behavior_stimulus_file=stimulus_file) - - expected = UUID('138531ab-fe59-4523-9154-07c8d97bbe03') - assert expected == metadata.behavior_session_uuid - - -def test_get_stimulus_frame_rate(MockBehaviorLimsApi): - api = MockBehaviorLimsApi - assert 62.0 == api.get_stimulus_frame_rate() - - -def test_get_date_of_acquisition(MockBehaviorLimsApi): - api = MockBehaviorLimsApi - expected = datetime(2019, 9, 26, 16, tzinfo=pytz.UTC) - actual = api.get_metadata().date_of_acquisition - assert expected == actual - - -def test_get_running_speed(MockBehaviorLimsApi): - expected = pd.DataFrame({ - "timestamps": [0.0, 0.1, 0.2], - "speed": [8.0, 15.0, 16.0]}) - api = MockBehaviorLimsApi - actual = api.get_running_speed() - pd.testing.assert_frame_equal(expected, actual) - - -def test_get_running_speed_raises_index_error(MockApiRunSpeedExpectedError): - with pytest.raises(DataFrameIndexError): - MockApiRunSpeedExpectedError.get_running_speed() - - -# def test_get_stimulus_presentations(MockBehaviorLimsApi): -# api = MockBehaviorLimsApi -# # TODO. This function is a monster with multiple dependencies, -# # no tests, and no documentation (for any of its dependencies). -# # Needs to be broken out into testable parts. - -@pytest.mark.requires_bamboo -@pytest.mark.nightly -class TestBehaviorRegression: - """ - Test whether behavior sessions (that are also ophys) loaded with - BehaviorLimsExtractor return the same results as sessions loaded - with BehaviorOphysLimsExtractor, for relevant functions. Do not check for - timestamps, which are from different files so will not be the same. - Also not checking for experiment_date, since they're from two different - sources (and I'm not sure how it's uploaded in the database). - - Do not test `get_licks` regression because the licks come from two - different sources and are recorded differently (behavior pickle file in - BehaviorLimsExtractor; sync file in BehaviorOphysLimeApi) - """ - @classmethod - def setup_class(cls): - cls.bd = BehaviorLimsApi(976012750) - cls.od = BehaviorOphysLimsApi(976255949) - - @classmethod - def teardown_class(cls): - cls.bd.cache_clear() - cls.od.cache_clear() - - def test_stim_file_regression(self): - assert (self.bd.extractor.get_behavior_stimulus_file() - == self.od.extractor.get_behavior_stimulus_file()) - - def test_get_rewards_regression(self): - bd_rewards = self.bd.get_rewards().drop(columns=['timestamps']) - od_rewards = self.od.get_rewards().drop(columns=['timestamps']) - pd.testing.assert_frame_equal(bd_rewards, od_rewards) - - def test_ophys_experiment_id_regression(self): - assert (self.bd.extractor.ophys_experiment_ids[0] - == self.od.extractor.ophys_experiment_id) - - def test_behavior_uuid_regression(self): - assert (self.bd.get_metadata().behavior_session_uuid - == self.od.get_metadata().behavior_session_uuid) - - def test_container_id_regression(self): - assert (self.bd.extractor.ophys_container_id - == self.od.extractor.get_ophys_container_id()) - - def test_stimulus_frame_rate_regression(self): - assert (self.bd.get_stimulus_frame_rate() - == self.od.get_stimulus_frame_rate()) - - def test_get_running_speed_regression(self): - """Can't test values because they're intrinsically linked to timestamps - """ - bd_speed = self.bd.get_running_speed(lowpass=False) - od_speed = self.od.get_running_speed(lowpass=False) - - assert len(bd_speed.values) == len(od_speed.values) - assert len(bd_speed.timestamps) == len(od_speed.timestamps) - - def test_get_running_acquisition_df_regression(self): - """Can't test values because they're intrinsically linked to timestamps - """ - bd_running = self.bd.get_running_acquisition_df(lowpass=False) - od_running = self.od.get_running_acquisition_df(lowpass=False) - - assert len(bd_running) == len(od_running) - assert list(bd_running) == list(od_running) - - def test_get_stimulus_presentations_regression(self): - drop_cols = ["start_time", "stop_time"] - bd_pres = self.bd.get_stimulus_presentations().drop(drop_cols, axis=1) - od_pres = self.od.get_stimulus_presentations().drop(drop_cols, axis=1) - # Duration needs less precision (timestamp-dependent) - pd.testing.assert_frame_equal(bd_pres, od_pres, check_less_precise=2) - - def test_get_stimulus_template_regression(self): - bd_template = self.bd.get_stimulus_templates() - od_template = self.od.get_stimulus_templates() - assert bd_template.keys() == od_template.keys() - for k in bd_template.keys(): - bd_template_img = bd_template[k] - od_template_img = od_template[k] - - assert np.allclose(bd_template_img.unwarped, - od_template_img.unwarped, - equal_nan=True) - assert np.allclose(bd_template_img.warped, - od_template_img.warped, - equal_nan=True) - - def test_get_task_parameters_regression(self): - bd_params = self.bd.get_task_parameters() - od_params = self.od.get_task_parameters() - # Have to do special checking because of nan equality - assert bd_params.keys() == od_params.keys() - for k in bd_params.keys(): - bd_v = bd_params[k] - od_v = od_params[k] - try: - if math.isnan(bd_v): - assert math.isnan(od_v) - else: - assert bd_v == od_v - except (AttributeError, TypeError): - assert bd_v == od_v - - def test_get_trials_regression(self): - """ A lot of timestamp dependent values. Test what we can.""" - cols_to_test = ["reward_volume", "hit", "false_alarm", "miss", - "stimulus_change", "aborted", "go", - "catch", "auto_rewarded", "correct_reject", - "trial_length", "change_frame", "initial_image_name", - "change_image_name"] - bd_trials = self.bd.get_trials()[cols_to_test] - od_trials = self.od.get_trials()[cols_to_test] - pd.testing.assert_frame_equal(bd_trials, od_trials, - check_less_precise=2) - - def test_get_sex_regression(self): - assert self.bd.extractor.get_sex() == self.od.extractor.get_sex() - - def test_get_equipment_name_regression(self): - assert (self.bd.extractor.get_equipment_name() - == self.od.extractor.get_equipment_name()) - - def test_get_stimulus_name_regression(self): - assert (self.bd.extractor.get_stimulus_name() - == self.od.extractor.get_stimulus_name()) - - def test_get_reporter_line_regression(self): - assert (self.bd.extractor.get_reporter_line() - == self.od.extractor.get_reporter_line()) - - def test_get_driver_line_regression(self): - assert (self.bd.extractor.get_driver_line() - == self.od.extractor.get_driver_line()) - - def test_get_external_specimen_name_regression(self): - assert (self.bd.extractor.get_mouse_id() - == self.od.extractor.get_mouse_id()) - - def test_get_full_genotype_regression(self): - assert (self.bd.extractor.get_full_genotype() - == self.od.extractor.get_full_genotype()) - - def test_get_date_of_acquisition_regression(self): - """Just testing the date since it comes from two different sources; - We expect that BehaviorOphysLimsApi will be earlier (more like when - rig was started up), while BehaviorLimsExtractor returns the start of - the actual behavior (from pkl file)""" - assert (self.bd.get_metadata().date_of_acquisition.date() - == self.od.get_metadata().date_of_acquisition.date()) diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_metadata.py b/allensdk/test/brain_observatory/behavior/test_behavior_metadata_legacy.py similarity index 65% rename from allensdk/test/brain_observatory/behavior/test_behavior_metadata.py rename to allensdk/test/brain_observatory/behavior/test_behavior_metadata_legacy.py index 0537295d2..44e4aee0c 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_metadata.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_metadata_legacy.py @@ -6,9 +6,23 @@ import pandas as pd import pytz -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import ( +from allensdk.brain_observatory.behavior.data_files import StimulusFile +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.behavior_metadata import ( description_dict, get_task_parameters, get_expt_description, BehaviorMetadata) +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.date_of_acquisition import \ + DateOfAcquisition +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.age import \ + Age +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.full_genotype import \ + FullGenotype +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .subject_metadata.reporter_line import \ + ReporterLine @pytest.mark.parametrize("data, expected", @@ -246,7 +260,7 @@ def test_get_task_parameters_task_id_exception(): } with pytest.raises(RuntimeError) as error: - get_task_parameters(input_data) + _ = get_task_parameters(input_data) assert "does not know how to parse 'task_id'" in error.value.args[0] @@ -285,7 +299,7 @@ def test_get_task_parameters_flash_duration_exception(): } with pytest.raises(RuntimeError) as error: - get_task_parameters(input_data) + _ = get_task_parameters(input_data) shld_be = "'images' and/or 'grating' not a valid key" assert shld_be in error.value.args[0] @@ -322,266 +336,3 @@ def test_get_expt_description_with_valid_session_type(session_type, def test_get_expt_description_raises_with_invalid_session_type(session_type): with pytest.raises(RuntimeError, match="session type should match.*"): get_expt_description(session_type) - - -def test_cre_line(monkeypatch): - """Tests that cre_line properly parsed from driver_line""" - with monkeypatch.context() as ctx: - def dummy_init(self): - pass - - def full_genotype(self): - return 'Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt' - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - ctx.setattr(BehaviorMetadata, - 'full_genotype', - property(full_genotype)) - - metadata = BehaviorMetadata() - - assert metadata.cre_line == 'Sst-IRES-Cre' - - -def test_cre_line_bad_full_genotype(monkeypatch): - """Test that cre_line is None and no error raised""" - with monkeypatch.context() as ctx: - def dummy_init(self): - pass - - def full_genotype(self): - return 'foo' - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - ctx.setattr(BehaviorMetadata, - 'full_genotype', - property(full_genotype)) - - metadata = BehaviorMetadata() - - with pytest.warns(UserWarning) as record: - cre_line = metadata.cre_line - assert cre_line is None - assert str(record[0].message) == 'Unable to parse cre_line from ' \ - 'full_genotype' - - -def test_reporter_line(monkeypatch): - """Test that reporter line properly parsed from list""" - - class MockExtractor: - def get_reporter_line(self): - return ['foo'] - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - - metadata = BehaviorMetadata() - - assert metadata.reporter_line == 'foo' - - -def test_reporter_line_str(monkeypatch): - """Test that reporter line returns itself if str""" - - class MockExtractor: - def get_reporter_line(self): - return 'foo' - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - - metadata = BehaviorMetadata() - - assert metadata.reporter_line == 'foo' - - -@pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( - (('foo', 'bar'), 'More than 1 reporter line. ' - 'Returning the first one', 'foo'), - (None, 'Error parsing reporter line. It is null.', None), - ([], 'Error parsing reporter line. The array is empty', None) -) - ) -def test_reporter_edge_cases(monkeypatch, input_reporter_line, warning_msg, - expected): - """Test reporter line edge cases""" - - class MockExtractor: - def get_reporter_line(self): - return input_reporter_line - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - metadata = BehaviorMetadata() - - with pytest.warns(UserWarning) as record: - reporter_line = metadata.reporter_line - - assert reporter_line == expected - assert str(record[0].message) == warning_msg - - -def test_age_in_days(monkeypatch): - """Test that age_in_days properly parsed from age""" - - class MockExtractor: - def get_age(self): - return 'P123' - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - - metadata = BehaviorMetadata() - - assert metadata.age_in_days == 123 - - -@pytest.mark.parametrize("input_age, warning_msg, expected", ( - ('unkown', 'Could not parse numeric age from age code ' - '(age code does not start with "P")', None), - ('P', 'Could not parse numeric age from age code ' - '(no numeric values found in age code)', None) -) - ) -def test_age_in_days_edge_cases(monkeypatch, input_age, warning_msg, - expected): - """Test age in days edge cases""" - - class MockExtractor: - def get_age(self): - return input_age - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - - metadata = BehaviorMetadata() - - with pytest.warns(UserWarning) as record: - age_in_days = metadata.age_in_days - - assert age_in_days is None - assert str(record[0].message) == warning_msg - - -@pytest.mark.parametrize("test_params, expected_warn_msg", [ - # Vanilla test case - ({ - "extractor_expt_date": datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "behavior_session_id": 1 - }, None), - - # pkl expt date stored in unix format - ({ - "extractor_expt_date": datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": 1615716855.0, - "behavior_session_id": 2 - }, None), - - # Extractor and pkl dates differ significantly - ({ - "extractor_expt_date": datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": datetime.strptime("2021-03-14 20:14:15", - "%Y-%m-%d %H:%M:%S"), - "behavior_session_id": 3 - }, - "The `date_of_acquisition` field in LIMS *"), - - # pkl file contains an unparseable datetime - ({ - "extractor_expt_date": datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": None, - "behavior_session_id": 4 - }, - "Could not parse the acquisition datetime *"), -]) -def test_get_date_of_acquisition(monkeypatch, tmp_path, test_params, - expected_warn_msg): - mock_session_id = test_params["behavior_session_id"] - - pkl_save_path = tmp_path / f"mock_pkl_{mock_session_id}.pkl" - with open(pkl_save_path, 'wb') as handle: - pickle.dump({"start_time": test_params['pkl_expt_date']}, handle) - behavior_stimulus_file = pd.read_pickle(pkl_save_path) - - tz = pytz.timezone("America/Los_Angeles") - extractor_expt_date = tz.localize( - test_params['extractor_expt_date']).astimezone(pytz.utc) - - class MockExtractor(): - def get_date_of_acquisition(self): - return extractor_expt_date - - def get_behavior_session_id(self): - return test_params['behavior_session_id'] - - def get_behavior_stimulus_file(self): - return pkl_save_path - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self, extractor, behavior_stimulus_file): - self._extractor = extractor - self._behavior_stimulus_file = behavior_stimulus_file - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_init) - - metadata = BehaviorMetadata( - extractor=extractor, - behavior_stimulus_file=behavior_stimulus_file) - - if expected_warn_msg: - with pytest.warns(Warning, match=expected_warn_msg): - obt_date = metadata.date_of_acquisition - else: - obt_date = metadata.date_of_acquisition - - assert obt_date == extractor_expt_date diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_data_xforms.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_data_xforms.py deleted file mode 100644 index 53713bf19..000000000 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_data_xforms.py +++ /dev/null @@ -1,516 +0,0 @@ -import pytest -import logging -import numpy as np -import pandas as pd - -from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.brain_observatory.behavior.session_apis.data_transforms import BehaviorOphysDataTransforms # noqa: E501 -from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner - - -@pytest.mark.parametrize("roi_ids,expected", [ - [ - 1, - np.array([ - [1, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ]) - ], - [ - None, - np.array([ - [ - [1, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ], - [ - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0] - ] - ]) - ] -]) -# cell_specimen_table_api fixture from allensdk.test.brain_observatory.conftest -def test_get_roi_masks_by_cell_roi_id(roi_ids, expected, - cell_specimen_table_api): - api = cell_specimen_table_api - obtained = api.get_roi_masks_by_cell_roi_id(roi_ids) - assert np.allclose(expected, obtained.values) - assert np.allclose(obtained.coords['row'], - [0.5, 1.5, 2.5, 3.5, 4.5]) - assert np.allclose(obtained.coords['column'], - [0.25, 0.75, 1.25, 1.75, 2.25]) - - -def test_get_rewards(monkeypatch): - """ - Test that BehaviorOphysDataTransforms.get_rewards() returns - expected results (main nuance is that timestamps should be - determined by applying the reward frame as an index to - stimulus_timestamps) - """ - - def dummy_init(self): - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - trial_log = [] - trial_log.append({'rewards': [(0.001, -1.0, 4)], - 'trial_params': {'auto_reward': True}}) - trial_log.append({'rewards': []}) - trial_log.append({'rewards': [(0.002, -1.0, 10)], - 'trial_params': {'auto_reward': False}}) - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorOphysDataTransforms() - - rewards = xforms.get_rewards() - - expected_dict = {'volume': [0.001, 0.002], - 'timestamps': [0.04, 0.1], - 'autorewarded': [True, False]} - expected_df = pd.DataFrame(expected_dict) - expected_df = expected_df - assert expected_df.equals(rewards) - - -def test_get_licks(monkeypatch): - """ - Test that BehaviorOphysDataTransforms.get_licks() a dataframe - of licks whose timestamps are based on their frame number - with respect to the stimulus_timestamps - """ - - def dummy_init(self): - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136] - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorOphysDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], - 'frame': [12, 15, 90, 136]} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_almost_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy(), - decimal=10) - np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy(), - decimal=10) - - -def test_get_licks_excess(monkeypatch): - """ - Test that BehaviorOphysDataTransforms.get_licks() in the case where - there is an extra frame at the end of the trial log and the mouse - licked on that frame - - https://github.com/AllenInstitute/visual_behavior_analysis/blob/master/visual_behavior/translator/foraging2/extract.py#L640-L647 - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136, 200] # len(timestamps) == 200 - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorOphysDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [0.12, 0.15, 0.90, 1.36], - 'frame': [12, 15, 90, 136]} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_almost_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy(), - decimal=10) - np.testing.assert_array_almost_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy(), - decimal=10) - - -def test_empty_licks(monkeypatch): - """ - Test that BehaviorOphysDataTransforms.get_licks() in the case where - there are no licks - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [] - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorOphysDataTransforms() - - licks = xforms.get_licks() - - expected_dict = {'timestamps': [], - 'frame': []} - expected_df = pd.DataFrame(expected_dict) - assert expected_df.columns.equals(licks.columns) - np.testing.assert_array_equal(expected_df.timestamps.to_numpy(), - licks.timestamps.to_numpy()) - np.testing.assert_array_equal(expected_df.frame.to_numpy(), - licks.frame.to_numpy()) - - -def test_get_licks_failure(monkeypatch): - """ - Test that BehaviorOphysDataTransforms.get_licks() fails if the last lick - is more than one frame beyond the end of the timestamps - """ - - def dummy_init(self): - self.logger = logging.getLogger('dummy') - pass - - def dummy_stimulus_timestamps(self): - return np.arange(0, 2.0, 0.01) - - def dummy_stimulus_file(self): - - # in this test, the trial log exists to make sure - # that get_licks is *not* reading the licks from - # here - trial_log = [] - trial_log.append({'licks': [(-1.0, 100), (-1.0, 200)]}) - trial_log.append({'licks': [(-1.0, 300), (-1.0, 400)]}) - trial_log.append({'licks': [(-1.0, 500), (-1.0, 600)]}) - - lick_events = [12, 15, 90, 136, 201] # len(timestamps) == 200 - lick_events = [{'lick_events': lick_events}] - - data = {} - data['items'] = {} - data['items']['behavior'] = {} - data['items']['behavior']['trial_log'] = trial_log - data['items']['behavior']['lick_sensors'] = lick_events - return data - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_stimulus_timestamps', - dummy_stimulus_timestamps) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - dummy_stimulus_file) - - xforms = BehaviorOphysDataTransforms() - with pytest.raises(IndexError): - xforms.get_licks() - - -def test_timestamps_and_delay(monkeypatch): - """ - Test that BehaviorOphysDataTransforms returns the right values - with get_stimulus_timestamps and get_monitor_delay - """ - def dummy_loader(self): - self._stimulus_timestamps = np.array([2, 3, 7]) - self._monitor_delay = 99.3 - - def dummy_init(self): - pass - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - "__init__", - dummy_init) - ctx.setattr(BehaviorOphysDataTransforms, - "_load_stimulus_timestamps_and_delay", - dummy_loader) - - xforms = BehaviorOphysDataTransforms() - np.testing.assert_array_equal(xforms.get_stimulus_timestamps(), - np.array([2, 3, 7])) - assert abs(xforms.get_monitor_delay() - 99.3) < 1.0e-10 - - # need to reverse order to make sure loader works - # correctly - xforms = BehaviorOphysDataTransforms() - assert abs(xforms.get_monitor_delay() - 99.3) < 1.0e-10 - np.testing.assert_array_equal(xforms.get_stimulus_timestamps(), - np.array([2, 3, 7])) - - -def test_monitor_delay(monkeypatch): - """ - Check that BehaviorOphysDataTransforms can handle all - edge cases of monitor delay calculation - """ - - # first test case where monitor delay calculation succeeds - class DummyExtractor(object): - def get_sync_file(self): - return '' - - def get_equipment_name(self): - return 'spam' - - def xform_init(self): - self.extractor = DummyExtractor() - - def aligner_init(self, sync_file=None): - self._monitor_delay = None - self._clipped_stim_ts_delta = None - - def dummy_clipped(self): - return np.array([1, 2, 3, 4, 5], dtype=int), -1 - - def dummy_delay(self): - return 1.12 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - xform_init) - - ctx.setattr(OphysTimeAligner, - '__init__', - aligner_init) - - ctx.setattr(OphysTimeAligner, - '_get_clipped_stim_timestamps', - dummy_clipped) - - ctx.setattr(OphysTimeAligner, - '_get_monitor_delay', - dummy_delay) - - xforms = BehaviorOphysDataTransforms() - assert abs(xforms.get_monitor_delay() - 1.12) < 1.0e-6 - np.testing.assert_array_equal(xforms.get_stimulus_timestamps(), - np.array([1, 2, 3, 4, 5], dtype=int)) - - # now try case where monitor delay fails, but value can - # be looked up - def dummy_delay(self): - raise ValueError("that did not work") - - delay_lookup = {'CAM2P.1': 0.020842, - 'CAM2P.2': 0.037566, - 'CAM2P.3': 0.021390, - 'CAM2P.4': 0.021102, - 'CAM2P.5': 0.021192, - 'MESO.1': 0.03613} - - def dummy_get_metadata(self): - def dummy_metadata_init(self, extractor): - self._extractor = extractor - - ctx.setattr(BehaviorMetadata, - '__init__', - dummy_metadata_init) - - class DummyExtractor: - def get_sync_file(self): - return '' - - def get_equipment_name(self): - return equipment_name - - metadata = BehaviorMetadata( - extractor=DummyExtractor()) - return metadata - - for equipment_name in delay_lookup.keys(): - expected_delay = delay_lookup[equipment_name] - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - xform_init) - - ctx.setattr(BehaviorOphysDataTransforms, - 'get_metadata', - dummy_get_metadata) - - ctx.setattr(OphysTimeAligner, - '__init__', - aligner_init) - - ctx.setattr(OphysTimeAligner, - '_get_clipped_stim_timestamps', - dummy_clipped) - - ctx.setattr(OphysTimeAligner, - '_get_monitor_delay', - dummy_delay) - - xforms = BehaviorOphysDataTransforms() - with pytest.warns(UserWarning): - m = xforms.get_monitor_delay() - assert abs(m - expected_delay) < 1.0e-6 - np.testing.assert_array_equal(xforms.get_stimulus_timestamps(), - np.array([1, 2, 3, 4, 5], dtype=int)) - - # finally, try case with unknown rig name - def dummy_delay(self): - raise ValueError("that did not work") - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - xform_init) - equipment_name = 'spam' - ctx.setattr(BehaviorOphysDataTransforms, - 'get_metadata', - dummy_get_metadata) - - ctx.setattr(OphysTimeAligner, - '__init__', - aligner_init) - - ctx.setattr(OphysTimeAligner, - '_get_clipped_stim_timestamps', - dummy_clipped) - - ctx.setattr(OphysTimeAligner, - '_get_monitor_delay', - dummy_delay) - - xforms = BehaviorOphysDataTransforms() - with pytest.raises(RuntimeError): - xforms.get_monitor_delay() diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py index 3833e0fcf..d35d2bcf8 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py @@ -5,39 +5,56 @@ import pandas as pd import pytz import numpy as np -from imageio import imread -from unittest.mock import MagicMock +from unittest.mock import create_autospec + +from pynwb import NWBHDF5IO from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ BehaviorOphysExperiment -from allensdk.brain_observatory.behavior.write_nwb.__main__ import \ - BehaviorOphysJsonApi -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi, BehaviorOphysLimsApi) +from allensdk.brain_observatory.behavior.behavior_session import \ + BehaviorSession +from allensdk.brain_observatory.behavior.data_files import SyncFile +from allensdk.brain_observatory.behavior.data_files.eye_tracking_file import \ + EyeTrackingFile +from allensdk.brain_observatory.behavior.data_files\ + .rigid_motion_transform_file import \ + RigidMotionTransformFile +from allensdk.brain_observatory.behavior.data_objects import \ + BehaviorSessionId, StimulusTimestamps +from allensdk.brain_observatory.behavior.data_objects.cell_specimens\ + .cell_specimens import \ + CellSpecimens +from allensdk.brain_observatory.behavior.data_objects.eye_tracking\ + .eye_tracking_table import \ + EyeTrackingTable +from allensdk.brain_observatory.behavior.data_objects.eye_tracking\ + .rig_geometry import \ + RigGeometry as EyeTrackingRigGeometry +from allensdk.brain_observatory.behavior.data_objects.metadata \ + .behavior_metadata.date_of_acquisition import \ + DateOfAcquisitionOphys +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_metadata.foraging_id import \ + ForagingId +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .behavior_ophys_metadata import \ + BehaviorOphysMetadata +from allensdk.brain_observatory.behavior.data_objects.metadata\ + .ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group \ + import \ + ImagingPlaneGroup +from allensdk.brain_observatory.behavior.data_objects.projections import \ + Projections +from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \ + calculate_monitor_delay +from allensdk.brain_observatory.behavior.data_objects.timestamps\ + .ophys_timestamps import \ + OphysTimestamps from allensdk.brain_observatory.session_api_utils import ( - sessions_are_equal, compare_session_fields) + sessions_are_equal) from allensdk.brain_observatory.stimulus_info import MONITOR_DIMENSIONS - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize("get_expected,get_from_session", [ - [ - lambda ssn_data: ssn_data["ophys_experiment_id"], - lambda ssn: ssn.ophys_experiment_id - ], - [ - lambda ssn_data: imread(ssn_data["max_projection_file"]) / 255, - lambda ssn: ssn.max_projection - ] -]) -def test_session_from_json(tmpdir_factory, session_data, get_expected, - get_from_session): - session = BehaviorOphysExperiment(api=BehaviorOphysJsonApi(session_data)) - - expected = get_expected(session_data) - obtained = get_from_session(session) - - compare_session_fields(expected, obtained) +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator @pytest.mark.requires_bamboo @@ -52,17 +69,21 @@ def test_nwb_end_to_end(tmpdir_factory): 'nwbfile.nwb') d1 = BehaviorOphysExperiment.from_lims(oeid) - BehaviorOphysNwbApi(nwb_filepath).save(d1) + nwbfile = d1.to_nwb() + with NWBHDF5IO(nwb_filepath, 'w') as nwb_file_writer: + nwb_file_writer.write(nwbfile) - d2 = BehaviorOphysExperiment(api=BehaviorOphysNwbApi(nwb_filepath)) + d2 = BehaviorOphysExperiment.from_nwb(nwbfile=nwbfile) - assert sessions_are_equal(d1, d2, reraise=True) + assert sessions_are_equal(d1, d2, reraise=True, + ignore_keys={'metadata': {'project_code'}}) @pytest.mark.nightly def test_visbeh_ophys_data_set(): ophys_experiment_id = 789359614 - data_set = BehaviorOphysExperiment.from_lims(ophys_experiment_id) + data_set = BehaviorOphysExperiment.from_lims(ophys_experiment_id, + exclude_invalid_rois=False) # TODO: need to improve testing here: # for _, row in data_set.roi_metrics.iterrows(): @@ -71,14 +92,20 @@ def test_visbeh_ophys_data_set(): # for _, row in data_set.roi_masks.iterrows(): # print(np.array(row.to_dict()['mask']).sum()) + lims_db = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + behavior_session_id = BehaviorSessionId.from_lims( + db=lims_db, ophys_experiment_id=ophys_experiment_id) + # All sorts of assert relationships: - assert data_set.api.extractor.get_foraging_id() == \ - str(data_set.api.get_metadata().behavior_session_uuid) + assert ForagingId.from_lims(behavior_session_id=behavior_session_id.value, + lims_db=lims_db).value == \ + data_set.metadata['behavior_session_uuid'] - stimulus_templates = data_set._stimulus_templates + stimulus_templates = data_set.stimulus_templates assert len(stimulus_templates) == 8 - assert stimulus_templates['im000'].warped.shape == MONITOR_DIMENSIONS - assert stimulus_templates['im000'].unwarped.shape == MONITOR_DIMENSIONS + assert stimulus_templates.loc['im000'].warped.shape == MONITOR_DIMENSIONS + assert stimulus_templates.loc['im000'].unwarped.shape == MONITOR_DIMENSIONS assert len(data_set.licks) == 2421 and set(data_set.licks.columns) \ == set(['timestamps', 'frame']) @@ -144,13 +171,13 @@ def test_visbeh_ophys_data_set(): @pytest.mark.requires_bamboo def test_legacy_dff_api(): ophys_experiment_id = 792813858 - api = BehaviorOphysLimsApi(ophys_experiment_id) - session = BehaviorOphysExperiment(api) + session = BehaviorOphysExperiment.from_lims( + ophys_experiment_id=ophys_experiment_id) _, dff_array = session.get_dff_traces() for csid in session.dff_traces.index.values: dff_trace = session.dff_traces.loc[csid]['dff'] - ind = session.get_cell_specimen_indices([csid])[0] + ind = session.cell_specimen_table.index.get_loc(csid) np.testing.assert_array_almost_equal(dff_trace, dff_array[ind, :]) assert dff_array.shape[0] == session.dff_traces.shape[0] @@ -167,55 +194,109 @@ def test_stimulus_presentations_omitted(ophys_experiment_id, number_omitted): assert df['omitted'].sum() == number_omitted -@pytest.mark.skip -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [ - pytest.param(789359614), - pytest.param(792813858) -]) -def test_trial_response_window_bounds_reward(ophys_experiment_id): - api = BehaviorOphysLimsApi(ophys_experiment_id) - session = BehaviorOphysExperiment(api) - response_window = session.task_parameters['response_window_sec'] - for _, row in session.trials.iterrows(): - - lick_times = [(t - row.change_time) for t in row.lick_times] - if not np.isnan(row.reward_time): - - # monitor delay is incorporated into the trials table change time - # TODO: where is this set in the session object? - camstim_change_time = row.change_time - 0.0351 - - reward_time = (row.reward_time - camstim_change_time) - assert response_window[0] < reward_time + 1 / 60 - assert reward_time < response_window[1] + 1 / 60 - if len(session.licks) > 0: - assert lick_times[0] < reward_time - - @pytest.mark.parametrize( - "dilation_frames, z_threshold, eye_tracking_start_value", [ - (5, 9, None), - (1, 2, None), - (3, 3, pd.DataFrame([5, 6, 7])) + "dilation_frames, z_threshold", [ + (5, 9), + (1, 2) ]) -def test_eye_tracking(dilation_frames, z_threshold, eye_tracking_start_value): - mock = MagicMock() - mock.get_eye_tracking.return_value = pd.DataFrame([1, 2, 3]) - session = BehaviorOphysExperiment( - api=mock, - eye_tracking_z_threshold=z_threshold, - eye_tracking_dilation_frames=dilation_frames) - - if eye_tracking_start_value is not None: - # Tests that eye_tracking can be set - session.eye_tracking = eye_tracking_start_value - obtained = session.eye_tracking - assert obtained.equals(eye_tracking_start_value) - else: - obtained = session.eye_tracking - assert obtained.equals(pd.DataFrame([1, 2, 3])) - assert session.api.get_eye_tracking.called_with( +def test_eye_tracking(dilation_frames, z_threshold, monkeypatch): + """A very long test just to test that eye tracking arguments are sent to + EyeTrackingTable factory method from BehaviorOphysExperiment.from_lims""" + expected = EyeTrackingTable(eye_tracking=pd.DataFrame([1, 2, 3])) + EyeTrackingTable_mock = create_autospec(EyeTrackingTable) + EyeTrackingTable_mock.from_data_file.return_value = expected + + etf = create_autospec(EyeTrackingFile, instance=True) + sf = create_autospec(SyncFile, instance=True) + + with monkeypatch.context() as ctx: + ctx.setattr('allensdk.brain_observatory.behavior.' + 'behavior_ophys_experiment.db_connection_creator', + create_autospec(db_connection_creator, instance=True)) + ctx.setattr( + SyncFile, 'from_lims', + lambda db, ophys_experiment_id: sf) + ctx.setattr( + StimulusTimestamps, 'from_sync_file', + lambda sync_file: create_autospec(StimulusTimestamps, + instance=True)) + ctx.setattr( + BehaviorSessionId, 'from_lims', + lambda db, ophys_experiment_id: create_autospec(BehaviorSessionId, + instance=True)) + ctx.setattr( + ImagingPlaneGroup, 'from_lims', + lambda lims_db, ophys_experiment_id: None) + ctx.setattr( + BehaviorOphysMetadata, 'from_lims', + lambda lims_db, ophys_experiment_id, + is_multiplane: create_autospec(BehaviorOphysMetadata, + instance=True)) + ctx.setattr('allensdk.brain_observatory.behavior.' + 'behavior_ophys_experiment.calculate_monitor_delay', + create_autospec(calculate_monitor_delay)) + ctx.setattr( + DateOfAcquisitionOphys, 'from_lims', + lambda lims_db, ophys_experiment_id: create_autospec( + DateOfAcquisitionOphys, instance=True)) + ctx.setattr( + BehaviorSession, 'from_lims', + lambda lims_db, behavior_session_id, + stimulus_timestamps, monitor_delay, date_of_acquisition: + BehaviorSession( + behavior_session_id=None, + stimulus_timestamps=None, + running_acquisition=None, + raw_running_speed=None, + running_speed=None, + licks=None, + rewards=None, + stimuli=None, + task_parameters=None, + trials=None, + metadata=None, + date_of_acquisition=None, + )) + ctx.setattr( + OphysTimestamps, 'from_sync_file', + lambda sync_file: create_autospec(OphysTimestamps, + instance=True)) + ctx.setattr( + Projections, 'from_lims', + lambda lims_db, ophys_experiment_id: create_autospec( + Projections, instance=True)) + ctx.setattr( + CellSpecimens, 'from_lims', + lambda lims_db, ophys_experiment_id, ophys_timestamps, + segmentation_mask_image_spacing, events_params, + exclude_invalid_rois: create_autospec( + BehaviorSession, instance=True)) + ctx.setattr( + RigidMotionTransformFile, 'from_lims', + lambda db, ophys_experiment_id: create_autospec( + RigidMotionTransformFile, instance=True)) + ctx.setattr( + EyeTrackingFile, 'from_lims', + lambda db, ophys_experiment_id: etf) + ctx.setattr( + EyeTrackingTable, 'from_data_file', + lambda data_file, sync_file, z_threshold, dilation_frames: + EyeTrackingTable_mock.from_data_file( + data_file=data_file, sync_file=sync_file, + z_threshold=z_threshold, dilation_frames=dilation_frames)) + ctx.setattr( + EyeTrackingRigGeometry, 'from_lims', + lambda lims_db, ophys_experiment_id: create_autospec( + EyeTrackingRigGeometry, instance=True)) + boe = BehaviorOphysExperiment.from_lims( + ophys_experiment_id=1, eye_tracking_z_threshold=z_threshold, + eye_tracking_dilation_frames=dilation_frames) + + obtained = boe.eye_tracking + assert obtained.equals(expected.value) + EyeTrackingTable_mock.from_data_file.assert_called_with( + data_file=etf, + sync_file=sf, z_threshold=z_threshold, dilation_frames=dilation_frames) @@ -252,7 +333,8 @@ def test_BehaviorOphysExperiment_property_data(): assert dataset.ophys_experiment_id == 960410026 -def test_behavior_ophys_experiment_list_data_attributes_and_methods(): +def test_behavior_ophys_experiment_list_data_attributes_and_methods( + monkeypatch): # Test that data related methods/attributes/properties for # BehaviorOphysExperiment are returned properly. @@ -294,7 +376,12 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods(): 'trials' } - behavior_ophys_experiment = BehaviorOphysExperiment(api=MagicMock()) - obt = behavior_ophys_experiment.list_data_attributes_and_methods() + def dummy_init(self): + pass + + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorOphysExperiment, '__init__', dummy_init) + boe = BehaviorOphysExperiment() + obt = boe.list_data_attributes_and_methods() assert any(expected ^ set(obt)) is False diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py deleted file mode 100644 index d0851c9d1..000000000 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py +++ /dev/null @@ -1,468 +0,0 @@ -from pathlib import Path - -import pytest -import pandas as pd -import numpy as np -import h5py -import os -from contextlib import contextmanager - -from allensdk.internal.api import OneResultExpectedError -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysLimsApi, BehaviorOphysLimsExtractor) -from allensdk.brain_observatory.behavior.mtrain import ExtendedTrialSchema -from marshmallow.schema import ValidationError - - -@contextmanager -def does_not_raise(enter_result=None): - """ - Context to help parametrize tests that may raise errors. - If we start supporting only python 3.7+, switch to - contextlib.nullcontext - """ - yield enter_result - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize("ophys_experiment_id", [ - pytest.param(511458874), -]) -def test_get_cell_roi_table(ophys_experiment_id): - api = BehaviorOphysLimsApi(ophys_experiment_id) - assert len(api.get_cell_specimen_table()) == 128 - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize("ophys_experiment_id, compare_val", [ - pytest.param(789359614, - ("/allen/programs/braintv/production/visualbehavior/prod0" - "/specimen_756577249/behavior_session_789295700/" - "789220000.pkl")), - pytest.param(0, None) -]) -def test_get_behavior_stimulus_file(ophys_experiment_id, compare_val): - - if compare_val is None: - expected_fail = False - try: - api = BehaviorOphysLimsApi(ophys_experiment_id) - api.extractor.get_behavior_stimulus_file() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - api = BehaviorOphysLimsApi(ophys_experiment_id) - assert api.extractor.get_behavior_stimulus_file() == compare_val - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize("ophys_experiment_id", [789359614]) -def test_get_extended_trials(ophys_experiment_id): - - api = BehaviorOphysLimsApi(ophys_experiment_id) - df = api.get_extended_trials() - ets = ExtendedTrialSchema(partial=False, many=True) - data_list_cs = df.to_dict("records") - data_list_cs_sc = ets.dump(data_list_cs) - ets.load(data_list_cs_sc) - - df_fail = df.drop(["behavior_session_uuid"], axis=1) - ets = ExtendedTrialSchema(partial=False, many=True) - data_list_cs = df_fail.to_dict("records") - data_list_cs_sc = ets.dump(data_list_cs) - try: - ets.load(data_list_cs_sc) - raise RuntimeError("This should have failed with " - "marshmallow.schema.ValidationError") - except ValidationError: - pass - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize("ophys_experiment_id", [860030092]) -def test_get_nwb_filepath(ophys_experiment_id): - - api = BehaviorOphysLimsApi(ophys_experiment_id) - assert api.extractor.get_nwb_filepath() == ( - "/allen/programs/braintv/production/visualbehavior/prod0/" - "specimen_823826986/ophys_session_859701393/" - "ophys_experiment_860030092/behavior_ophys_session_860030092.nwb") - - -@pytest.mark.parametrize( - "timestamps,plane_group,group_count,expected", - [ - (np.ones(10), 1, 0, np.ones(10)), - (np.ones(10), 1, 0, np.ones(10)), - # middle - (np.array([0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0]), 1, 3, np.ones(4)), - # first - (np.array([1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0]), 0, 4, np.ones(3)), - # last - (np.array([0, 1, 0, 1, 0, 1, 0, 1]), 1, 2, np.ones(4)), - # only one group - (np.ones(10), 0, 1, np.ones(10)) - ] -) -def test_process_ophys_plane_timestamps( - timestamps, plane_group, group_count, expected): - actual = BehaviorOphysLimsApi._process_ophys_plane_timestamps( - timestamps, plane_group, group_count) - np.testing.assert_array_equal(expected, actual) - - -@pytest.mark.parametrize( - "plane_group, ophys_timestamps, dff_traces, expected, context", - [ - (None, np.arange(10), np.arange(5).reshape(1, 5), - np.arange(5), does_not_raise()), - (None, np.arange(10), np.arange(20).reshape(1, 20), - None, pytest.raises(RuntimeError)), - (0, np.arange(10), np.arange(5).reshape(1, 5), - np.arange(0, 10, 2), does_not_raise()), - (0, np.arange(20), np.arange(5).reshape(1, 5), - None, pytest.raises(RuntimeError)) - ], - ids=["scientifica-truncate", "scientifica-raise", "mesoscope-good", - "mesoscope-raise"] -) -def test_get_ophys_timestamps(monkeypatch, plane_group, ophys_timestamps, - dff_traces, expected, context): - """Test the acquisition frame truncation only happens for - non-mesoscope data (and raises error for scientifica data with - longer trace frames than acquisition frames (ophys_timestamps)).""" - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, - "get_behavior_session_id", lambda x: 123) - ctx.setattr(BehaviorOphysLimsExtractor, "_get_ids", lambda x: {}) - patched_extractor = BehaviorOphysLimsExtractor(123) - - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - # Mocking any db calls - monkeypatch.setattr(api, "get_sync_data", - lambda: {"ophys_frames": ophys_timestamps}) - monkeypatch.setattr(api, "get_raw_dff_data", lambda: dff_traces) - monkeypatch.setattr(api.extractor, "get_imaging_plane_group", - lambda: plane_group) - monkeypatch.setattr(api.extractor, "get_plane_group_count", lambda: 2) - with context: - actual = api.get_ophys_timestamps() - if expected is not None: - np.testing.assert_array_equal(expected, actual) - - -def test_dff_trace_order(monkeypatch, tmpdir): - """ - Test that BehaviorOphysLimsApi.get_raw_dff_data can reorder - ROIs to align with what is in the cell_specimen_table - """ - - out_fname = os.path.join(tmpdir, "dummy_dff_data.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((5, n_t)) - roi_names = np.array([5, 3, 4, 2, 1]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, "get_dff_file", - lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_cell_roi_ids", - lambda *args: np.array([1, 2, 3, 4, 5]).astype(bytes)) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - dff_traces = api.get_raw_dff_data() - - # compare the returned traces with the input data - # mapping to the order of the monkeypatched cell_roi_id list - np.testing.assert_array_almost_equal( - dff_traces[0, :], data[4, :], decimal=10) - np.testing.assert_array_almost_equal( - dff_traces[1, :], data[3, :], decimal=10) - np.testing.assert_array_almost_equal( - dff_traces[2, :], data[1, :], decimal=10) - np.testing.assert_array_almost_equal( - dff_traces[3, :], data[2, :], decimal=10) - np.testing.assert_array_almost_equal( - dff_traces[4, :], data[0, :], decimal=10) - - -def test_dff_trace_exceptions(monkeypatch, tmpdir): - """ - Test that BehaviorOphysLimsApi.get_raw_dff_data() raises exceptions when - dff trace file and cell_specimen_table contain different ROI IDs - """ - - # check that an exception is raised if dff_traces has an ROI ID - # that cell_specimen_table does not - out_fname = os.path.join(tmpdir, "dummy_dff_data_for_exceptions.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((5, n_t)) - roi_names = np.array([5, 3, 4, 2, 1]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, "get_dff_file", - lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_cell_roi_ids", - lambda *args: np.array([1, 3, 4, 5]).astype(bytes)) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - with pytest.raises(RuntimeError): - _ = api.get_raw_dff_data() - - # check that an exception is raised if the cell_specimen_table - # has an ROI ID that dff_traces does not - out_fname = os.path.join(tmpdir, "dummy_dff_data_for_exceptions2.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((5, n_t)) - roi_names = np.array([5, 3, 4, 2, 1]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, "get_dff_file", - lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_cell_roi_ids", - lambda *args: np.array([1, 2, 3, 4, 5, 6]).astype(bytes)) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - with pytest.raises(RuntimeError): - _ = api.get_raw_dff_data() - - -def test_corrected_fluorescence_trace_order(monkeypatch, tmpdir): - """ - Test that BehaviorOphysLimsApi.get_corrected_fluorescence_traces - can reorder ROIs to align with what is in the cell_specimen_table - """ - - out_fname = os.path.join(tmpdir, "dummy_ftrace_data.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((5, n_t)) - roi_names = np.array([5, 3, 4, 2, 1]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - cell_data = {"junk": [6, 7, 8, 9, 10], - "cell_roi_id": [b"1", b"2", b"3", b"4", b"5"]} - - cell_table = pd.DataFrame(data=cell_data, - index=pd.Index([10, 20, 30, 40, 50], - name="cell_specimen_id")) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, - "get_demix_file", lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_ophys_timestamps", - lambda *args: np.zeros(n_t)) - ctx.setattr(BehaviorOphysLimsApi, "get_cell_specimen_table", - lambda *args: cell_table) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - f_traces = api.get_corrected_fluorescence_traces() - - # check that the f_traces data frame was correctly joined - # on roi_id - colname = "corrected_fluorescence" - roi_to_dex = {1: 4, 2: 3, 3: 1, 4: 2, 5: 0} - for ii, roi_id in enumerate([1, 2, 3, 4, 5]): - cell = (f_traces.loc[f_traces.cell_roi_id - == bytes(f"{roi_id}", "utf-8")]) - assert cell.index.values[0] == 10*roi_id - np.testing.assert_array_almost_equal(cell[colname].values[0], - data[roi_to_dex[roi_id]], - decimal=10) - - -def test_corrected_fluorescence_trace_exceptions(monkeypatch, tmpdir): - """ - Test that BehaviorOphysLimsApi.get_corrected_fluorescence_traces - raises exceptions when the trace file and cell_specimen_table have - different ROI IDs - - Check case where cell_specimen_table has an ROI that - the fluorescence traces do not - """ - - out_fname = os.path.join(tmpdir, "dummy_ftrace_data_exc.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((4, n_t)) - roi_names = np.array([5, 3, 4, 2]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - cell_data = {"junk": [6, 7, 8, 9, 10], - "cell_roi_id": [b"1", b"2", b"3", b"4", b"5"]} - - cell_table = pd.DataFrame(data=cell_data, - index=pd.Index([10, 20, 30, 40, 50], - name="cell_specimen_id")) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, - "get_demix_file", lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_ophys_timestamps", - lambda *args: np.zeros(n_t)) - ctx.setattr(BehaviorOphysLimsApi, "get_cell_specimen_table", - lambda *args: cell_table) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - with pytest.raises(RuntimeError): - _ = api.get_corrected_fluorescence_traces() - - -def test_corrected_fluorescence_trace_exceptions2(monkeypatch, tmpdir): - """ - Test that BehaviorOphysLimsApi.get_corrected_fluorescence_traces - raises exceptions when the trace file and cell_specimen_table have - different ROI IDs - - Check case where fluorescence traces have an ROI that - the cell_specimen_table does not - """ - - out_fname = os.path.join(tmpdir, "dummy_ftrace_data_exc2.h5") - rng = np.random.RandomState(1234) - n_t = 100 - data = rng.random_sample((5, n_t)) - roi_names = np.array([1, 5, 3, 4, 2]) - with h5py.File(out_fname, "w") as out_file: - out_file.create_dataset("data", data=data) - out_file.create_dataset("roi_names", data=roi_names.astype(bytes)) - - cell_data = {"junk": [6, 7, 8, 9, 10, 11], - "cell_roi_id": [b"1", b"2", b"3", b"4", b"5", b"6"]} - - cell_table = pd.DataFrame(data=cell_data, - index=pd.Index([10, 20, 30, 40, 50, 60], - name="cell_specimen_id")) - - def dummy_init(self, ophys_experiment_id, **kwargs): - self.ophys_experiment_id = ophys_experiment_id - self.get_behavior_session_id = 2 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, "__init__", dummy_init) - ctx.setattr(BehaviorOphysLimsExtractor, - "get_demix_file", lambda *args: out_fname) - patched_extractor = BehaviorOphysLimsExtractor(123) - - ctx.setattr(BehaviorOphysLimsApi, "get_ophys_timestamps", - lambda *args: np.zeros(n_t)) - ctx.setattr(BehaviorOphysLimsApi, "get_cell_specimen_table", - lambda *args: cell_table) - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - with pytest.raises(RuntimeError): - _ = api.get_corrected_fluorescence_traces() - - -def test_eye_tracking_rig_geometry_returns_single_rig(monkeypatch): - """ - This test tests that when there are multiple rig geometries for an experiment, - that only the most recent is returned - """ - def dummy_init(self, ophys_experiment_id): - self.ophys_experiment_id = ophys_experiment_id - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysLimsExtractor, '__init__', dummy_init) - patched_extractor = BehaviorOphysLimsExtractor(123) - - api = BehaviorOphysLimsApi(extractor=patched_extractor) - - resources_dir = Path(os.path.dirname(__file__)) / 'resources' - rig_geometry = ( - pd.read_pickle(resources_dir - / 'rig_geometry_multiple_rig_configs.pkl')) - rig_geometry = api.extractor._process_eye_tracking_rig_geometry( - rig_geometry=rig_geometry) - - expected = { - 'camera_position_mm': [102.8, 74.7, 31.6], - 'led_position': [246.0, 92.3, 52.6], - 'monitor_position_mm': [118.6, 86.2, 31.6], - 'camera_rotation_deg': [0.0, 0.0, 2.8], - 'monitor_rotation_deg': [0.0, 0.0, 0.0], - 'equipment': 'CAM2P.5' - } - - assert rig_geometry == expected - - -@pytest.mark.requires_bamboo -def test_rig_geometry_newer_than_experiment(): - """ - This test ensures that if the experiment date_of_acquisition - is before a rig activate_date that it is not returned as the rig - used for the experiment - """ - # This experiment has rig config more recent than the - # experiment date_of_acquisition - ophys_experiment_id = 521405260 - api = BehaviorOphysLimsApi(ophys_experiment_id) - rig_geometry = api.get_eye_tracking_rig_geometry() - - expected = { - 'camera_position_mm': [130.0, 0.0, 0.0], - 'led_position': [265.1, -39.3, 1.0], - 'monitor_position_mm': [170.0, 0.0, 0.0], - 'camera_rotation_deg': [0.0, 0.0, 13.1], - 'monitor_rotation_deg': [0.0, 0.0, 0.0], - 'equipment': 'CAM2P.1' - } - assert rig_geometry == expected diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_metadata.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_metadata.py deleted file mode 100644 index 1e8c8bf5d..000000000 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_metadata.py +++ /dev/null @@ -1,58 +0,0 @@ -import pytest - -from allensdk.brain_observatory.behavior.metadata.behavior_ophys_metadata \ - import BehaviorOphysMetadata - - -def test_indicator(monkeypatch): - """Test that indicator is parsed from full_genotype""" - - class MockExtractor: - def get_reporter_line(self): - return 'Ai148(TIT2L-GC6f-ICL-tTA2)' - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorOphysMetadata, - '__init__', - dummy_init) - - metadata = BehaviorOphysMetadata() - - assert metadata.indicator == 'GCaMP6f' - - -@pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( - (None, 'Error parsing reporter line. It is null.', None), - ('foo', 'Could not parse indicator from reporter because none' - 'of the expected substrings were found in the reporter', None) -) - ) -def test_indicator_edge_cases(monkeypatch, input_reporter_line, warning_msg, - expected): - """Test indicator parsing edge cases""" - - class MockExtractor: - def get_reporter_line(self): - return input_reporter_line - - extractor = MockExtractor() - - with monkeypatch.context() as ctx: - def dummy_init(self): - self._extractor = extractor - - ctx.setattr(BehaviorOphysMetadata, - '__init__', - dummy_init) - - metadata = BehaviorOphysMetadata() - - with pytest.warns(UserWarning) as record: - indicator = metadata.indicator - assert indicator is expected - assert str(record[0].message) == warning_msg diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_session.py index 99f0d6b5f..013fe0264 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_session.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_session.py @@ -1,81 +1,19 @@ -import logging - -from unittest.mock import MagicMock from allensdk.brain_observatory.behavior.behavior_session import ( BehaviorSession) -class DummyApi(object): - def __init__(self): - pass - - def get_method(self): - """Method docstring""" - pass - - def get_no_docstring_method(self): - pass - - def _other_method(self): - """Other Method docstring""" - pass - +def test_behavior_session_list_data_attributes_and_methods(monkeypatch): + """Test that data related methods/attributes/properties for + BehaviorSession are returned properly.""" -class DummyApiCache(object): - def cache_clear(self): + def dummy_init(self): pass + with monkeypatch.context() as ctx: + ctx.setattr(BehaviorSession, '__init__', dummy_init) + bs = BehaviorSession() + obt = bs.list_data_attributes_and_methods() -class SimpleBehaviorSession(BehaviorSession): - """For the purposes of testing, this class overrides the default - __init__ of the BehaviorSession. The default __init__ uses - LazyProperties which expect certain api methods to exist that - DummyApi and DummyApiCache don't have. - """ - def __init__(self, api): - self.api = api - - -class TestBehaviorSession: - """Tests for BehaviorSession. - The vast majority of methods in BehaviorSession are simply calling - functions from the underlying API. The API required for instantiating a - BehaviorSession is annotated to show that it requires an class that - inherits from BehaviorBase, it is ensured that those methods exist in - the API class. These methods should be covered by unit tests on the - API class and will not be re-tested here. - """ - @classmethod - def setup_class(cls): - cls.behavior_session = SimpleBehaviorSession(api=DummyApi()) - - def test_list_api_methods(self): - expected = [("get_method", "Method docstring"), - ("get_no_docstring_method", "")] - actual = self.behavior_session.list_api_methods() - assert expected == actual - - def test_cache_clear_raises_warning(self, caplog): - expected_msg = ("Attempted to clear API cache, but method" - " `cache_clear` does not exist on DummyApi") - self.behavior_session.cache_clear() - assert caplog.record_tuples == [ - ("BehaviorSession", logging.WARNING, expected_msg)] - - def test_cache_clear_no_warning(self, caplog): - caplog.clear() - bs = SimpleBehaviorSession(api=DummyApiCache()) - bs.cache_clear() - assert len(caplog.record_tuples) == 0 - - -def test_behavior_session_list_data_attributes_and_methods(): - # Test that data related methods/attributes/properties for - # BehaviorSession are returned properly. - - # This test will need to be updated if: - # 1. Data being returned by class has changed - # 2. Inheritance of class has changed expected = { 'behavior_session_id', 'get_performance_metrics', @@ -93,7 +31,4 @@ def test_behavior_session_list_data_attributes_and_methods(): 'trials' } - behavior_session = BehaviorSession(api=MagicMock()) - obt = behavior_session.list_data_attributes_and_methods() - assert any(expected ^ set(obt)) is False diff --git a/allensdk/test/brain_observatory/behavior/test_get_trial_methods.py b/allensdk/test/brain_observatory/behavior/test_get_trial_methods.py deleted file mode 100644 index daf2c4fc3..000000000 --- a/allensdk/test/brain_observatory/behavior/test_get_trial_methods.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -The tests in this file are little better than smoke tests. -They monkeypatch BehaviorDataTransforms and BehaviorOphysDataTransforms -to read the behavior_stimulus_pickle_file from - -/resources/example_stimulus.pkl.gz - -which is a copy of the pickle file for behavior session 891635659, -normally found at - -/allen/programs/braintv/production/visualbehavior/prod2/specimen_850862430/behavior_session_891635659/190620111806_457841_c428be61-87d2-44f6-b00d-3401f28fa201.pkl - -and just attempt to run get_trials() to make sure that passes safely -through that method. A more thorough testing of this method will require -a significant amount of work, as there are many edge cases and the -documentation of the stimulus pickle file is sparse. That should be the -focus of a future ticket. -""" - -import os -import numpy as np -import pandas as pd -from allensdk.brain_observatory.behavior.session_apis.data_transforms import BehaviorDataTransforms # noqa: E501 -from allensdk.brain_observatory.behavior.session_apis.data_transforms import BehaviorOphysDataTransforms # noqa: E501 - - -def test_behavior_get_trials(monkeypatch): - - this_dir = os.path.dirname(os.path.abspath(__file__)) - resource_dir = os.path.join(this_dir, 'resources') - pkl_name = os.path.join(resource_dir, 'example_stimulus.pkl.gz') - pkl_data = pd.read_pickle(pkl_name) - - def dummy_init(self): - pass - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorDataTransforms, - '__init__', - dummy_init) - ctx.setattr(BehaviorDataTransforms, - '_behavior_stimulus_file', - lambda x: pkl_data) - - xforms = BehaviorDataTransforms() - _ = xforms.get_trials() - - -def test_behavior_ophys_get_trials(monkeypatch): - - this_dir = os.path.dirname(os.path.abspath(__file__)) - resource_dir = os.path.join(this_dir, 'resources') - pkl_name = os.path.join(resource_dir, 'example_stimulus.pkl.gz') - pkl_data = pd.read_pickle(pkl_name) - - def dummy_init(self): - pass - - n_t = len(pkl_data['items']['behavior']['intervalsms']) + 1 - timestamps = np.linspace(0, 1, n_t) - - def dummy_loader(self): - self._stimulus_timestamps = np.copy(timestamps) - self._monitor_delay = 0.021 - - with monkeypatch.context() as ctx: - ctx.setattr(BehaviorOphysDataTransforms, - '__init__', - dummy_init) - - ctx.setattr(BehaviorOphysDataTransforms, - '_load_stimulus_timestamps_and_delay', - dummy_loader) - - ctx.setattr(BehaviorOphysDataTransforms, - '_behavior_stimulus_file', - lambda x: pkl_data) - - xforms = BehaviorOphysDataTransforms() - _ = xforms.get_trials() diff --git a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py deleted file mode 100644 index 02ea3c2a5..000000000 --- a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py +++ /dev/null @@ -1,301 +0,0 @@ -import os - -import pytest - -from allensdk.internal.api import OneResultExpectedError, \ - OneOrMoreResultExpectedError -from allensdk.brain_observatory.behavior.session_apis.data_io \ - import OphysLimsExtractor - - -@pytest.fixture(scope="function") -def api_data(): - return { - 702134928: { - 'ophys_dir': '/allen/programs/braintv/production/neuralcoding' - '/prod0/specimen_652073919/ophys_session_702013508' - '/ophys_experiment_702134928/', - 'demix_file': '/allen/programs/braintv/production/neuralcoding' - '/prod0/specimen_652073919/ophys_session_702013508' - '/ophys_experiment_702134928/demix/' - '702134928_demixed_traces.h5', - 'maxint_file': '/allen/programs/braintv/production/neuralcoding' - '/prod0/specimen_652073919/ophys_session_702013508/' - 'ophys_experiment_702134928/processed/' - 'ophys_cell_segmentation_run_814561221/' - 'maxInt_a13a.png', - 'avgint_a1X_file': - '/allen/programs/braintv/production/neuralcoding/prod0' - '/specimen_652073919/ophys_session_702013508' - '/ophys_experiment_702134928/processed' - '/ophys_cell_segmentation_run_814561221/avgInt_a1X.png', - 'rigid_motion_transform_file': - '/allen/programs/braintv/production/neuralcoding/prod0' - '/specimen_652073919/ophys_session_702013508' - '/ophys_experiment_702134928/processed' - '/702134928_rigid_motion_transform.csv', - 'input_extract_traces_file': - '/allen/programs/braintv/production/neuralcoding/prod0' - '/specimen_652073919/ophys_session_702013508' - '/ophys_experiment_702134928/processed' - '/702134928_input_extract_traces.json', - 'targeted_structure': 'VISal', - 'imaging_depth': 175, - 'stimulus_name': 'Unknown', - 'reporter_line': ['Ai148(TIT2L-GC6f-ICL-tTA2)'], - 'driver_line': ['Vip-IRES-Cre'], - 'mouse_id': 363887, - 'full_genotype': 'Vip-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt', - 'workflow_state': 'passed', - } - } - - -def expected_fail(func, *args, **kwargs): - expected_fail = False - try: - func(*args, **kwargs) - except (OneResultExpectedError, OneOrMoreResultExpectedError): - expected_fail = True - - assert expected_fail is True - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_ophys_experiment_dir(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_ophys_experiment_dir - key = 'ophys_dir' - if ophys_experiment_id in api_data: - assert f() == os.path.normpath(api_data[ophys_experiment_id][key]) - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_demix_file(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_demix_file - key = 'demix_file' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_maxint_file(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_max_projection_file - key = 'maxint_file' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_average_intensity_projection_image(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_average_intensity_projection_image_file - key = 'avgint_a1X_file' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_rigid_motion_transform_file(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_rigid_motion_transform_file - key = 'rigid_motion_transform_file' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_targeted_structure(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_targeted_structure - key = 'targeted_structure' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_imaging_depth(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_imaging_depth - key = 'imaging_depth' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_stimulus_name(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_stimulus_name - key = 'stimulus_name' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_reporter_line(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_reporter_line - key = 'reporter_line' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_driver_line(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_driver_line - key = 'driver_line' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_mouse_ID(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_external_specimen_name - key = 'mouse_id' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_full_genotype(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_full_genotype - key = 'full_genotype' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [702134928, 0]) -def test_get_workflow_state(ophys_experiment_id, api_data): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - f = ophys_lims_api.get_workflow_state - key = 'workflow_state' - if ophys_experiment_id in api_data: - assert f() == api_data[ophys_experiment_id][key] - else: - expected_fail(f) - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id, compare_val', [ - pytest.param(511458874, - '/allen/programs/braintv/production/neuralcoding/prod6' - '/specimen_503292442/ophys_experiment_511458874/511458874' - '.nwb'), - pytest.param(0, None) -]) -def test_get_nwb_filepath(ophys_experiment_id, compare_val): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - if compare_val is None: - expected_fail = False - try: - ophys_lims_api.get_nwb_filepath() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - assert ophys_lims_api.get_nwb_filepath() == compare_val - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_experiment_id', [ - pytest.param(511458874), -]) -def test_get_ophys_segmentation_run_id(ophys_experiment_id): - ophys_lims_api = OphysLimsExtractor(ophys_experiment_id) - _ = ophys_lims_api.get_ophys_cell_segmentation_run_id() - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_lims_experiment_id, compare_val', [ - pytest.param(511458874, 0.785203), - pytest.param(0, None) -]) -def test_get_surface_2p_pixel_size_um(ophys_lims_experiment_id, compare_val): - ophys_lims_api = OphysLimsExtractor(ophys_lims_experiment_id) - if compare_val is None: - expected_fail = False - try: - ophys_lims_api.get_surface_2p_pixel_size_um() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - assert ophys_lims_api.get_surface_2p_pixel_size_um() == compare_val - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_lims_experiment_id, compare_val', [ - pytest.param(842510825, 'M'), - pytest.param(0, None) -]) -def test_get_sex(ophys_lims_experiment_id, compare_val): - ophys_lims_api = OphysLimsExtractor(ophys_lims_experiment_id) - if compare_val is None: - expected_fail = False - try: - ophys_lims_api.get_sex() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - assert ophys_lims_api.get_sex() == compare_val - - -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('ophys_lims_experiment_id, compare_val', [ - pytest.param(842510825, 'P157'), - pytest.param(0, None) -]) -def test_get_age(ophys_lims_experiment_id, compare_val): - ophys_lims_api = OphysLimsExtractor(ophys_lims_experiment_id) - if compare_val is None: - expected_fail = False - try: - ophys_lims_api.get_age() - except OneResultExpectedError: - expected_fail = True - assert expected_fail is True - else: - assert ophys_lims_api.get_age() == compare_val diff --git a/allensdk/test/brain_observatory/behavior/test_stimulus_processing.py b/allensdk/test/brain_observatory/behavior/test_stimulus_processing.py index 5313b7558..02592a2c4 100644 --- a/allensdk/test/brain_observatory/behavior/test_stimulus_processing.py +++ b/allensdk/test/brain_observatory/behavior/test_stimulus_processing.py @@ -8,7 +8,7 @@ get_stimulus_presentations, _get_stimulus_epoch, _get_draw_epochs, get_visual_stimuli_df, get_stimulus_metadata, get_gratings_metadata, get_stimulus_templates, is_change_event) -from allensdk.brain_observatory.behavior.stimulus_processing \ +from allensdk.brain_observatory.behavior.data_objects.stimuli\ .stimulus_templates import StimulusImage from allensdk.test.brain_observatory.behavior.conftest import get_resources_dir diff --git a/allensdk/test/brain_observatory/behavior/test_swdb_behavior_project_cache.py b/allensdk/test/brain_observatory/behavior/test_swdb_behavior_project_cache.py deleted file mode 100644 index 16a529418..000000000 --- a/allensdk/test/brain_observatory/behavior/test_swdb_behavior_project_cache.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import numpy as np -import pandas as pd -import pytest -from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc - - -@pytest.fixture -def cache_test_base(): - return '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/test_data' - -@pytest.fixture -def cache(cache_test_base): - return bpc.BehaviorProjectCache(cache_test_base) - -@pytest.fixture -def session(cache): - return cache.get_session(792815735) - -# Test trials extra columns -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_extra_trials_columns(session): - for new_key in ['reward_rate', 'response_binary']: - assert new_key in session.trials.keys() - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_extra_stimulus_presentation_columns(session): - for new_key in [ - 'absolute_flash_number', - 'time_from_last_lick', - 'time_from_last_reward', - 'time_from_last_change', - 'block_index', - 'image_block_repetition', - 'repeat_within_block']: - assert new_key in session.stimulus_presentations.keys() - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_stimulus_presentations_image_set(session): - # We made the image set just 'A' or 'B' - assert session.stimulus_presentations['image_set'].unique() == np.array(['A']) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_stimulus_templates(session): - # Was a dict with only one key, where the value was a 3d array. - # We made it a dict with image names as keys and 2d arrs (the images) as values - for image_name, image_arr in session.stimulus_templates.items(): - assert image_arr.ndim == 2 - -# Test trial response df -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('key, output', [ - ('mean_response', 0.0053334), - ('baseline_response', -0.0020357), - ('p_value', 0.6478659), -]) -def test_session_trial_response(key, output, session): - trial_response = session.trial_response_df - np.testing.assert_almost_equal(trial_response.query("cell_specimen_id == 817103993").iloc[0][key], output, decimal=6) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -@pytest.mark.parametrize('key, output', [ - ('time_from_last_lick', 7.3577), - ('mean_running_speed', 22.143871), - ('duration', 0.25024), -]) -def test_session_flash_response(key, output, session): - flash_response = session.flash_response_df - np.testing.assert_almost_equal(flash_response.query("cell_specimen_id == 817103993").iloc[0][key], output, decimal=6) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_analysis_files_metadata(cache): - assert cache.analysis_files_metadata[ - 'trial_response_df_params' - ]['response_window_duration_seconds'] == 0.5 - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_session_image_loading(session): - assert isinstance(session.max_projection.data, np.ndarray) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_no_invalid_rois(session): - # We made the cache return sessions without the invalid rois - assert session.cell_specimen_table['valid_roi'].all() - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_get_container_sessions(cache): - container_id = cache.experiment_table['container_id'].unique()[0] - container_sessions = cache.get_container_sessions(container_id) - session = container_sessions['OPHYS_1_images_A'] - assert isinstance(session, bpc.ExtendedBehaviorOphysExperiment) - np.testing.assert_almost_equal(session.dff_traces.loc[817103993]['dff'][0], 0.3538657529565) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_binarized_segmentation_mask_image(session): - np.testing.assert_array_equal( - np.unique(np.array(session.segmentation_mask_image.data).ravel()), - np.array([0, 1]) - - ) - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_no_nan_flash_running_speed(session): - assert not pd.isnull(session.stimulus_presentations['mean_running_speed']).any() - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_licks_correct_colname(session): - assert session.licks.columns == ['timestamps'] - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_rewards_correct_colname(session): - assert (session.rewards.columns == ['timestamps', 'volume', 'autorewarded']).all() - - -@pytest.mark.skip(reason="deprecated") -@pytest.mark.requires_bamboo -def test_dff_traces_correct_colname(session): - # This is a Friday-harbor specific change - assert 'cell_roi_id' not in session.dff_traces.columns diff --git a/allensdk/test/brain_observatory/behavior/test_trials_processing.py b/allensdk/test/brain_observatory/behavior/test_trials_processing.py index 0694ab680..5480f9204 100644 --- a/allensdk/test/brain_observatory/behavior/test_trials_processing.py +++ b/allensdk/test/brain_observatory/behavior/test_trials_processing.py @@ -3,9 +3,10 @@ import numpy as np from itertools import combinations -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorLimsApi) +from allensdk.brain_observatory.behavior.data_files import StimulusFile from allensdk.brain_observatory.behavior import trials_processing +from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP +from allensdk.internal.api import db_connection_creator @pytest.mark.requires_bamboo @@ -26,10 +27,13 @@ def test_get_ori_info_from_trial(behavior_experiment_id, ----- - i may be rewriting code here but its more a sanity check really... """ - lims_api = BehaviorLimsApi(behavior_session_id=behavior_experiment_id) - stim_output = pd.read_pickle( - lims_api.extractor.get_behavior_stimulus_file() - ) + def _get_stimulus_data(): + lims_db = db_connection_creator( + fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + stimulus_file = StimulusFile.from_lims( + db=lims_db, behavior_session_id=behavior_experiment_id) + return stimulus_file.data + stim_output = _get_stimulus_data() trial_log = stim_output['items']['behavior']['trial_log'] if exception: diff --git a/allensdk/test/brain_observatory/behavior/test_write_behavior_nwb.py b/allensdk/test/brain_observatory/behavior/test_write_behavior_nwb.py index d7a2bc2a7..d37ee6271 100644 --- a/allensdk/test/brain_observatory/behavior/test_write_behavior_nwb.py +++ b/allensdk/test/brain_observatory/behavior/test_write_behavior_nwb.py @@ -1,250 +1,11 @@ -import math import mock from pathlib import Path - -import numpy as np -import pandas as pd -import pynwb import pytest -import allensdk.brain_observatory.nwb as nwb -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorNwbApi) -from allensdk.brain_observatory.behavior.stimulus_processing import \ - StimulusTemplate, get_stimulus_templates - from allensdk.brain_observatory.behavior.write_behavior_nwb.__main__ import \ write_behavior_nwb # noqa: E501 -# pytest fixtures: -# nwbfile: test.brain_observatory.conftest -# roundtripper: test.brain_observatory.conftest -# running_speed: test.brain_observatory.conftest -# running_acquisition_df_fixture: test.brain_observatory.behavior.conftest - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_running_acquisition_to_nwbfile(nwbfile, roundtrip, roundtripper, - running_acquisition_df_fixture): - nwbfile = nwb.add_running_acquisition_to_nwbfile( - nwbfile, running_acquisition_df_fixture) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - obt_running_acq_df = obt.get_running_acquisition_df() - - pd.testing.assert_frame_equal(running_acquisition_df_fixture, - obt_running_acq_df, - check_like=True) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_running_speed_to_nwbfile(nwbfile, running_speed, - roundtrip, roundtripper): - - nwbfile = nwb.add_running_speed_to_nwbfile(nwbfile, running_speed) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - obt_running_speed = obt.get_running_speed() - - assert np.allclose(running_speed.timestamps, - obt_running_speed['timestamps']) - assert np.allclose(running_speed.values, - obt_running_speed['speed']) - - -@pytest.mark.parametrize('roundtrip,behavior_stimuli_data_fixture', - [(True, {}), (False, {})], - indirect=["behavior_stimuli_data_fixture"]) -def test_add_stimulus_templates(nwbfile, behavior_stimuli_data_fixture, - roundtrip, roundtripper): - stimulus_templates = get_stimulus_templates(behavior_stimuli_data_fixture, - grating_images_dict={}) - - nwb.add_stimulus_template(nwbfile, stimulus_templates) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - stimulus_templates_obt = obt.get_stimulus_templates() - - assert stimulus_templates_obt == stimulus_templates - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_stimulus_presentations(nwbfile, stimulus_presentations_behavior, - stimulus_timestamps, roundtrip, - roundtripper, - stimulus_templates: StimulusTemplate): - nwb.add_stimulus_timestamps(nwbfile, stimulus_timestamps) - nwb.add_stimulus_presentations(nwbfile, stimulus_presentations_behavior) - nwb.add_stimulus_template(nwbfile=nwbfile, - stimulus_template=stimulus_templates) - - # Add index for this template to NWB in-memory object: - nwb_template = nwbfile.stimulus_template[stimulus_templates.image_set_name] - compare = (stimulus_presentations_behavior['image_set'] == - nwb_template.name) - curr_stimulus_index = stimulus_presentations_behavior[compare] - nwb.add_stimulus_index(nwbfile, curr_stimulus_index, nwb_template) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - expected = stimulus_presentations_behavior.copy() - expected['is_change'] = [False, True, True, True, True] - - obtained = obt.get_stimulus_presentations() - - pd.testing.assert_frame_equal(expected[sorted(expected.columns)], - obtained[sorted(obtained.columns)], - check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_stimulus_timestamps(nwbfile, stimulus_timestamps, - roundtrip, roundtripper): - - nwb.add_stimulus_timestamps(nwbfile, stimulus_timestamps) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - np.testing.assert_array_almost_equal(stimulus_timestamps, - obt.get_stimulus_timestamps()) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_trials(nwbfile, roundtrip, roundtripper, trials): - - nwb.add_trials(nwbfile, trials, {}) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(trials, obt.get_trials(), check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_licks(nwbfile, roundtrip, roundtripper, licks): - - nwb.add_licks(nwbfile, licks) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(licks, obt.get_licks(), check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_rewards(nwbfile, roundtrip, roundtripper, rewards): - - nwb.add_rewards(nwbfile, rewards) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(rewards, obt.get_rewards(), - check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_behavior_only_metadata(roundtrip, roundtripper, - behavior_only_metadata_fixture): - - metadata = behavior_only_metadata_fixture - nwbfile = pynwb.NWBFile( - session_description='asession', - identifier='afile', - session_start_time=metadata['date_of_acquisition'] - ) - nwb.add_metadata(nwbfile, metadata, behavior_only=True) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - metadata_obt = obt.get_metadata() - - assert len(metadata_obt) == len(metadata) - for key, val in metadata.items(): - assert val == metadata_obt[key] - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_task_parameters(nwbfile, roundtrip, - roundtripper, task_parameters): - - nwb.add_task_parameters(nwbfile, task_parameters) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - task_parameters_obt = obt.get_task_parameters() - - assert len(task_parameters_obt) == len(task_parameters) - for key, val in task_parameters.items(): - if key == 'omitted_flash_fraction': - if math.isnan(val): - assert math.isnan(task_parameters_obt[key]) - if math.isnan(task_parameters_obt[key]): - assert math.isnan(val) - else: - assert val == task_parameters_obt[key] - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_task_parameters_stim_nan(nwbfile, roundtrip, - roundtripper, - task_parameters_nan_stimulus_duration): - """ - Same as test_add_task_parameters, but stimulus_duration_sec is NaN - """ - task_params = task_parameters_nan_stimulus_duration - nwb.add_task_parameters(nwbfile, task_params) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorNwbApi) - else: - obt = BehaviorNwbApi.from_nwbfile(nwbfile) - - task_parameters_obt = obt.get_task_parameters() - - assert len(task_parameters_obt) == len(task_params) - for key, val in task_params.items(): - if key in ('omitted_flash_fraction', - 'stimulus_duration_sec'): - if math.isnan(val): - assert math.isnan(task_parameters_obt[key]) - if math.isnan(task_parameters_obt[key]): - assert math.isnan(val) - else: - assert val == task_parameters_obt[key] - - def test_write_behavior_nwb_no_file(): """ This function is testing the fail condition of the write_behavior_nwb diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py index 7d20621b7..44a101647 100644 --- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py +++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py @@ -1,400 +1,13 @@ -import math import mock from pathlib import Path -import warnings -import numpy as np -import pandas as pd -import pynwb import pytest -import allensdk.brain_observatory.nwb as nwb -from allensdk.brain_observatory.behavior.session_apis.data_io import ( - BehaviorOphysNwbApi) -from allensdk.test.brain_observatory.behavior.test_eye_tracking_processing import ( # noqa: E501 - create_refined_eye_tracking_df) from allensdk.brain_observatory.behavior.write_nwb.__main__ import \ write_behavior_ophys_nwb # noqa: E501 -@pytest.fixture -def rig_geometry(): - """Returns mock rig geometry data""" - return { - "monitor_position_mm": [1., 2., 3.], - "monitor_rotation_deg": [4., 5., 6.], - "camera_position_mm": [7., 8., 9.], - "camera_rotation_deg": [10., 11., 12.], - "led_position": [13., 14., 15.], - "equipment": "test_rig"} - - -@pytest.fixture -def eye_tracking_data(): - return create_refined_eye_tracking_df( - np.array([[0.1, 12 * np.pi, 72 * np.pi, 196 * np.pi, False, - 196 * np.pi, 12 * np.pi, 72 * np.pi, - 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., - 13., 14., 15.], - [0.2, 20 * np.pi, 90 * np.pi, 225 * np.pi, False, - 225 * np.pi, 20 * np.pi, 90 * np.pi, - 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., - 14., 15., 16.]]) - ) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_stimulus_timestamps(nwbfile, stimulus_timestamps, - roundtrip, roundtripper): - nwb.add_stimulus_timestamps(nwbfile, stimulus_timestamps) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - np.testing.assert_array_almost_equal(stimulus_timestamps, - obt.get_stimulus_timestamps()) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_trials(nwbfile, roundtrip, roundtripper, trials): - nwb.add_trials(nwbfile, trials, {}) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(trials, obt.get_trials(), check_dtype=False) - - -# licks fixture from test.brain_observatory.behavior.conftest -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_licks(nwbfile, roundtrip, roundtripper, licks): - nwb.add_licks(nwbfile, licks) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(licks, obt.get_licks(), check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_rewards(nwbfile, roundtrip, roundtripper, rewards): - nwb.add_rewards(nwbfile, rewards) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal(rewards, obt.get_rewards(), - check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_max_projection(nwbfile, roundtrip, roundtripper, - max_projection, image_api): - nwb.add_max_projection(nwbfile, max_projection) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - assert image_api.deserialize(max_projection) == \ - image_api.deserialize(obt.get_max_projection()) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_average_image(nwbfile, roundtrip, roundtripper, average_image, - image_api): - nwb.add_average_image(nwbfile, average_image) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - assert image_api.deserialize(average_image) == \ - image_api.deserialize(obt.get_average_projection()) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_segmentation_mask_image(nwbfile, roundtrip, roundtripper, - segmentation_mask_image, image_api): - nwb.add_segmentation_mask_image(nwbfile, segmentation_mask_image) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - assert image_api.deserialize(segmentation_mask_image) == \ - image_api.deserialize(obt.get_segmentation_mask_image()) - - -@pytest.mark.parametrize('test_partial_metadata', [True, False]) -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_partial_metadata(test_partial_metadata, roundtrip, roundtripper, - cell_specimen_table, - metadata_fixture, partial_metadata_fixture): - if test_partial_metadata: - meta = partial_metadata_fixture - else: - meta = metadata_fixture - - nwbfile = pynwb.NWBFile( - session_description='asession', - identifier='afile', - session_start_time=meta['date_of_acquisition'] - ) - nwb.add_metadata(nwbfile, meta, behavior_only=False) - if not test_partial_metadata: - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, meta) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - if not test_partial_metadata: - metadata_obt = obt.get_metadata() - else: - with warnings.catch_warnings(record=True) as record: - metadata_obt = obt.get_metadata() - exp_warn_msg = "Could not locate 'ophys' module in NWB" - print(record) - - assert record[0].message.args[0].startswith(exp_warn_msg) - - assert len(metadata_obt) == len(meta) - for key, val in meta.items(): - assert val == metadata_obt[key] - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_add_task_parameters(nwbfile, roundtrip, - roundtripper, task_parameters): - nwb.add_task_parameters(nwbfile, task_parameters) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - task_parameters_obt = obt.get_task_parameters() - - assert len(task_parameters_obt) == len(task_parameters) - for key, val in task_parameters.items(): - if key == 'omitted_flash_fraction': - if math.isnan(val): - assert math.isnan(task_parameters_obt[key]) - if math.isnan(task_parameters_obt[key]): - assert math.isnan(val) - else: - assert val == task_parameters_obt[key] - - -@pytest.mark.parametrize('roundtrip', [True, False]) -@pytest.mark.parametrize("filter_invalid_rois", [True, False]) -def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois, - valid_roi_ids, roundtripper, - cell_specimen_table, metadata_fixture, - ophys_timestamps): - nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi, - filter_invalid_rois=filter_invalid_rois) - else: - obt = BehaviorOphysNwbApi.from_nwbfile( - nwbfile, filter_invalid_rois=filter_invalid_rois) - - if filter_invalid_rois: - cell_specimen_table = \ - cell_specimen_table[ - cell_specimen_table["cell_roi_id"].isin( - valid_roi_ids)] - - pd.testing.assert_frame_equal( - cell_specimen_table, - obt.get_cell_specimen_table(), - check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -@pytest.mark.parametrize("filter_invalid_rois", [True, False]) -def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, - roundtripper, dff_traces, cell_specimen_table, - metadata_fixture, ophys_timestamps): - nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture) - nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi, - filter_invalid_rois=filter_invalid_rois) - else: - obt = BehaviorOphysNwbApi.from_nwbfile( - nwbfile, filter_invalid_rois=filter_invalid_rois) - - if filter_invalid_rois: - dff_traces = dff_traces[dff_traces["cell_roi_id"].isin(valid_roi_ids)] - - print(dff_traces) - - print(obt.get_dff_traces()) - - pd.testing.assert_frame_equal( - dff_traces, obt.get_dff_traces(), check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -@pytest.mark.parametrize("filter_invalid_rois", [True, False]) -def test_get_corrected_fluorescence_traces( - nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, - dff_traces, corrected_fluorescence_traces, cell_specimen_table, - metadata_fixture, ophys_timestamps): - nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture) - nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) - nwb.add_corrected_fluorescence_traces(nwbfile, - corrected_fluorescence_traces) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi, - filter_invalid_rois=filter_invalid_rois) - else: - obt = BehaviorOphysNwbApi.from_nwbfile( - nwbfile, filter_invalid_rois=filter_invalid_rois) - - if filter_invalid_rois: - corrected_fluorescence_traces = corrected_fluorescence_traces[ - corrected_fluorescence_traces["cell_roi_id"].isin(valid_roi_ids)] - - print(corrected_fluorescence_traces) - print(obt.get_corrected_fluorescence_traces()) - - pd.testing.assert_frame_equal( - corrected_fluorescence_traces, - obt.get_corrected_fluorescence_traces(), check_dtype=False) - - -@pytest.mark.parametrize('roundtrip', [True, False]) -def test_get_motion_correction(nwbfile, roundtrip, roundtripper, - motion_correction, ophys_timestamps, - metadata_fixture, cell_specimen_table, - dff_traces): - nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture) - nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) - nwb.add_motion_correction(nwbfile, motion_correction) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - pd.testing.assert_frame_equal( - motion_correction, - obt.get_motion_correction(), - check_dtype=False) - - -@pytest.mark.parametrize("roundtrip", [True, False]) -@pytest.mark.parametrize("expected", [ - ({ - "monitor_position_mm": [1., 2., 3.], - "monitor_rotation_deg": [4., 5., 6.], - "camera_position_mm": [7., 8., 9.], - "camera_rotation_deg": [10., 11., 12.], - "led_position": [13., 14., 15.], - "equipment": "test_rig" - }), -]) -def test_add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile, roundtripper, - roundtrip, - rig_geometry, - expected): - api = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - nwbfile = api.add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile, - rig_geometry) - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - obtained_eye_rig_geometry = obt.get_eye_tracking_rig_geometry() - - assert obtained_eye_rig_geometry == expected - - -# NOTE: uses fixtures -# 'nwbfile' and 'roundtripper' -# from allensdk/test/brain_observatory/conftest.py -@pytest.mark.parametrize("roundtrip", [True, False]) -def test_add_eye_tracking_data_to_nwbfile( - tmp_path, nwbfile, eye_tracking_data, - rig_geometry, roundtripper, roundtrip): - api = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - nwbfile = api.add_eye_tracking_data_to_nwb( - nwbfile=nwbfile, - eye_tracking_df=eye_tracking_data, - eye_tracking_rig_geometry=rig_geometry - ) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - obtained = obt.get_eye_tracking() - - pd.testing.assert_frame_equal(obtained, - eye_tracking_data, check_like=True) - - -@pytest.mark.parametrize("roundtrip", [True, False]) -def test_add_events(tmp_path, nwbfile, roundtripper, roundtrip, - cell_specimen_table, metadata_fixture, dff_traces, - ophys_timestamps): - # Need to add metadata, cell specimen table, dff traces first - nwb.add_metadata(nwbfile, metadata_fixture, behavior_only=False) - nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata_fixture) - nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps) - - events = pd.DataFrame({ - 'events': [np.array([0., 0., .69]), np.array([.3, 0.0, .2])], - 'filtered_events': [ - np.array([0.0, 0.0, 0.22949295]), - np.array([0.09977954, 0.08805513, 0.127039049]) - ], - 'lambda': [0., 1.0], - 'noise_std': [.25, .3], - 'cell_roi_id': [123, 321] - }, index=pd.Index([42, 84], name='cell_specimen_id')) - - api = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - nwbfile = api.add_events( - nwbfile=nwbfile, - events=events - ) - - if roundtrip: - obt = roundtripper(nwbfile, BehaviorOphysNwbApi) - else: - obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile) - - obtained = obt.get_events() - - pd.testing.assert_frame_equal(obtained, events, check_like=True) - - def test_write_behavior_ophys_nwb_no_file(): """ This function is testing the fail condition of the diff --git a/allensdk/test/brain_observatory/nwb/test_nwb_utils.py b/allensdk/test/brain_observatory/nwb/test_nwb_utils.py index 4591018df..30e0df090 100644 --- a/allensdk/test/brain_observatory/nwb/test_nwb_utils.py +++ b/allensdk/test/brain_observatory/nwb/test_nwb_utils.py @@ -1,6 +1,4 @@ import pytest -import pandas as pd - from allensdk.brain_observatory.nwb import nwb_utils @@ -30,29 +28,3 @@ def test_get_stimulus_name_column_exceptions(input_cols, nwb_utils.get_column_name(input_cols, possible_names) for expected_value in expected_excep_cols: assert expected_value in str(error.value) - - -@pytest.mark.parametrize("stimulus_table, expected_table_data", [ - ({'image_index': [8, 9], - 'image_name': ['omitted', 'not_omitted'], - 'image_set': ['omitted', 'not_omitted'], - 'index': [201, 202], - 'omitted': [True, False], - 'start_frame': [231060, 232340], - 'start_time': [0, 250], - 'stop_time': [None, 1340509]}, - {'image_index': [8, 9], - 'image_name': ['omitted', 'not_omitted'], - 'image_set': ['omitted', 'not_omitted'], - 'index': [201, 202], - 'omitted': [True, False], - 'start_frame': [231060, 232340], - 'start_time': [0, 250], - 'stop_time': [0.25, 1340509]} - ) -]) -def test_set_omitted_stop_time(stimulus_table, expected_table_data): - stimulus_table = pd.DataFrame.from_dict(data=stimulus_table) - expected_table = pd.DataFrame.from_dict(data=expected_table_data) - nwb_utils.set_omitted_stop_time(stimulus_table) - assert stimulus_table.equals(expected_table) diff --git a/allensdk/test/internal/test_mtrain_api.py b/allensdk/test/internal/test_mtrain_api.py index cecfa0e14..9e1028352 100644 --- a/allensdk/test/internal/test_mtrain_api.py +++ b/allensdk/test/internal/test_mtrain_api.py @@ -115,20 +115,3 @@ def test_get_session(behavior_session_uuid, behavior_session_id): u'VisualBehavior_Task1A_v1.0.1' }], u'default_y': False} - -# def test_get_ophys_experiment_dir(ophys_experiment_id, compare_val): - -# api = LimsOphysAPI() - -# if compare_val is None: -# expected_fail = False -# try: -# api.get_ophys_experiment_dir(ophys_experiment_id) -# except OneResultExpectedError: -# expected_fail = True -# assert expected_fail == True - -# else: -# api.get_ophys_experiment_dir(ophys_experiment_id=ophys_experiment_id) -# assert api.get_ophys_experiment_dir( -# ophys_experiment_id=ophys_experiment_id) == compare_val diff --git a/doc_template/index.rst b/doc_template/index.rst index 281fb9cbb..d6f595cf8 100644 --- a/doc_template/index.rst +++ b/doc_template/index.rst @@ -118,6 +118,10 @@ The Allen SDK provides Python code for accessing experimental metadata along wit See the `mouse connectivity section `_ for more details. +What's New - 2.13.0 +----------------------------------------------------------------------- +- Major internal refactor to BehaviorSession, BehaviorOphysExperiment classes. Implements DataObject pattern for fetching and serialization of data. + What's New - 2.12.4 ----------------------------------------------------------------------- - Documentation changes ahead of SWDB 2021 diff --git a/requirements.txt b/requirements.txt index 5a61b9179..294a8ea24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ tqdm>=4.27 ndx-events<=0.2.0 boto3==1.17.21 semver +cachetools>=4.2.1,<5.0.0 \ No newline at end of file