From 89da3c16dd26ec819d74dfaf9b06cc64d3286ff6 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 15 Feb 2023 11:49:00 -0800 Subject: [PATCH 1/4] Use ImageSeries for original and labeled videos --- spec/ndx-pose.extensions.yaml | 35 ++++++++- spec/ndx-pose.namespace.yaml | 2 +- src/pynwb/ndx_pose/io/pose.py | 15 ++++ src/pynwb/ndx_pose/pose.py | 123 ++++++++++++++++++++++-------- src/spec/create_extension_spec.py | 61 +++++++++++---- 5 files changed, 185 insertions(+), 51 deletions(-) diff --git a/spec/ndx-pose.extensions.yaml b/spec/ndx-pose.extensions.yaml index 0beec02..21f22e9 100644 --- a/spec/ndx-pose.extensions.yaml +++ b/spec/ndx-pose.extensions.yaml @@ -57,7 +57,8 @@ groups: shape: - null doc: Paths to the original video files. The number of files should equal the number - of camera devices. + of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in original_videos_series + instead. quantity: '?' - name: labeled_videos dtype: text @@ -66,7 +67,8 @@ groups: shape: - null doc: Paths to the labeled video files. The number of files should equal the number - of camera devices. + of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in labeled_videos_series + instead. quantity: '?' - name: dimensions dtype: uint8 @@ -76,7 +78,8 @@ groups: shape: - null - 2 - doc: Dimensions of each labeled video file. + doc: Dimensions of each labeled video file. Deprecated in version 0.2.0. Use "dimension" + in original_videos_series instead. quantity: '?' - name: scorer dtype: text @@ -114,5 +117,29 @@ groups: quantity: '?' groups: - neurodata_type_inc: PoseEstimationSeries - doc: Estimated position data for each body part. + doc: Estimated position data for each body part. Deprecated in version 0.2.0. + Use the "positions" group instead. quantity: '*' + - name: pose_estimates + doc: Estimated position data for each body part. + quantity: '?' + groups: + - neurodata_type_inc: PoseEstimationSeries + doc: Estimated position data for each body part. + quantity: '*' + - name: original_videos_series + doc: Links to the original video files. + quantity: '?' + links: + - target_type: ImageSeries + doc: Links to the original video files. + quantity: '*' + - name: labeled_videos_series + doc: The labeled videos. The number of files should equal the number of original + videos. + quantity: '?' + datasets: + - neurodata_type_inc: ImageSeries + doc: The labeled videos. The number of files should equal the number of original + videos. + quantity: '*' diff --git a/spec/ndx-pose.namespace.yaml b/spec/ndx-pose.namespace.yaml index 7146cd3..2195f1c 100644 --- a/spec/ndx-pose.namespace.yaml +++ b/spec/ndx-pose.namespace.yaml @@ -15,4 +15,4 @@ namespaces: - SpatialSeries - NWBDataInterface - source: ndx-pose.extensions.yaml - version: 0.1.1 + version: 0.2.0 diff --git a/src/pynwb/ndx_pose/io/pose.py b/src/pynwb/ndx_pose/io/pose.py index 9d7b873..8fe7595 100644 --- a/src/pynwb/ndx_pose/io/pose.py +++ b/src/pynwb/ndx_pose/io/pose.py @@ -21,3 +21,18 @@ def __init__(self, spec): super().__init__(spec) source_software_spec = self.spec.get_dataset('source_software') self.map_spec('source_software_version', source_software_spec.get_attribute('version')) + + # TODO if reading a file without the pose_estimates group, load the PoseEstimationSeries from the + # main PoseEstimation group into the pose_estimation_series variable + + pose_estimates_spec = self.spec.get_group('pose_estimates') + self.unmap(pose_estimates_spec) + self.map_spec('pose_estimation_series', pose_estimates_spec.get_neurodata_type('PoseEstimationSeries')) + + original_videos_series_spec = self.spec.get_group('original_videos_series') + self.unmap(original_videos_series_spec) + self.map_spec('original_videos_series', original_videos_series_spec.get_neurodata_type('ImageSeries')) + + labeled_videos_series_spec = self.spec.get_group('labeled_videos_series') + self.unmap(labeled_videos_series_spec) + self.map_spec('labeled_videos_series', labeled_videos_series_spec.get_neurodata_type('ImageSeries')) diff --git a/src/pynwb/ndx_pose/pose.py b/src/pynwb/ndx_pose/pose.py index c3533df..3421b49 100644 --- a/src/pynwb/ndx_pose/pose.py +++ b/src/pynwb/ndx_pose/pose.py @@ -1,9 +1,11 @@ -from hdmf.utils import docval, popargs, get_docval, call_docval_func, AllowPositional +from hdmf.utils import docval, popargs, get_docval, AllowPositional from pynwb import register_class, TimeSeries from pynwb.behavior import SpatialSeries from pynwb.core import MultiContainerInterface -from pynwb.device import Device +from pynwb.image import ImageSeries + +import warnings @register_class('PoseEstimationSeries', 'ndx-pose') @@ -23,7 +25,7 @@ class PoseEstimationSeries(SpatialSeries): 'doc': ('Estimated position (x, y) or (x, y, z).')}, {'name': 'reference_frame', 'type': str, # required 'doc': 'Description defining what the zero-position (0, 0) or (0, 0, 0) is.'}, - {'name': 'confidence', 'type': ('array_data', 'data'), 'shape': (None, ), + {'name': 'confidence', 'type': ('array_data', 'data'), 'shape': (None, ), 'doc': ('Confidence or likelihood of the estimated positions, scaled to be between 0 and 1.'), 'default': None,}, {'name': 'unit', 'type': str, @@ -42,7 +44,7 @@ class PoseEstimationSeries(SpatialSeries): def __init__(self, **kwargs): """Construct a new PoseEstimationSeries representing pose estimates for a particular body part.""" confidence, confidence_definition = popargs('confidence', 'confidence_definition', kwargs) - call_docval_func(super().__init__, kwargs) + super().__init__(**kwargs) self.confidence = confidence self.confidence_definition = confidence_definition @@ -55,30 +57,40 @@ class PoseEstimation(MultiContainerInterface): __clsconf__ = [ { + # NOTE pose_estimation_series was remapped in version 0.2.0 to live under the pose_estimates subgroup 'add': 'add_pose_estimation_series', 'get': 'get_pose_estimation_series', 'create': 'create_pose_estimation_series', 'type': PoseEstimationSeries, 'attr': 'pose_estimation_series' }, - # { - # 'add': 'add_device', - # 'get': 'get_devices', - # 'type': Device, - # 'attr': 'devices' - # # TODO prevent these from being children / add better support for links - # # may require update to HDMF to add a key 'child': False - # } + { + 'add': 'add_original_videos_series', + 'get': 'get_original_videos_series', + 'create': 'create_original_videos_series', + 'type': ImageSeries, + 'attr': 'original_videos_series' + }, + { # TODO how to check that these are links and not subgroups? + 'add': 'add_labeled_videos_series', + 'get': 'get_labeled_videos_series', + 'create': 'create_labeled_videos_series', + 'type': ImageSeries, + 'attr': 'labeled_videos_series' + }, ] - __nwbfields__ = ('description', 'original_videos', 'labeled_videos', 'dimensions', 'scorer', 'source_software', - 'source_software_version', 'nodes', 'edges') + __nwbfields__ = ('description', 'original_videos', 'labeled_videos', 'dimensions', 'scorer', + 'source_software', 'source_software_version', 'nodes', 'edges') # custom mapper in ndx_pose.io.pose maps: - # 'source_software' dataset -> 'version' attribute to 'source_software_version' field + # 'source_software' dataset, 'version' attribute to 'source_software_version' field + # 'pose_estimates' untyped group, 'PoseEstimationSeries' subgroup to 'pose_estimates' field + # 'original_videos_series' untyped group, 'ImageSeries' subgroup to 'original_videos_series' field + # 'labeled_videos_series' untyped group, 'ImageSeries' subgroup to 'labeled_videos_series' field @docval( # all fields optional - {'name': 'pose_estimation_series', 'type': ('array_data', 'data'), + {'name': 'pose_estimation_series', 'type': (list, tuple), 'doc': ('Estimated position data for each body part.'), 'default': None}, {'name': 'name', 'type': str, @@ -88,13 +100,20 @@ class PoseEstimation(MultiContainerInterface): 'doc': ('Description of the pose estimation procedure and output.'), 'default': None}, {'name': 'original_videos', 'type': ('array_data', 'data'), 'shape': (None, ), - 'doc': ('Paths to the original video files. The number of files should equal the number of camera devices.'), + 'doc': ('The original video files.'), + 'default': None}, + {'name': 'original_videos_series', 'type': (list, tuple), 'shape': (None, ), + 'doc': ('The original video files.'), 'default': None}, {'name': 'labeled_videos', 'type': ('array_data', 'data'), 'shape': (None, ), - 'doc': ('Paths to the labeled video files. The number of files should equal the number of camera devices.'), + 'doc': ('Links to the labeled video files. The number of files should equal the number of original videos.'), + 'default': None}, + {'name': 'labeled_videos_series', 'type': (list, tuple), 'shape': (None, ), + 'doc': ('Links to the labeled videos. The number of files should equal the number of original videos.'), 'default': None}, {'name': 'dimensions', 'type': ('array_data', 'data'), 'shape': ((None, 2)), - 'doc': ('Dimensions of each labeled video file.'), + 'doc': ('Dimensions of each labeled video file. Deprecated in version 0.2.0. ' + 'Use "dimension" in original_videos instead.'), 'default': None}, {'name': 'scorer', 'type': str, 'doc': ('Name of the scorer / algorithm used.'), @@ -113,38 +132,76 @@ class PoseEstimation(MultiContainerInterface): 'doc': ("Array of pairs of indices corresponding to edges between nodes. Index values correspond to row " "indices of the 'nodes' field. Index values use 0-indexing."), 'default': None}, - # {'name': 'devices', 'type': ('array_data', 'data'), - # 'doc': ('Cameras used to record the videos.'), - # 'default': None}, allow_positional=AllowPositional.ERROR ) def __init__(self, **kwargs): pose_estimation_series, description = popargs('pose_estimation_series', 'description', kwargs) - original_videos, labeled_videos, = popargs('original_videos', 'labeled_videos', kwargs) + original_videos, labeled_videos = popargs('original_videos', 'labeled_videos', kwargs) + original_videos_series = popargs('original_videos_series', kwargs) + labeled_videos_series = popargs('labeled_videos_series', kwargs) dimensions, scorer = popargs('dimensions', 'scorer', kwargs) source_software, source_software_version = popargs('source_software', 'source_software_version', kwargs) nodes, edges = popargs('nodes', 'edges', kwargs) - # devices = popargs('devices', kwargs) - call_docval_func(super().__init__, kwargs) + + super().__init__(**kwargs) self.pose_estimation_series = pose_estimation_series self.description = description + + if original_videos is not None: + warnings.warn("The 'original_videos' field has been deprecated in version 0.2.0. Use " + "'original_videos_series' instead. The provided " + "file paths will be converted to ImageSeries objects where the 'external_file' field is set " + "to each file path.", + DeprecationWarning) + if original_videos_series is None: + warnings.warn( + "The provided file paths in 'original_videos' will be converted to ImageSeries objects where the " + "'external_file' field is set to each file path.", + DeprecationWarning + ) + original_videos_series = list() + for i, file_path in enumerate(original_videos): + image_series = ImageSeries( + name="original_video" + str(i), + format="external", + external_file=file_path, + dimension=dimensions[0] if dimensions is not None and dimensions[0] is not None else None, + ) + original_videos_series.append(image_series) self.original_videos = original_videos + self.original_videos_series = original_videos_series + + if labeled_videos is not None: + warnings.warn("The 'labeled_videos' field has been deprecated in version 0.2.0. Use " + "'labeled_videos_series' instead.", DeprecationWarning) + if labeled_videos_series is None: + warnings.warn( + "The provided file paths in 'labeled_videos' will be converted to ImageSeries objects where the " + "'external_file' field is set to each file path.", + DeprecationWarning + ) + labeled_videos_series = list() + for i, file_path in enumerate(labeled_videos): + image_series = ImageSeries( + name="labeled_video" + str(i), + format="external", + external_file=file_path, + dimension=dimensions[0] if dimensions is not None and dimensions[0] is not None else None, + ) + labeled_videos_series.append(image_series) self.labeled_videos = labeled_videos + self.labeled_videos_series = labeled_videos_series + + if dimensions is not None: + warnings.warn("The 'dimensions' field has been deprecated in version 0.2.0. " + "Use 'dimension' in 'original_videos' instead.", DeprecationWarning) self.dimensions = dimensions self.scorer = scorer self.source_software = source_software self.source_software_version = source_software_version self.nodes = nodes self.edges = edges - # self.devices = devices # TODO include calibration images for 3D estimates? - # if original_videos is not None and (devices is None or len(original_videos) != len(devices)): - # raise ValueError("The number of original videos should equal the number of camera devices.") - # if labeled_videos is not None and (devices is None or len(labeled_videos) != len(devices)): - # raise ValueError("The number of labeled videos should equal the number of camera devices.") - # if dimensions is not None and (devices is None or len(dimensions) != len(devices)): - # raise ValueError("The number of dimensions should equal the number of camera devices.") - # TODO validate nodes and edges correspondence, convert edges to uint diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index e0f8b6d..3a6669a 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -9,7 +9,7 @@ def main(): ns_builder = NWBNamespaceBuilder( doc='NWB extension to store pose estimation data', name='ndx-pose', - version='0.1.1', + version='0.2.0', author=['Ryan Ly', 'Ben Dichter', 'Alexander Mathis'], contact=['rly@lbl.gov', 'bdichter@lbl.gov', 'alexander.mathis@epfl.ch'], ) @@ -46,6 +46,7 @@ def main(): dtype='float32', dims=['num_frames'], shape=[None], + quantity='?', attributes=[ NWBAttributeSpec( name='definition', @@ -68,9 +69,46 @@ def main(): groups=[ NWBGroupSpec( neurodata_type_inc='PoseEstimationSeries', - doc='Estimated position data for each body part.', + doc=('Estimated position data for each body part. Deprecated in version 0.2.0. Use the "positions" ' + 'group instead.'), quantity='*', ), + NWBGroupSpec( + name="pose_estimates", + doc="Estimated position data for each body part.", + groups=[ + NWBGroupSpec( + neurodata_type_inc='PoseEstimationSeries', + doc='Estimated position data for each body part.', + quantity='*', + ) + ], + quantity='?', + ), + NWBGroupSpec( + name="original_videos_series", + doc="Links to the original video files.", + links=[ + NWBLinkSpec( + target_type='ImageSeries', + doc='Links to the original video files.', + quantity='*', + ), + ], + quantity='?', + ), + NWBGroupSpec( + name="labeled_videos_series", + doc="The labeled videos. The number of files should equal the number of original videos.", + datasets=[ + NWBDatasetSpec( + neurodata_type_inc="ImageSeries", + doc='The labeled videos. The number of files should equal the number of original videos.', + quantity='*', + ), + ], + quantity='?', + ), ], datasets=[ NWBDatasetSpec( @@ -81,7 +119,9 @@ def main(): ), NWBDatasetSpec( name='original_videos', - doc='Paths to the original video files. The number of files should equal the number of camera devices.', + doc=('Paths to the original video files. The number of files should equal the number of ' + 'camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in original_videos_series ' + 'instead'), dtype='text', dims=['num_files'], shape=[None], @@ -89,7 +129,9 @@ def main(): ), NWBDatasetSpec( name='labeled_videos', - doc='Paths to the labeled video files. The number of files should equal the number of camera devices.', + doc=('Paths to the labeled video files. The number of files should equal the number of ' + 'camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in labeled_videos_series ' + 'instead'), dtype='text', dims=['num_files'], shape=[None], @@ -97,7 +139,8 @@ def main(): ), NWBDatasetSpec( name='dimensions', - doc='Dimensions of each labeled video file.', + doc=('Dimensions of each labeled video file. Deprecated in version 0.2.0. ' + 'Use "dimension" in original_videos_series instead.'), dtype='uint8', dims=['num_files', 'width, height'], shape=[None, 2], @@ -142,14 +185,6 @@ def main(): quantity='?', ), ], - # TODO: collections of multiple links is currently buggy in PyNWB/HDMF - # links=[ - # NWBLinkSpec( - # target_type='Device', - # doc='Cameras used to record the videos.', - # quantity='*', - # ), - # ], ) new_data_types = [pose_estimation_series, pose_estimation] From 43029e587f94e49b4436c1bcd99ee535f7e4c35b Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 15 Feb 2023 11:56:59 -0800 Subject: [PATCH 2/4] Rename field --- spec/ndx-pose.extensions.yaml | 6 +++--- src/pynwb/ndx_pose/io/pose.py | 2 +- src/pynwb/ndx_pose/pose.py | 4 ++-- src/spec/create_extension_spec.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/ndx-pose.extensions.yaml b/spec/ndx-pose.extensions.yaml index 21f22e9..d6a0f3f 100644 --- a/spec/ndx-pose.extensions.yaml +++ b/spec/ndx-pose.extensions.yaml @@ -58,7 +58,7 @@ groups: - null doc: Paths to the original video files. The number of files should equal the number of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in original_videos_series - instead. + instead quantity: '?' - name: labeled_videos dtype: text @@ -68,7 +68,7 @@ groups: - null doc: Paths to the labeled video files. The number of files should equal the number of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in labeled_videos_series - instead. + instead quantity: '?' - name: dimensions dtype: uint8 @@ -120,7 +120,7 @@ groups: doc: Estimated position data for each body part. Deprecated in version 0.2.0. Use the "positions" group instead. quantity: '*' - - name: pose_estimates + - name: pose_estimation_series doc: Estimated position data for each body part. quantity: '?' groups: diff --git a/src/pynwb/ndx_pose/io/pose.py b/src/pynwb/ndx_pose/io/pose.py index 8fe7595..06884a1 100644 --- a/src/pynwb/ndx_pose/io/pose.py +++ b/src/pynwb/ndx_pose/io/pose.py @@ -25,7 +25,7 @@ def __init__(self, spec): # TODO if reading a file without the pose_estimates group, load the PoseEstimationSeries from the # main PoseEstimation group into the pose_estimation_series variable - pose_estimates_spec = self.spec.get_group('pose_estimates') + pose_estimates_spec = self.spec.get_group('pose_estimation_series') self.unmap(pose_estimates_spec) self.map_spec('pose_estimation_series', pose_estimates_spec.get_neurodata_type('PoseEstimationSeries')) diff --git a/src/pynwb/ndx_pose/pose.py b/src/pynwb/ndx_pose/pose.py index 3421b49..9a97b12 100644 --- a/src/pynwb/ndx_pose/pose.py +++ b/src/pynwb/ndx_pose/pose.py @@ -57,7 +57,7 @@ class PoseEstimation(MultiContainerInterface): __clsconf__ = [ { - # NOTE pose_estimation_series was remapped in version 0.2.0 to live under the pose_estimates subgroup + # NOTE pose_estimation_series was remapped in version 0.2.0 to live in the pose_estimation_series subgroup 'add': 'add_pose_estimation_series', 'get': 'get_pose_estimation_series', 'create': 'create_pose_estimation_series', @@ -85,7 +85,7 @@ class PoseEstimation(MultiContainerInterface): # custom mapper in ndx_pose.io.pose maps: # 'source_software' dataset, 'version' attribute to 'source_software_version' field - # 'pose_estimates' untyped group, 'PoseEstimationSeries' subgroup to 'pose_estimates' field + # 'pose_estimation_series' untyped group, 'PoseEstimationSeries' subgroup to 'pose_estimation_series' field # 'original_videos_series' untyped group, 'ImageSeries' subgroup to 'original_videos_series' field # 'labeled_videos_series' untyped group, 'ImageSeries' subgroup to 'labeled_videos_series' field diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 3a6669a..03da9c2 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -74,7 +74,7 @@ def main(): quantity='*', ), NWBGroupSpec( - name="pose_estimates", + name="pose_estimation_series", doc="Estimated position data for each body part.", groups=[ NWBGroupSpec( From faa8e003f1cace0eb45fe2f257417c12b14c0d74 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 15 Feb 2023 11:58:20 -0800 Subject: [PATCH 3/4] Fix doc --- spec/ndx-pose.extensions.yaml | 2 +- src/spec/create_extension_spec.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/ndx-pose.extensions.yaml b/spec/ndx-pose.extensions.yaml index d6a0f3f..28490a0 100644 --- a/spec/ndx-pose.extensions.yaml +++ b/spec/ndx-pose.extensions.yaml @@ -118,7 +118,7 @@ groups: groups: - neurodata_type_inc: PoseEstimationSeries doc: Estimated position data for each body part. Deprecated in version 0.2.0. - Use the "positions" group instead. + Place the PoseEstimationSeries in the "pose_estimation_series" group instead. quantity: '*' - name: pose_estimation_series doc: Estimated position data for each body part. diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 03da9c2..e57d6d8 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -69,8 +69,8 @@ def main(): groups=[ NWBGroupSpec( neurodata_type_inc='PoseEstimationSeries', - doc=('Estimated position data for each body part. Deprecated in version 0.2.0. Use the "positions" ' - 'group instead.'), + doc=('Estimated position data for each body part. Deprecated in version 0.2.0. Place the ' + 'PoseEstimationSeries in the "pose_estimation_series" group instead.'), quantity='*', ), NWBGroupSpec( From 5c6bce957d9b0d65667502cc93a705b87a3c1cd2 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 21 Nov 2023 23:52:10 -0800 Subject: [PATCH 4/4] Update pose.py --- src/pynwb/ndx_pose/io/pose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/ndx_pose/io/pose.py b/src/pynwb/ndx_pose/io/pose.py index 06884a1..787d492 100644 --- a/src/pynwb/ndx_pose/io/pose.py +++ b/src/pynwb/ndx_pose/io/pose.py @@ -31,7 +31,7 @@ def __init__(self, spec): original_videos_series_spec = self.spec.get_group('original_videos_series') self.unmap(original_videos_series_spec) - self.map_spec('original_videos_series', original_videos_series_spec.get_neurodata_type('ImageSeries')) + self.map_spec('original_videos_series', original_videos_series_spec.get_target_type('ImageSeries')) labeled_videos_series_spec = self.spec.get_group('labeled_videos_series') self.unmap(labeled_videos_series_spec)