Asdf read speed

dkist/conftest.py

@@ -302,10 +302,12 @@ def eit_dataset():
 @pytest.fixture
 def simple_tiled_dataset(dataset):
     datasets = [copy.deepcopy(dataset) for i in range(4)]
+    headers = []
     for ds in datasets:
         ds.meta["inventory"] = dataset.meta["inventory"]
+        headers.append(ds.headers)
     dataset_array = np.array(datasets).reshape((2,2))
-    return TiledDataset(dataset_array, dataset.meta["inventory"])
+    return TiledDataset(dataset_array, dataset.meta["inventory"], headers=headers)
 
 
 @pytest.fixture

dkist/dataset/dataset.py

@@ -100,7 +100,7 @@ class Dataset(NDCube):
     _file_manager = FileManagerDescriptor(default_type=FileManager)
 
     def __init__(self, data, wcs=None, uncertainty=None, mask=None, meta=None,
-                 unit=None, copy=False):
+                 unit=None, copy=False, is_tile=False):
 
         # Do some validation
         if (not isinstance(wcs, gwcs.WCS) and
@@ -123,6 +123,8 @@ def __init__(self, data, wcs=None, uncertainty=None, mask=None, meta=None,
         if "inventory" not in meta:
             raise ValueError("The meta dict must contain the inventory record.")
 
+        self._is_mosaic_tile = is_tile
+
         super().__init__(data, wcs, uncertainty=uncertainty, mask=mask, meta=meta,
                          unit=unit, copy=copy)
 

dkist/dataset/tests/test_tiled_dataset.py

@@ -56,6 +56,7 @@ def test_tiled_dataset_invalid_construction(dataset, dataset_4d):
         TiledDataset(np.array((dataset, ds2)), dataset.meta["inventory"])
 
 
+@pytest.mark.accept_cli_dataset
 def test_tiled_dataset_from_components(dataset):
     shape = (2, 2)
     file_managers = [dataset._file_manager] * 4
@@ -70,7 +71,7 @@ def test_tiled_dataset_from_components(dataset):
     for ds, fm, headers in zip(tiled_ds.flat, file_managers, header_tables):
         assert ds.files == fm
         assert ds.meta["inventory"] is inventory
-        assert ds.meta["headers"] is headers
+        assert (ds.meta["headers"] == headers).all()
 
 
 @figure_test

dkist/dataset/tiled_dataset.py

@@ -74,17 +74,20 @@ def _from_components(cls, shape, file_managers, wcses, header_tables, inventory)
         assert len(file_managers) == len(wcses) == len(header_tables)
 
         datasets = np.empty(len(file_managers), dtype=object)
-        for i, (fm, wcs, headers) in enumerate(zip(file_managers, wcses, header_tables)):
+        all_headers = vstack(header_tables)
+        for i, (fm, wcs) in enumerate(zip(file_managers, wcses)):
+            headers = all_headers[i*len(fm):(i+1)*len(fm)]
             meta = {"inventory": inventory, "headers": headers}
-            datasets[i] = Dataset(fm._generate_array(), wcs=wcs, meta=meta)
+            datasets[i] = Dataset(fm._generate_array(), wcs=wcs, meta=meta, is_tile=True)
             datasets[i]._file_manager = fm
         datasets = datasets.reshape(shape)
 
-        return cls(datasets, inventory)
+        return cls(datasets, inventory, all_headers)
 
-    def __init__(self, dataset_array, inventory=None):
+    def __init__(self, dataset_array, inventory=None, headers=None):
         self._data = np.array(dataset_array, dtype=object)
         self._inventory = inventory or {}
+        self.combined_headers = vstack(headers) if headers else None
         self._validate_component_datasets(self._data, inventory)
 
     def __contains__(self, x):
@@ -131,14 +134,6 @@ def inventory(self):
         """
         return self._inventory
 
-    @property
-    def combined_headers(self):
-        """
-        A single `astropy.table.Table` containing all the FITS headers for all
-        files in this dataset.
-        """
-        return vstack([ds.meta["headers"] for ds in self._data.flat])
-
     @property
     def shape(self):
         """

dkist/io/asdf/converters/dataset.py

@@ -63,6 +63,8 @@
                              "it was not constructed from a set of FITS files.")
         node = {}
         node["meta"] = dataset.meta or {}
+        if dataset._is_mosaic_tile and node.get("meta"):
+            node["meta"]["headers"] = None
         node["wcs"] = dataset.wcs
         node["data"] = dataset.files
         if dataset.unit:

dkist/io/asdf/converters/tiled_dataset.py

@@ -1,20 +1,33 @@
 from asdf.extension import Converter
+from astropy.table import Table, vstack
 
 
 class TiledDatasetConverter(Converter):
     tags = [
         "tag:dkist.nso.edu:dkist/tiled_dataset-0.1.0",
         "asdf://dkist.nso.edu/tags/tiled_dataset-1.0.0",
+        "asdf://dkist.nso.edu/tags/tiled_dataset-1.1.0",
     ]
     types = ["dkist.dataset.tiled_dataset.TiledDataset"]
 
     def from_yaml_tree(cls, node, tag, ctx):
         from dkist.dataset.tiled_dataset import TiledDataset
 
-        return TiledDataset(node["datasets"], node["inventory"])
+        for row in node["datasets"]:
+            for ds in row:
+                if ds:
+                    ds._is_mosaic_tile = True
+
+        if node.get("headers"):
+            headers = node["headers"]
+        else:
+            headers = vstack([Table(ds.headers) for ds in row for row in node["datasets"]])
+
+        return TiledDataset(node["datasets"], node["inventory"], headers)
 
     def to_yaml_tree(cls, tiled_dataset, tag, ctx):
         tree = {}
         tree["inventory"] = tiled_dataset._inventory
         tree["datasets"] = tiled_dataset._data.tolist()
+        tree["headers"] = tiled_dataset.combined_headers.as_array()
         return tree

dkist/io/asdf/entry_points.py

@@ -39,6 +39,8 @@ def get_extensions():
     dkist_converters = [FileManagerConverter(), DatasetConverter(), TiledDatasetConverter()]
     wcs_converters = [VaryingCelestialConverter(), CoupledCompoundConverter(), RavelConverter(), AsymmetricMappingConverter()]
     return [
+        ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-1.3.0",
+                                   converters=dkist_converters),
         ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-1.2.0",
                                    converters=dkist_converters),
         ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-1.1.0",

dkist/io/asdf/resources/manifests/dkist-1.3.0.yaml

@@ -0,0 +1,14 @@
+%YAML 1.1
+---
+id: asdf://dkist.nso.edu/manifests/dkist-1.2.0
+extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.3.0
+title: DKIST extension
+description: ASDF schemas and tags for DKIST classes.
+
+tags:
+  - schema_uri: "asdf://dkist.nso.edu/schemas/file_manager-1.0.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/file_manager-1.0.0"
+  - schema_uri: "asdf://dkist.nso.edu/schemas/dataset-1.2.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/dataset-1.2.0"
+  - schema_uri: "asdf://dkist.nso.edu/schemas/tiled_dataset-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/tiled_dataset-1.1.0"

dkist/io/asdf/resources/schemas/dataset-1.2.0.yaml

@@ -0,0 +1,50 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/dataset-1.2.0"
+
+title: |
+  A DKIST Level 1 Dataset object.
+description:
+  The container for a distributed dataset object.
+
+type: object
+properties:
+  data:
+    tag: "asdf://dkist.nso.edu/tags/file_manager-1.0.0"
+
+  wcs:
+    description: The coordinate system for the complete dataset.
+    tag: "tag:stsci.edu:gwcs/wcs-1.*"
+
+  mask:
+    tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
+
+  unit:
+    tag: "tag:stsci.edu:asdf/unit/unit-1.*"
+
+  meta:
+    description: Dataset metadata, describing the whole dataset.
+    type: object
+    properties:
+      headers:
+        description: A table of all the headers for the constituent files.
+        anyOf:
+          - tag: "tag:astropy.org:astropy/table/table-1.*"
+          - tag: 'tag:stsci.edu:asdf/core/ndarray-1.*'
+          - null: true
+
+      quality:
+        description: A copy of the quality report of these observations.
+        type: object
+
+      inventory:
+        description: A copy of the inventory record for this dataset.
+        type: object
+
+    required: [headers, inventory]
+    additionalProperties: true
+
+required: [data, wcs, unit]
+additionalProperties: true
+...

dkist/io/asdf/resources/schemas/tiled_dataset-1.1.0.yaml

@@ -0,0 +1,31 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/tiled_dataset-1.1.0"
+
+title: |
+  A DKIST Tiled Dataset object.
+description:
+  The container for a set of Dataset objects.
+
+type: object
+properties:
+  datasets:
+    description: A nested structure of Dataset objects
+    type: array
+    items:
+      type: array
+      items:
+        - tag: "asdf://dkist.nso.edu/tags/dataset-1.*"
+  inventory:
+    description: A copy of the inventory record for this dataset.
+    type: object
+  headers:
+    description: A table of all the headers for the constituent files.
+    anyOf:
+      - tag: "tag:astropy.org:astropy/table/table-1.*"
+      - tag: 'tag:stsci.edu:asdf/core/ndarray-1.*'
+
+required: [datasets, inventory, headers]
+additionalProperties: false
+...

dkist/io/level_1_dataset_schema.yaml

@@ -17,8 +17,10 @@ properties:
       - $ref: "asdf://dkist.nso.edu/schemas/dataset-0.3.0"
       - $ref: "asdf://dkist.nso.edu/schemas/dataset-1.0.0"
       - $ref: "asdf://dkist.nso.edu/schemas/dataset-1.1.0"
+      - $ref: "asdf://dkist.nso.edu/schemas/dataset-1.2.0"
       - $ref: "asdf://dkist.nso.edu/schemas/tiled_dataset-0.1.0"
       - $ref: "asdf://dkist.nso.edu/schemas/tiled_dataset-1.0.0"
+      - $ref: "asdf://dkist.nso.edu/schemas/tiled_dataset-1.1.0"
 
 required: [dataset]
 additionalProperties: true