WIP expand Variables class to handle s3 urls from NSIDC

icepyx/core/read.py

@@ -269,10 +269,6 @@ class Read:
     data_source : string, List
         A string or list which specifies the files to be read. The string can be either: 1) the path of a single file 2) the path to a directory or 3) a [glob string](https://docs.python.org/3/library/glob.html).
         The List must be a list of strings, each of which is the path of a single file.
-
-    product : string
-        ICESat-2 data product ID, also known as "short name" (e.g. ATL03).
-        Available data products can be found at: https://nsidc.org/data/icesat-2/data-sets
         **Deprecation warning:** This argument is no longer required and will be deprecated in version 1.0.0. The dataset product is read from the file metadata.
 
     filename_pattern : string, default None
@@ -289,6 +285,9 @@ class Read:
     glob_kwargs : dict, default {}
         Additional arguments to be passed into the [glob.glob()](https://docs.python.org/3/library/glob.html#glob.glob)function
 
+    glob_kwargs : dict, default {}
+        Additional arguments to be passed into the [glob.glob()](https://docs.python.org/3/library/glob.html#glob.glob)function
+
     out_obj_type : object, default xarray.Dataset
         The desired format for the data to be read in.
         Currently, only xarray.Dataset objects (default) are available.
@@ -320,10 +319,10 @@ class Read:
 
     # ----------------------------------------------------------------------
     # Constructors
-
+    
     def __init__(
         self,
-        data_source=None,
+        data_source=None,  # DevNote: Make this a required arg when catalog is removed
         product=None,
         filename_pattern=None,
         catalog=None,
@@ -336,7 +335,7 @@ def __init__(
                 "The `catalog` argument has been deprecated and intake is no longer supported. "
                 "Please use the `data_source` argument to specify your dataset instead."
             )
-
+            
         if data_source is None:
             raise ValueError("data_source is a required arguemnt")
 
@@ -381,7 +380,6 @@ def __init__(
         product_dict = {}
         for file_ in self._filelist:
             product_dict[file_] = self._extract_product(file_)
-
         # Raise warnings or errors for multiple products or products not matching the user-specified product
         all_products = list(set(product_dict.values()))
         if len(all_products) > 1:
@@ -425,7 +423,6 @@ def __init__(
                 " metadata {self._product}",
                 stacklevel=2,
             )
-
         if out_obj_type is not None:
             print(
                 "Output object type will be an xarray DataSet - "
@@ -461,6 +458,20 @@ def vars(self):
             )
 
         return self._read_vars
+
+    @property
+    def filelist(self):
+        """
+        Return the list of files represented by this Read object.
+        """
+        return self._filelist
+
+    @property
+    def product(self):
+        """
+        Return the product associated with the Read object.
+        """
+        return self._product
 
     @property
     def filelist(self):
@@ -478,7 +489,21 @@ def product(self):
 
     # ----------------------------------------------------------------------
     # Methods
-
+
+    @staticmethod
+    def _extract_product(filepath):
+        """
+        Read the product type from the metadata of the file. Return the product as a string.
+        """
+        with h5py.File(filepath, 'r') as f:
+            try: 
+                product = f.attrs['short_name'].decode()
+                product = is2ref._validate_product(product)
+            # TODO test that this is the proper error
+            except KeyError:
+                raise 'Unable to parse the product name from file metadata'
+        return product
+
     @staticmethod
     def _extract_product(filepath):
         """

icepyx/core/variables.py

@@ -1,4 +1,3 @@
-import numpy as np
 import os
 import pprint
 
@@ -27,7 +26,7 @@ class Variables(EarthdataAuthMixin):
     Parameters
     ----------
     vartype : string
-        One of ['order', 'file'] to indicate the source of the input variables.
+        One of ['order', 'file', 'nsidc-s3'] to indicate the source of the input variables.
         This field will be auto-populated when a variable object is created as an
         attribute of a query object.
     avail : dictionary, default None
@@ -75,6 +74,14 @@ def __init__(
         elif self._vartype == "file":
             # DevGoal: check that the list or string are valid dir/files
             self.path = path
+        elif self._vartype == "nsidc-s3":
+            # Grab metadata from s3 path
+            template = ('s3://nsidc-cumulus-prod-protected/ATLAS/{product}/{version}/' 
+            '{year}/{month}/{day}/{filename}')
+            s3_pathinfo = parse.parse(template, path)
+            self._version = s3_pathinfo['version']
+            self._product = s3_pathinfo['product']
+            self.path = path
 
     # @property
     # def wanted(self):
@@ -101,7 +108,7 @@ def avail(self, options=False, internal=False):
         #         return self._avail
         # else:
         if not hasattr(self, "_avail") or self._avail == None:
-            if self._vartype == "order":
+            if self._vartype in ["order", "nsidc-s3"]:
                 self._avail = is2ref._get_custom_options(
                     self.session, self.product, self._version
                 )["variables"]