LDDS-528 (#135)

Co-authored-by: Christine Brown-Stevenson <[email protected]>

CHANGELOG.md

@@ -5,6 +5,9 @@ versioning. Rather than a static releases, this repository contains of a number
 of regression tests that are each semi-independent.  This CHANGELOG file should be used
 to document pull requests to this repository.
 
+## 2025-02-12 ([#135](https://github.com/nasa/harmony-regression-tests/pull/135))
+- Added band subsetting test case to subset-band-name tests
+- Added Production IDs to geoloco tests.
 
 ## 2025-02-05 ([#134](https://github.com/nasa/harmony-regression-tests/pull/134))
 

test/geoloco/Geoloco_Regression.ipynb

@@ -74,7 +74,8 @@
    "source": [
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'\n",
     "# harmony_host_url = 'https://harmony.sit.earthdata.nasa.gov'\n",
-    "# harmony_host_url = 'http://localhost:3000'"
+    "# harmony_host_url = 'http://localhost:3000'\n",
+    "# harmony_host_url = 'https://harmony.earthdata.nasa.gov'"
    ]
   },
   {
@@ -150,6 +151,27 @@
     "    'downscale_size': [2, 2],\n",
     "}\n",
     "\n",
+    "mod021km_production_info = {\n",
+    "    'collection': Collection(id='C1378227407-LAADS'),\n",
+    "    'granule_id': 'G2576316022-LAADS',\n",
+    "    'variable': ['EV_250_Aggr1km_RefSB'],\n",
+    "    'downscale_size': [0.01802, 0.01802],\n",
+    "}\n",
+    "\n",
+    "mod35l2_production_info = {\n",
+    "    'collection': Collection(id='C1443561895-LAADS'),\n",
+    "    'granule_id': 'G2692389759-LAADS',\n",
+    "    'variable': ['Cloud_Mask'],\n",
+    "    'downscale_size': [0.01802, 0.01802],\n",
+    "}\n",
+    "\n",
+    "mod08d3_production_info = {\n",
+    "    'collection': Collection(id='C1443727145-LAADS'),\n",
+    "    'granule_id': 'G2576607791-LAADS',\n",
+    "    'variable': ['Aerosol_Optical_Depth_Land_Ocean_Mean'],\n",
+    "    'downscale_size': [2, 2],\n",
+    "}\n",
+    "\n",
     "geo_proj4_string = '+a=6378137.0 +b=6356752.3142451793 +no_defs +proj=latlong'\n",
     "\n",
     "resampling_string = 'NN'\n",
@@ -186,16 +208,19 @@
     "    Environment.LOCAL: mod021km_non_production_info,\n",
     "    Environment.UAT: mod021km_non_production_info,\n",
     "    Environment.SIT: mod021km_non_production_info,\n",
+    "    Environment.PROD: mod021km_production_info,\n",
     "}\n",
     "mod35l2_geoloco_env = {\n",
     "    Environment.LOCAL: mod35l2_non_production_info,\n",
     "    Environment.UAT: mod35l2_non_production_info,\n",
     "    Environment.SIT: mod35l2_non_production_info,\n",
+    "    Environment.PROD: mod35l2_production_info,\n",
     "}\n",
     "mod08d3_geoloco_env = {\n",
     "    Environment.LOCAL: mod08d3_non_production_info,\n",
     "    Environment.UAT: mod08d3_non_production_info,\n",
     "    Environment.SIT: mod08d3_non_production_info,\n",
+    "    Environment.PROD: mod08d3_production_info,\n",
     "}\n",
     "\n",
     "if harmony_environment in mod021km_geoloco_env:\n",

test/geoloco/version.txt

@@ -1 +1 @@
-1.0.3
+1.0.4

test/subset-band-name/SubsetBandName_Regression.ipynb

@@ -37,7 +37,7 @@
     "    submit_and_download,\n",
     ")\n",
     "\n",
-    "from harmony import Client, Collection, Environment, Request\n",
+    "from harmony import Client, Collection, Environment, Request, Dimension\n",
     "\n",
     "from subset_band_name_utitlities import (\n",
     "    remove_results_files,\n",
@@ -143,10 +143,11 @@
     "    'variable': ['EV_250_Aggr500_RefSB'],\n",
     "}\n",
     "\n",
-    "file_indicators = {'MOD02HKM': 'MOD02HKM.EV_250_Aggr500_RefSB_output.hdf'}\n",
+    "file_indicators = {'MOD02HKM': 'MOD02HKM.A2023309.2305.061.2023310073403_subsetted.hdf'}\n",
     "\n",
     "reference_data = {\n",
-    "    'MOD02HKM': 'reference_data/MOD02HKM.A2023309.2305.061.pscs_001729112207.hdf'\n",
+    "    'variable_subset': 'reference_data/MOD02HKM.A2023309.2305.061.pscs_001729112207.hdf',\n",
+    "    'band_subset': 'reference_data/MOD02HKM.A2023309.2305.061.2023310073403.bscs_000502329841.hdf',\n",
     "}"
    ]
   },
@@ -191,7 +192,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mod02hkm_test = True\n",
+    "mod02hkm_variable_test = True\n",
     "\n",
     "if mod02hkm_subsetbandname_info is not None:\n",
     "\n",
@@ -204,14 +205,70 @@
     "    submit_and_download(harmony_client, mod02hkm_request, file_indicators['MOD02HKM'])\n",
     "\n",
     "    if not compare_data(\n",
-    "        reference_data['MOD02HKM'], file_indicators['MOD02HKM'], 'EV_250_Aggr500_RefSB'\n",
+    "        reference_data['variable_subset'],\n",
+    "        file_indicators['MOD02HKM'],\n",
+    "        'EV_250_Aggr500_RefSB',\n",
     "    ):\n",
     "        print_error('MOD02HKM data mismatch.')\n",
-    "        mod02hkm_test = False\n",
+    "        mod02hkm_variable_test = False\n",
     "\n",
     "    remove_results_files()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "abcf8f0d-6c00-4588-8237-1fd6f05f84ad",
+   "metadata": {},
+   "source": [
+    "## Band Subsetting Test\n",
+    "In the cell below, band subsetting is tested by by subsetting bands 1-3 MOD02HKM. \n",
+    "The ouput should yield:\n",
+    "- EV_250_Aggr1km_RefSB: 1,2\n",
+    "- EV_250_Aggr1km_RefSB_Uncert_Indexes: 1,2\n",
+    "- EV_250_Aggr1km_RefSB_Samples_Used: 1,2\n",
+    "- Band_250M: 1,2\n",
+    "- EV_500_Aggr1km_RefSB: 1\n",
+    "- EV_500_Aggr1km_RefSB_Uncert_Indexes: 1\n",
+    "- EV_500_Aggr1km_RefSB_Samples_Used: 1\n",
+    "- Band_500M: 1\n",
+    "\n",
+    "The results are then compared against the reference data file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d3db34f-43b3-467a-83c1-bda0d49ef2ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mod02hkm_band_test = True\n",
+    "\n",
+    "if mod02hkm_subsetbandname_info is not None:\n",
+    "\n",
+    "    mod02hkm_request = Request(\n",
+    "        collection=mod02hkm_subsetbandname_info['collection'],\n",
+    "        granule_id=mod02hkm_subsetbandname_info['granule_id'],\n",
+    "        dimensions=[Dimension(name='band', min=1, max=3)],\n",
+    "    )\n",
+    "\n",
+    "    submit_and_download(harmony_client, mod02hkm_request, file_indicators['MOD02HKM'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8989f028-adab-49e8-9758-6a1ea4f2d570",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not compare_data(reference_data['band_subset'], file_indicators['MOD02HKM'], None):\n",
+    "    print_error('MOD02HKM data mismatch.')\n",
+    "    mod02hkm_band_test = False\n",
+    "\n",
+    "remove_results_files()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "8abafe2f-2183-43aa-909c-fc845eb8b04a",
@@ -227,7 +284,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "subsetbandname_tests = mod02hkm_test\n",
+    "subsetbandname_tests = mod02hkm_variable_test and mod02hkm_band_test\n",
     "\n",
     "if mod02hkm_subsetbandname_info is not None:\n",
     "    if subsetbandname_tests:\n",

test/subset-band-name/subset_band_name_utitlities.py

@@ -4,6 +4,8 @@
 
 import os
 from pyhdf.SD import SD, SDC
+from pyhdf.HDF import HDF
+from pyhdf.VS import VS
 import numpy
 
 
@@ -18,13 +20,35 @@ def get_sds_data(file: str, sds_name: str):
         sds_data = dataset.get()
 
     file_sd.end()
+
     return sds_data
 
 
+def get_all_sds_names(file: str):
+    """Get all SDS names from HDF-4 file"""
+    file_sd = SD(file, SDC.READ)
+    datasets = file_sd.datasets()
+    file_sd.end()
+
+    return datasets.keys()
+
+
+def get_vdata(file: str, vdata_name: str):
+    """Retrieve VData frm HDF-4 file"""
+    file_hdf = HDF(file)
+    file_vs = VS(file_hdf)
+    vd = file_vs.attach(vdata_name)
+    vdata = vd[:]
+    vd.detach()
+    file_vs.end()
+    file_hdf.close()
+
+    return vdata
+
+
 def remove_results_files() -> None:
     """Remove all HDF-4 files downloaded during the Subset-Band-Name
     regression tests.
-
     """
     directory_files = os.listdir()
 
@@ -35,7 +59,32 @@ def remove_results_files() -> None:
 
 def compare_data(reference_file: str, test_file: str, sds_name: str) -> bool:
     """Compares two data dimension sizes"""
-    reference_data = get_sds_data(reference_file, sds_name)
-    test_data = get_sds_data(test_file, sds_name)
 
-    return numpy.array_equal(reference_data, test_data)
+    # Compare one variable at a time
+    if sds_name is not None:
+        reference_data = get_sds_data(reference_file, sds_name)
+        test_data = get_sds_data(test_file, sds_name)
+
+        return numpy.array_equal(reference_data, test_data)
+
+    # Compare all variables and select VData in files
+    else:
+        reference_sds_names = get_all_sds_names(reference_file)
+
+        for sds_name in reference_sds_names:
+            reference_data = get_sds_data(reference_file, sds_name)
+            test_data = get_sds_data(test_file, sds_name)
+
+            if not numpy.array_equal(reference_data, test_data):
+                return False
+
+        vdata_names = ['Band_250M', 'Band_500M']
+
+        for name in vdata_names:
+            reference_vdata = get_vdata(reference_file, name)
+            test_vdata = get_vdata(test_file, name)
+
+            if not numpy.array_equal(reference_vdata, test_vdata):
+                return False
+
+        return True

test/subset-band-name/version.txt

@@ -1 +1 @@
-0.0.1
+0.0.2