Merge branch 'first_disp' into create_sw_tile

CHANGELOG.md

@@ -12,4 +12,5 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 * Design document with implementation details for tile generation
 * Utility for identifying California test dataset
 * Utility for generating metadata tile images
+* Utility for generating temporary S3 credentials for [NASA Thin Egress Apps](https://nasa.github.io/cumulus/docs/deployment/thin_egress_app/)
 

README.md

@@ -2,8 +2,38 @@
 
 Package for [OPERA DISP](https://www.jpl.nasa.gov/go/opera/products/disp-product-suite/) Tile Map Server (TMS) creation.
 
-## Scripts
-`generate_coh_tiles.py`: Generate a test dataset with a similar extent and result to the OPERA DISP dataset for testing.
+## Installation
+See [Develop Setup](#developer-setup)
+
+## Credentials
+This repository assumes that you have credentials for `urs.earthdata.nasa.gov` (Earthdata login) and `uat.urs.earthdata.nasa.gov` (Earthdata login UAT) configured in your `netrc` file.
+
+For instructions on setting up your Earthdata login (and Earthdata login UAT) via a `.netrc` file, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started).
+
+## Usage
+### Create a frame metadata tile
+These tiles serve as the foundation for the creation of all other Tile Map Server datasets. More details on the structure of these datasets can be found in the [Design.md](https://github.com/ASFHyP3/OPERA-DISP-TMS/blob/develop/Design.md) document.
+
+> [!WARNING]
+> Products in ASF's Cumulus UAT environment are not publicly accessible. To run this workflow using data hosted in ASF's Cumulus UAT environment you first need to generate temporary S3 access credentials while on the NASA or ASF DAAC VPN, then copy the created credentials to an AWS us-west-2 compute environment at the location `~/LOCAL_OPERA-DISP-TMS_INSTALL_PATH/src/opera_disp_tms/credentials.json`. See [Create temp S3 credentials](#create-temp-s3-credentials) for more details.
+
+The `generate_frame_tile` CLI command can be used to generate a frame metadata tile:
+```bash
+generate_frame_tile -122 37 -121 38 --ascending
+```
+Where `-122 37 -121 38` is a desired bounding box in **integer** `minx, miny, max, maxy` longitude/latitude values, and `--ascending` specifies which orbit direction you want to generate a frame metadata tile for (`--ascending` for ascending, omit for descending).
+
+For TMS generation ASF will be using 1x1 degree tiles.
+
+### Create temp S3 credentials
+The `get_tmp_s3_creds` CLI command can be used to generate temporary S3 access credentials for a NASA Cumulus Thin Egress App (TEA):
+```bash
+get_tmp_s3_creds --tea-url https://cumulus-test.asf.alaska.edu/s3credentials --creds-path ./creds.json
+```
+Where `--tea-url` is the TEA S3 endpoint you want to generate credentials for, and `--creds-path` is the path to save the credentials to.
+When called with no arguments (`get_tmp_s3_creds`) the CLI commands defaults to the ASF Cumulus UAT's TEA, and to the credentials path `~/LOCAL_OPERA-DISP-TMS_INSTALL_PATH/src/opera_disp_tms/credentials.json`.
+
+**These temporary credentials expire every hour and will need to be regenerated accordingly.**
 
 ## Developer Setup
 1. Ensure that conda is installed on your system (we recommend using [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) to reduce setup times).

pyproject.toml

@@ -50,6 +50,7 @@ Documentation = "https://github.com/ASFHyP3/opera-disp-tms/tree/develop?tab=read
 [project.scripts]
 generate_frame_tile = "opera_disp_tms.generate_frame_tile:main"
 generate_sw_disp_tile = "opera_disp_tms.generate_sw_disp_tile:main"
+get_tmp_s3_creds = "opera_disp_tms.tmp_s3_access:main"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

src/opera_disp_tms/find_california_dataset.py

@@ -105,8 +105,10 @@ def from_search_result(cls, search_product: asf.ASFProduct):
         scene_name = search_product.properties['sceneName']
         frame = int(scene_name.split('_')[4][1:])
         orbit_pass = 'ASCENDING' if frame in ASC_FRAMES else 'DESCENDING'
-        url = f'https://cumulus-test.asf.alaska.edu/RTC/OPERA-S1/OPERA_L3_DISP-S1_PROVISIONAL_V0/{scene_name}/{scene_name}.nc'
-        s3_uri = f's3://asf-cumulus-test-opera-products/OPERA_L3_DISP-S1_PROVISIONAL_V0/{scene_name}/{scene_name}.nc'
+        url_base = 'https://cumulus-test.asf.alaska.edu/RTC/OPERA-S1/OPERA_L3_DISP-S1_PROVISIONAL_V0'
+        url = f'{url_base}/{scene_name}/{scene_name}.nc'
+        s3_base = 's3://asf-cumulus-test-opera-products/OPERA_L3_DISP-S1_PROVISIONAL_V0'
+        s3_uri = f'{s3_base}/OPERA_L3_DISP-S1_PROVISIONAL_V0/{scene_name}/{scene_name}.nc'
         reference_date = datetime.strptime(scene_name.split('_')[-4], DATE_FORMAT)
         secondary_date = datetime.strptime(scene_name.split('_')[-3], DATE_FORMAT)
         creation_date = datetime.strptime(scene_name.split('_')[-1], DATE_FORMAT)
@@ -129,7 +131,7 @@ def from_tuple(cls, tup: Tuple):
             frame=int(frame),
             orbit_pass=orbit_pass,
             url=url,
-            s3_uri = s3_uri,
+            s3_uri=s3_uri,
             reference_date=datetime.strptime(reference_date, DATE_FORMAT),
             secondary_date=datetime.strptime(secondary_date, DATE_FORMAT),
             creation_date=datetime.strptime(creation_date, DATE_FORMAT),

src/opera_disp_tms/frames.py

@@ -1,12 +1,13 @@
 import sqlite3
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterable, Optional, Union
+from typing import Iterable, Optional, Tuple
 
-import requests
 from shapely import from_wkt
 from shapely.geometry import Polygon, box
 
+from opera_disp_tms.utils import download_file
+
 
 DB_PATH = Path(__file__).parent / 'opera-s1-disp-0.5.0.post1.dev20-2d.gpkg'
 
@@ -34,60 +35,43 @@ def from_row(cls, row):
         )
 
 
-def download_file(
-    url: str,
-    download_path: Union[Path, str] = '.',
-    chunk_size=10 * (2**20),
-) -> Path:
-    """Download a file without authentication.
-
-    Args:
-        url: URL of the file to download
-        download_path: Path to save the downloaded file to
-        chunk_size: Size to chunk the download into
-
-    Returns:
-        download_path: The path to the downloaded file
-    """
-    session = requests.Session()
-
-    with session.get(url, stream=True) as s:
-        s.raise_for_status()
-        with open(download_path, 'wb') as f:
-            for chunk in s.iter_content(chunk_size=chunk_size):
-                if chunk:
-                    f.write(chunk)
-    session.close()
-
-
-def download_frame_db() -> Path:
+def download_frame_db(db_path: Path = DB_PATH) -> Path:
     """Download the OPERA burst database.
     Currently using a version created using opera-adt/burst_db v0.5.0, but hope to switch to ASF-provide source.
 
+    Args:
+        db_path: Path to save the database file to
+
     Returns:
         Path to the downloaded database
     """
-    if DB_PATH.exists():
-        return DB_PATH
+    if db_path.exists():
+        return db_path
 
     print('Downloading frame database...')
-    url = f'https://opera-disp-tms-dev.s3.us-west-2.amazonaws.com/{DB_PATH.name}'
-    download_file(url, DB_PATH)
+    url = f'https://opera-disp-tms-dev.s3.us-west-2.amazonaws.com/{db_path.name}'
+    download_file(url, db_path)
 
 
-def intersect(
+def build_query(
     bbox: Iterable[int],
     orbit_pass: Optional[str] = None,
     is_north_america: Optional[bool] = None,
     is_land: Optional[bool] = None,
-) -> Iterable[Frame]:
-    """Query for frames intersecting a given bounding box or WKT geometry, optionally filtering by orbit pass."""
-    if orbit_pass and orbit_pass not in ['ASCENDING', 'DESCENDING']:
-        raise ValueError('orbit_pass must be either "ASCENDING" or "DESCENDING"')
+) -> Tuple:
+    """Build a query for identifying OPERA frames intersecting a given bounding box,
+    optionally filtering by more fields.
 
-    download_frame_db()
-    wkt_str = box(*bbox).wkt
+    Args:
+        bbox: Bounding box to query for
+        orbit_pass: Filter by orbit pass (either 'ASCENDING' or 'DESCENDING')
+        is_north_america: Filter by whether the frame intersects North America
+        is_land: Filter by whether the frame intersects land
 
+    Returns:
+        Tuple with the query and parameters to pass to the query
+    """
+    wkt_str = box(*bbox).wkt
     query = """
 WITH given_geom AS (
     SELECT PolygonFromText(?, 4326) as g
@@ -129,6 +113,32 @@ def intersect(
         query += ' AND is_land = ?'
         params.append(int(is_land))
 
+    return query, params
+
+
+def intersect(
+    bbox: Iterable[int],
+    orbit_pass: Optional[str] = None,
+    is_north_america: Optional[bool] = None,
+    is_land: Optional[bool] = None,
+) -> Iterable[Frame]:
+    """Query OPERA frame database to obtain frames intersecting a given bounding box,
+    optionally filtering by more fields.
+
+    Args:
+        bbox: Bounding box to query for
+        orbit_pass: Filter by orbit pass (either 'ASCENDING' or 'DESCENDING')
+        is_north_america: Filter by whether the frame intersects North America
+        is_land: Filter by whether the frame intersects land
+
+    Returns:
+        Iterable of Frame objects matching the input criteria
+    """
+    if orbit_pass and orbit_pass not in ['ASCENDING', 'DESCENDING']:
+        raise ValueError('orbit_pass must be either "ASCENDING" or "DESCENDING"')
+
+    download_frame_db()
+    query, params = build_query(bbox, orbit_pass, is_north_america, is_land)
     with sqlite3.connect(DB_PATH) as con:
         con.enable_load_extension(True)
         con.load_extension('mod_spatialite')

src/opera_disp_tms/generate_frame_tile.py

@@ -11,7 +11,7 @@
 
 from opera_disp_tms.find_california_dataset import find_california_dataset
 from opera_disp_tms.frames import Frame, intersect
-from opera_disp_tms.tmp_aws import get_credentials
+from opera_disp_tms.tmp_s3_access import get_credentials
 
 
 gdal.UseExceptions()
@@ -23,6 +23,17 @@ def check_bbox_all_int(bbox: Iterable[int]):
 
 
 def create_product_name(parts: Iterable[str], orbit_pass: str, bbox: Iterable[int]) -> str:
+    """Create a product name for a frame metadata tile
+    Should be in the format: metadata_ascendign_N02E001_N04E003
+
+    Args:
+        parts: The parts of the product name
+        orbit_pass: The orbit pass of the frames
+        bbox: The bounding box of the frames
+
+    Returns:
+        The product name
+    """
     check_bbox_all_int(bbox)
 
     def lat_string(lat):
@@ -56,7 +67,6 @@ def reorder_frames(frame_list: Iterable[Frame], order_by: str = 'frame_number')
         if order_by == 'frame_number':
             metric = max([x.frame_id for x in frames])
         elif order_by == 'west_most':
-            # TODO: check anti-meridian crossing
             metric = max([x.geom.bounds[0] for x in frames])
         else:
             raise ValueError('Invalid order_by parameter. Please use "frame_number" or "west_most".')