Skip to content

Commit

Permalink
Add Landsat collections (AdrienWehrle#89)
Browse files Browse the repository at this point in the history 
* add LANDSAT collections

* add landsat data sets for download

* include source modifications in tests

* add link to all available data collections in README

* retry codecov
  • Loading branch information
@AdrienWehrle
AdrienWehrle authored Mar 23, 2023
1 parent d4d6584 commit 2f887ba
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 58 deletions.
4 changes: 2 additions & 2 deletions README.org
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

* earthspy 🛰️ :earth_africa: :earth_americas: :earth_asia: :moon:

=earthspy= is a wrapper around methods for the download of satellite data offered in the [[https://github.com/sentinel-hub/sentinelhub-py][sentinelhub Python package]]. This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications. Just like a spy!
=earthspy= is a wrapper around methods for the download of satellite data offered in the [[https://github.com/sentinel-hub/sentinelhub-py][sentinelhub Python package]]. This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications.

Some useful capabilities:
- Data download in multiprocessing
Expand Down Expand Up @@ -57,7 +57,7 @@ At present =earthspy= can be run by users within a couple of lines of Python cod
- set query parameters including Sentinel Hub API variables and =earthspy= additional ones (mainly for download efficiency)
- send request

Below is presented a simple application of =earthspy= for the download of Sentinel-2 data download around Ilulissat, Greenland for few days in August 2019 using a True Color custom script available on Sentinel Hub's [[https://custom-scripts.sentinel-hub.com][custom script online repository]].
Below is presented a simple application of =earthspy= for the download of Sentinel-2 data download around Ilulissat, Greenland for few days in August 2019 using a True Color custom script available on Sentinel Hub's [[https://custom-scripts.sentinel-hub.com][custom script online repository]]. All other available data collections can be found [[https://sentinelhub-py.readthedocs.io/en/latest/examples/data_collections.html][here]].

#+begin_src python

Expand Down
72 changes: 21 additions & 51 deletions earthspy/earthspy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def set_query_parameters(
self.data_collection_str = data_collection
self.get_data_collection()
self.get_satellite_name()
self.get_data_collection_resolution()
self.get_raw_data_collection_resolution()

# set number of cores
self.set_number_of_cores(nb_cores)
Expand Down Expand Up @@ -212,7 +212,10 @@ def get_available_data(self) -> list:
self.catalog_config = self.config.copy()

# set Sentinel-3 specific base URL of deployment
if self.data_collection_str == "SENTINEL3_OLCI":
if (
self.data_collection_str == "SENTINEL3_OLCI"
or self.data_collection_str == "LANDSAT_OT_L2"
):
self.catalog_config.sh_base_url = shb.DataCollection[
self.data_collection_str
].service_url
Expand Down Expand Up @@ -260,22 +263,26 @@ def get_satellite_name(self) -> str:

return self.satellite

def get_data_collection_resolution(self) -> int:
"""Get raw data collection resolution.
def get_raw_data_collection_resolution(self) -> int:
"""Get lowest raw data collection resolution possible (then
refined at the download stage).
:return: Data collection resolution.
:rtype: int
"""

# set default satellite resolution
if self.satellite == "SENTINEL1":
self.data_collection_resolution = 5
self.raw_data_collection_resolution = 5
elif self.satellite == "SENTINEL2":
self.data_collection_resolution = 10
self.raw_data_collection_resolution = 10
elif self.satellite == "SENTINEL3":
self.data_collection_resolution = 300
self.raw_data_collection_resolution = 300
elif self.satellite == "LANDSAT":
self.raw_data_collection_resolution = 15
else:
self.data_collection_resolution = 1000
self.raw_data_collection_resolution = 1000

if self.verbose:
print(
Expand All @@ -284,7 +291,7 @@ def get_data_collection_resolution(self) -> int:
"be refined."
)

return self.data_collection_resolution
return self.raw_data_collection_resolution

def set_number_of_cores(self, nb_cores) -> int:
"""Set number of cores if not specificed by user.
Expand Down Expand Up @@ -322,7 +329,6 @@ def get_date_range(

# if an integer, create a datetimeIndex with the number of days from present date
if isinstance(time_interval, int):

# keep time_interval positive
if time_interval < 0:
time_interval *= -1
Expand Down Expand Up @@ -373,7 +379,6 @@ def get_bounding_box(self, bounding_box: Union[list, str]) -> shb.geometry.BBox:

# if a list, set Sentinel Hub BBox wit bounding_box
if isinstance(bounding_box, list):

# create Sentinel Hub BBox
self.bounding_box = shb.BBox(bbox=bounding_box, crs=shb.CRS.WGS84)

Expand All @@ -382,12 +387,10 @@ def get_bounding_box(self, bounding_box: Union[list, str]) -> shb.geometry.BBox:

# if a string, extract bounding box from corresponding GEOJSON file
elif isinstance(bounding_box, str):

# list all available GEOJSON files
json_files = glob.glob("data/*.geojson")

for json_file in json_files:

# open GEOJSON file
with open(json_file) as f:
area_object = json.load(f)
Expand Down Expand Up @@ -434,7 +437,6 @@ def get_store_folder(self, store_folder: Union[str, None]) -> str:

# set Downloads folder as default main store folder
if not store_folder:

store_folder = f"{Path.home()}/Downloads/earthspy"

# set earthspy folder as default sub store folder
Expand Down Expand Up @@ -488,7 +490,7 @@ def get_max_resolution(self) -> Union[int, None]:
self.convert_bounding_box_coordinates()

# set an array of trial resolutions
trial_resolutions = np.arange(self.data_collection_resolution, 10000)
trial_resolutions = np.arange(self.raw_data_collection_resolution, 10000)

# compute x and y dimensions in meters
dx = np.abs(self.bounding_box_UTM_list[2] - self.bounding_box_UTM_list[0])
Expand All @@ -499,15 +501,13 @@ def get_max_resolution(self) -> Union[int, None]:
nb_ypixels = (dy / trial_resolutions).astype(int)

try:

# get max resolution while staying below Sentinel Hub max dimensions
max_resolution = trial_resolutions[
(nb_xpixels < 2500) & (nb_ypixels < 2500)
][1]

# identify why max resolution was not found in trial resolutions
except IndexError:

# find the origin
if np.sum(nb_xpixels < 2500) == 0 and np.sum(nb_ypixels < 2500) == 0:
origin = "x and y"
Expand Down Expand Up @@ -540,11 +540,10 @@ def set_correct_resolution(self) -> int:
if not self.resolution and self.download_mode == "D":
self.resolution = max_resolution
elif not self.resolution and self.download_mode == "SM":
self.resolution = self.data_collection_resolution
self.resolution = self.raw_data_collection_resolution

# resolution cant be higher than max resolution in D download mode
if self.download_mode == "D" and self.resolution < max_resolution:

self.resolution = max_resolution
if self.verbose:
print(
Expand All @@ -555,23 +554,11 @@ def set_correct_resolution(self) -> int:
"the highest resolution independently of the area."
)

# resolution can't be higher than raw data
if self.resolution < self.data_collection_resolution:

self.resolution = self.data_collection_resolution
if self.verbose:
print(
"The resolution prescribed is higher than "
"the raw data set resolution. Resolution is set "
"to raw resolution."
)

# dont use SM download mode if D can be used with full resolution
if (
max_resolution == self.data_collection_resolution
max_resolution == self.raw_data_collection_resolution
and self.download_mode == "SM"
):

self.download_mode = "D"
if self.verbose:
print(
Expand Down Expand Up @@ -602,8 +589,8 @@ def get_optimal_box_split(self) -> Tuple[int, int]:
trial_split_boxes = np.arange(2, 100)

# x and y pixel dimensions for each trial split box
boxes_pixels_x = (dx / trial_split_boxes) / self.data_collection_resolution
boxes_pixels_y = (dy / trial_split_boxes) / self.data_collection_resolution
boxes_pixels_x = (dx / trial_split_boxes) / self.resolution
boxes_pixels_y = (dy / trial_split_boxes) / self.resolution

# get minimum number of split boxes needed to stay below Sentinel Hub max dimensions
min_nb_boxes_x = int(trial_split_boxes[np.where(boxes_pixels_x <= 2500)[0][0]])
Expand Down Expand Up @@ -686,7 +673,6 @@ def get_evaluation_script(self, evaluation_script: Union[None, str]) -> str:

# if not set, set default evaluation script
if evaluation_script is None:

if self.satellite == "SENTINEL2":
self.get_evaluation_script_from_link(
"https://custom-scripts.sentinel-hub.com/custom-scripts/"
Expand All @@ -700,14 +686,12 @@ def get_evaluation_script(self, evaluation_script: Union[None, str]) -> str:

# if a URL, extract text
elif validators.url(evaluation_script):

self.evaluation_script = self.get_evaluation_script_from_link(
evaluation_script
)

# if a str, set attribute directly
elif isinstance(evaluation_script, str):

self.evaluation_script = evaluation_script

return self.evaluation_script
Expand Down Expand Up @@ -788,7 +772,6 @@ def sentinelhub_request(
)

if self.algorithm == "SICE":

self.response_files = [
"r_TOA_01",
"r_TOA_06",
Expand Down Expand Up @@ -880,7 +863,6 @@ def extract_sentinelhub_responses(self, folders: list) -> None:
"""

for folder in folders:

# open Tape ARchive file produced by Sentinel Hub API
tar = tarfile.open(f"{folder}/response.tar", "r:")

Expand Down Expand Up @@ -910,7 +892,6 @@ def rename_output_files(self) -> None:
self.output_filenames = []

for folder in folders:

# open request JSON file
with open(f"{folder}/request.json") as json_file:
request = json.load(json_file)
Expand All @@ -923,7 +904,6 @@ def rename_output_files(self) -> None:

# if SICE, store files in date subfolders if multiple outputs
if self.algorithm == "SICE":

# build folder name
date_folder = f"{self.store_folder}/{date}"

Expand All @@ -936,17 +916,14 @@ def rename_output_files(self) -> None:

# if D download mode, set file name using date and data collection
if self.download_mode == "D":

# build new file name
new_filename = (
f"{self.store_folder}/" + "{date}_{self.data_collection_str}.tif"
)

# If SICE, don't rename file but move to date folder
if self.algorithm == "SICE":

for f in date_files:

# include date in path
os.rename(
f, f"{self.store_folder}/{date}/{f.split(os.sep)[-1]}"
Expand All @@ -957,7 +934,6 @@ def rename_output_files(self) -> None:

# if SM download mode, set file name using date, data collection and box id
elif self.download_mode == "SM":

# recreate split box
split_box = shb.BBox(
list(request_tree.execute("$..bbox")),
Expand All @@ -977,9 +953,7 @@ def rename_output_files(self) -> None:

# if SICE, add split box id in all names and move to date folder
if self.algorithm == "SICE":

for f in date_files:

# extract absolute path
absolute_file_name = f.split(os.sep)[-1]
new_absolute_file_name = absolute_file_name.replace(
Expand All @@ -999,13 +973,11 @@ def rename_output_files(self) -> None:

# rename file using new file name
if os.path.exists(f"{folder}/response.tiff"):

os.rename(f"{folder}/response.tiff", new_filename)
self.output_filenames.append(new_filename)

# remove raw storage folders (and not date folders!)
for name in os.listdir(self.store_folder):

if (
os.path.isdir(os.path.join(self.store_folder, name))
) and "-" not in name:
Expand Down Expand Up @@ -1034,7 +1006,6 @@ def merge_rasters(self) -> None:
# merge rasters for each distinct date (work with distinct dates because
# of different trees depending on the algorithm used)
for date in distinct_dates:

# select files matching acquisition date only
date_output_files = [f for f in self.output_filenames if date in f]

Expand All @@ -1047,7 +1018,6 @@ def merge_rasters(self) -> None:
file_iterator = ["tif"]

for pattern in file_iterator:

date_response_files = [f for f in date_output_files if pattern in f]

# set file name of merged raster: replace split box id by mosaic
Expand Down
9 changes: 4 additions & 5 deletions tests/test_earthspy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@


class TestEarthspy:

# create local variables from environment secrets for convenience
SH_CLIENT_ID = os.environ["SH_CLIENT_ID"]
SH_CLIENT_SECRET = os.environ["SH_CLIENT_SECRET"]
Expand Down Expand Up @@ -127,13 +126,13 @@ def test_get_satellite_name(self) -> None:
assert isinstance(self.t1.satellite, str)
assert self.t1.satellite == "SENTINEL2"

def test_get_data_collection_resolution(self) -> None:
def test_get_raw_data_collection_resolution(self) -> None:
"""Test resolution selection"""

# check if data resolution was set correctly
assert self.t1.data_collection_resolution == 10
assert self.t2.data_collection_resolution == 10
assert self.t3.data_collection_resolution == 10
assert self.t1.raw_data_collection_resolution == 10
assert self.t2.raw_data_collection_resolution == 10
assert self.t3.raw_data_collection_resolution == 10

def test_set_number_of_cores(self) -> None:
"""Test selection of number of cores for multiprocessing"""
Expand Down

0 comments on commit 2f887ba

Please sign in to comment.