-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement Retries and Timeout for File Download (#275)
Previously, files were retrieved with `subprocess` + `wget`. Now, it uses `requests` with a fixed timeout and retries, plus exponential backoff. --------- Co-authored-by: wipacdevbot <[email protected]>
- Loading branch information
1 parent
0199888
commit c3b0755
Showing
5 changed files
with
109 additions
and
71 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,74 @@ | ||
| """Tests for file-staging logic.""" | ||
|
|
||
| import logging | ||
| import subprocess | ||
| from typing import Dict | ||
|
|
||
|
|
||
| from skymap_scanner.utils.data_handling import DataStager | ||
| from skymap_scanner import config as cfg | ||
| from skymap_scanner.utils.data_handling import DataStager | ||
|
|
||
| logging.basicConfig(level=logging.DEBUG) | ||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| # Build list of all local files. | ||
| local_file_list = [] | ||
| for path in cfg.LOCAL_DATA_SOURCES: | ||
| subpath = path / cfg.LOCAL_SPLINE_SUBDIR | ||
| directory_content = subpath.glob("*") | ||
| for path in directory_content: | ||
| if path.is_file(): # skip directories | ||
| local_file_list.append(path.name) # store filename without path | ||
| def test_file_staging() -> None: | ||
|
|
||
| # Declare at least one filename only expected to be available remotely. | ||
| remote_file_list = ["README"] | ||
| # Build list of all local files. | ||
| local_file_list = [] | ||
| for path in cfg.LOCAL_DATA_SOURCES: | ||
| subpath = path / cfg.LOCAL_SPLINE_SUBDIR | ||
| directory_content = subpath.glob("*") | ||
| for path in directory_content: | ||
| if path.is_file(): # skip directories | ||
| local_file_list.append(path.name) # store filename without path | ||
|
|
||
| # Declare at least one filename that does not exist. | ||
| invalid_file_list = ["NONEXISTENT_FILE"] | ||
| # Declare at least one filename only expected to be available remotely. | ||
| remote_file_list = ["README"] | ||
|
|
||
| datastager = DataStager( | ||
| local_paths=cfg.LOCAL_DATA_SOURCES, | ||
| local_subdir=cfg.LOCAL_SPLINE_SUBDIR, | ||
| remote_path=f"{cfg.REMOTE_DATA_SOURCE}/{cfg.REMOTE_SPLINE_SUBDIR}", | ||
| ) | ||
| # Declare at least one filename that does not exist. | ||
| invalid_file_list = ["NONEXISTENT_FILE"] | ||
|
|
||
| for file_list in [local_file_list, remote_file_list, invalid_file_list]: | ||
| try: | ||
| datastager = DataStager( | ||
| local_paths=cfg.LOCAL_DATA_SOURCES, | ||
| local_subdir=cfg.LOCAL_SPLINE_SUBDIR, | ||
| remote_path=f"{cfg.REMOTE_DATA_SOURCE}/{cfg.REMOTE_SPLINE_SUBDIR}", | ||
| ) | ||
|
|
||
| # test stage_files() | ||
| # -> OK | ||
| for file_list in [local_file_list, remote_file_list]: | ||
| datastager.stage_files(file_list) | ||
| except subprocess.CalledProcessError: | ||
| logger.debug(f"Staging failed as expected for invalid file.") | ||
| # -> ERROR | ||
| try: | ||
| datastager.stage_files(invalid_file_list) | ||
| except Exception as e: | ||
| assert isinstance(e, RuntimeError) | ||
| assert str(e) == ( | ||
| f"Download failed after {cfg.REMOTE_DATA_DOWNLOAD_RETRIES} retries: " | ||
| f"404 Client Error: Not Found for url: {datastager.remote_path}/{invalid_file_list[0]}" | ||
| ) | ||
|
|
||
| # ensure that filepaths can be retrieved for all local files | ||
| local_filepaths: Dict[str, str] = dict() | ||
| for filename in local_file_list: | ||
| logger.debug(f"Testing local file: {filename}.") | ||
| local_filepaths[filename] = datastager.get_local_filepath(filename) | ||
| assert local_filepaths[filename] == datastager.get_filepath(filename) | ||
| logger.debug(f"File available at {local_filepaths[filename]}.") | ||
|
|
||
| # ensure that filepaths can be retrieved for all local files | ||
| local_filepaths: Dict[str, str] = dict() | ||
| for filename in local_file_list: | ||
| logger.debug(f"Testing local file: {filename}.") | ||
| local_filepaths[filename] = datastager.get_local_filepath(filename) | ||
| assert local_filepaths[filename] == datastager.get_filepath(filename) | ||
| logger.debug(f"File available at {local_filepaths[filename]}.") | ||
| for filename in remote_file_list: | ||
| logger.debug(f"Testing staging of remote file: {filename}") | ||
| filepath: str = datastager.get_filepath(filename) | ||
| logger.debug(f"File available at {filepath}.") | ||
|
|
||
| for filename in remote_file_list: | ||
| logger.debug(f"Testing staging of remote file: {filename}") | ||
| filepath: str = datastager.get_filepath(filename) | ||
| logger.debug(f"File available at {filepath}.") | ||
| for filename in invalid_file_list: | ||
| logger.debug(f"Testing staging of remote file: {filename}") | ||
| try: | ||
| filepath = datastager.get_filepath(filename) | ||
| except FileNotFoundError: | ||
| logger.debug(f"File not available as expected.") | ||
| else: | ||
| assert 0 # we shouldn't get here! | ||
|
|
||
|
|
||
| for filename in invalid_file_list: | ||
| logger.debug(f"Testing staging of remote file: {filename}") | ||
| try: | ||
| filepath = datastager.get_filepath(filename) | ||
| except FileNotFoundError: | ||
| logger.debug(f"File not available as expected.") | ||
| if __name__ == "__main__": | ||
| test_file_staging() |