Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Api inherit Search and Download #1051

Merged
merged 2 commits into from
Mar 12, 2024
+75 −233
Merged

Api inherit Search and Download #1051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db70002
refactor: Api inherits from Search and Download
alambare Mar 5, 2024
c39bbc5
refactor: use if/else instead of try/except in get plugins
alambare Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 5 additions & 171 deletions eodag/plugins/apis/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,39 +17,16 @@
# limitations under the License.
from __future__ import annotations

import logging
from typing import TYPE_CHECKING
from eodag.plugins.download.base import Download
from eodag.plugins.search.base import Search

from pydantic.fields import Field, FieldInfo

from eodag.plugins.base import PluginTopic
from eodag.utils import (
DEFAULT_DOWNLOAD_TIMEOUT,
DEFAULT_DOWNLOAD_WAIT,
DEFAULT_ITEMS_PER_PAGE,
DEFAULT_PAGE,
Annotated,
deepcopy,
)

if TYPE_CHECKING:
from typing import Any, Dict, List, Optional, Tuple, Union

from requests.auth import AuthBase

from eodag.api.product import EOProduct
from eodag.api.search_result import SearchResult
from eodag.types.download_args import DownloadConf
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack

logger = logging.getLogger("eodag.apis.base")


class Api(PluginTopic):
class Api(Search, Download):
"""Plugins API Base plugin

An Api plugin has three download methods that it must implement:
An Api plugin inherit the methods from Search and Download plugins.

There are three methods that it must implement:
- ``query``: search for products
- ``download``: download a single :class:`~eodag.api.product._product.EOProduct`
- ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult`
Expand All @@ -76,146 +53,3 @@ class Api(PluginTopic):
product's file/directory. If the *record* file only is found, it must be deleted
(it certainly indicates that the download didn't complete)
"""

auth: Union[AuthBase, Dict[str, str]]
next_page_url: Optional[str]
next_page_query_obj: Optional[Dict[str, Any]]
_request: Any # needed by deprecated load_stac_items

def clear(self) -> None:
"""Method used to clear a search context between two searches."""
pass

def query(
self,
product_type: Optional[str] = None,
items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
page: int = DEFAULT_PAGE,
count: bool = True,
**kwargs: Any,
) -> Tuple[List[EOProduct], Optional[int]]:
"""Implementation of how the products must be searched goes here.

This method must return a tuple with (1) a list of EOProduct instances (see eodag.api.product module)
which will be processed by a Download plugin (2) and the total number of products matching
the search criteria. If ``count`` is False, the second element returned must be ``None``.
"""
raise NotImplementedError("A Api plugin must implement a method named query")

def discover_product_types(self) -> Optional[Dict[str, Any]]:
"""Fetch product types list from provider using `discover_product_types` conf"""
return None

def discover_queryables(
self, **kwargs: Any
) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
"""Fetch queryables list from provider using `discover_queryables` conf

:param kwargs: additional filters for queryables (`productType` and other search
arguments)
:type kwargs: Any
:returns: fetched queryable parameters dict
:rtype: Optional[Dict[str, Annotated[Any, FieldInfo]]]
"""
return None

def get_defaults_as_queryables(
self, product_type: str
) -> Dict[str, Annotated[Any, FieldInfo]]:
"""
Return given product type defaut settings as queryables

:param product_type: given product type
:type product_type: str
:returns: queryable parameters dict
:rtype: Dict[str, Annotated[Any, FieldInfo]]
"""
defaults = deepcopy(self.config.products.get(product_type, {}))
defaults.pop("metadata_mapping", None)

queryables = {}
for parameter, value in defaults.items():
queryables[parameter] = Annotated[type(value), Field(default=value)]
return queryables

def download(
self,
product: EOProduct,
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
progress_callback: Optional[ProgressCallback] = None,
wait: int = DEFAULT_DOWNLOAD_WAIT,
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
**kwargs: Unpack[DownloadConf],
) -> Optional[str]:
"""
Base download method. Not available, it must be defined for each plugin.

:param product: The EO product to download
:type product: :class:`~eodag.api.product._product.EOProduct`
:param auth: (optional) authenticated object
:type auth: Union[AuthBase, Dict[str, str]]
:param progress_callback: (optional) A progress callback
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
:param wait: (optional) If download fails, wait time in minutes between two download tries
:type wait: int
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
to download
:type timeout: int
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
and `dl_url_params` (dict) can be provided as additional kwargs
and will override any other values defined in a configuration
file or with environment variables.
:type kwargs: Union[str, bool, dict]
:returns: The absolute path to the downloaded product in the local filesystem
(e.g. '/tmp/product.zip' on Linux or
'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
:rtype: str
"""
raise NotImplementedError(
"An Api plugin must implement a method named download"
)

def download_all(
self,
products: SearchResult,
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
downloaded_callback: Optional[DownloadedCallback] = None,
progress_callback: Optional[ProgressCallback] = None,
wait: int = DEFAULT_DOWNLOAD_WAIT,
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
**kwargs: Unpack[DownloadConf],
) -> List[str]:
"""
Base download_all method.

:param products: Products to download
:type products: :class:`~eodag.api.search_result.SearchResult`
:param auth: (optional) authenticated object
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
:param downloaded_callback: (optional) A method or a callable object which takes
as parameter the ``product``. You can use the base class
:class:`~eodag.api.product.DownloadedCallback` and override
its ``__call__`` method. Will be called each time a product
finishes downloading
:type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
or None
:param progress_callback: (optional) A progress callback
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
:param wait: (optional) If download fails, wait time in minutes between two download tries
:type wait: int
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
to download
:type timeout: int
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
and `dl_url_params` (dict) can be provided as additional kwargs
and will override any other values defined in a configuration
file or with environment variables.
:type kwargs: Union[str, bool, dict]
:returns: List of absolute paths to the downloaded products in the local
filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
:rtype: list
"""
raise NotImplementedError(
"A Api plugin must implement a method named download_all"
)
8 changes: 6 additions & 2 deletions eodag/plugins/apis/cds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
logger = logging.getLogger("eodag.apis.cds")


class CdsApi(HTTPDownload, Api, BuildPostSearchResult):
class CdsApi(Api, HTTPDownload, BuildPostSearchResult):
"""A plugin that enables to build download-request and download data on CDS API.

Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
Expand Down Expand Up @@ -323,7 +323,7 @@ def authenticate(self) -> Dict[str, str]:

return auth_dict

def _prepare_download_link(self, product):
def _prepare_download_link(self, product: EOProduct) -> None:
"""Update product download link with http url obtained from cds api"""
# get download request dict from product.location/downloadLink url query string
# separate url & parameters
Expand Down Expand Up @@ -539,3 +539,7 @@ def discover_queryables(

python_queryables = create_model("m", **field_definitions).model_fields
return dict(default_queryables, **model_fields_to_annotated(python_queryables))

def clear(self) -> None:
"""Clear search context"""
pass
7 changes: 5 additions & 2 deletions eodag/plugins/apis/ecmwf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from ecmwfapi.api import APIException, Connection, get_apikey_values

from eodag.plugins.apis.base import Api
from eodag.plugins.download.base import Download
from eodag.plugins.search.base import Search
from eodag.plugins.search.build_search_result import BuildPostSearchResult
from eodag.rest.stac import DEFAULT_MISSION_START_DATE
Expand Down Expand Up @@ -60,7 +59,7 @@
ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}


class EcmwfApi(Download, Api, BuildPostSearchResult):
class EcmwfApi(Api, BuildPostSearchResult):
"""A plugin that enables to build download-request and download data on ECMWF MARS.

Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
Expand Down Expand Up @@ -271,3 +270,7 @@ def download_all(
timeout=timeout,
**kwargs,
)

def clear(self) -> None:
"""Clear search context"""
pass
3 changes: 1 addition & 2 deletions eodag/plugins/apis/usgs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
properties_from_json,
)
from eodag.plugins.apis.base import Api
from eodag.plugins.download.base import Download
from eodag.utils import (
DEFAULT_DOWNLOAD_TIMEOUT,
DEFAULT_DOWNLOAD_WAIT,
Expand Down Expand Up @@ -66,7 +65,7 @@
logger = logging.getLogger("eodag.apis.usgs")


class UsgsApi(Download, Api):
class UsgsApi(Api):
"""A plugin that enables to query and download data on the USGS catalogues"""

def __init__(self, provider: str, config: PluginConfig) -> None:
Expand Down
Loading
Loading