Merge branch 'develop'

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ fail_fast: false
 repos:
 
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: v3.4.0
+    rev: v4.3.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -17,13 +17,13 @@ repos:
     -   id: debug-statements
     -   id: check-merge-conflict
 
--   repo: 'https://gitlab.com/pycqa/flake8'
-    rev: 3.9.0
+-   repo: 'https://gitlab.com/PyCQA/flake8'
+    rev: 5.0.4
     hooks:
     -   id: flake8
 
 -   repo: 'https://github.com/ambv/black'
-    rev: 22.3.0
+    rev: 22.10.0
     hooks:
     -   id: black
         args: ['--safe']
@@ -34,6 +34,6 @@ repos:
     -   id: pydocstyle
 
 -   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0
+    rev: v5.10.1
     hooks:
     -   id: isort

CHANGES.rst

@@ -1,6 +1,14 @@
 Release history
 ---------------
 
+2.6.2 (2022-11-15)
+++++++++++++++++++
+
+* Added new methods to get assets filename from header (:pull:542)
+* All local files URI formats are now supported (:pull:545)
+* More tests (:pull:539)(:pull:549)
+* Various minor fixes and improvements (#535)(:pull:540)(:pull:541)(:pull:543)(:pull:544)
+
 2.6.1 (2022-10-19)
 ++++++++++++++++++
 

MANIFEST.in

@@ -1,5 +1,4 @@
 include README.rst LICENSE NOTICE requirements.txt
-include eodag/rest/description.md
 include eodag/rest/server.wsgi
 recursive-include docs *.rst
 graft eodag/resources

eodag/api/core.py

@@ -254,10 +254,6 @@ def build_index(self):
                     }
                 )
             ix_writer.commit()
-        else:
-            if self._product_types_index is None:
-                logger.debug("Opening product types index in %s", index_dir)
-                self._product_types_index = open_dir(index_dir)
 
     def set_preferred_provider(self, provider):
         """Set max priority for the given provider.
@@ -764,7 +760,7 @@ def guess_product_type(self, **kwargs):
 
         :param kwargs: A set of search parameters as keywords arguments
         :returns: The best match for the given parameters
-        :rtype: str
+        :rtype: list[str]
         :raises: :class:`~eodag.utils.exceptions.NoMatchingProductType`
         """
         supported_params = {
@@ -1708,7 +1704,7 @@ def download(
         If the metadata mapping for ``downloadLink`` is set to something that can be
         interpreted as a link on a
         local filesystem, the download is skipped (by now, only a link starting
-        with ``file://`` is supported). Therefore, any user that knows how to extract
+        with ``file:/`` is supported). Therefore, any user that knows how to extract
         product location from product metadata on a provider can override the
         ``downloadLink`` metadata mapping in the right way. For example, using the
         environment variable:
@@ -1742,7 +1738,7 @@ def download(
         :raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
         :raises: :class:`RuntimeError`
         """
-        if product.location.startswith("file://"):
+        if product.location.startswith("file:/"):
             logger.info("Local product detected. Download skipped")
             return uri_to_path(product.location)
         if product.downloader is None:

eodag/cli.py

@@ -674,8 +674,6 @@ def serve_rest(ctx, daemon, world, port, config, locs, debug):
     if config:
         os.environ["EODAG_CFG_FILE"] = config
 
-    if config:
-        os.environ["EODAG_CFG_FILE"] = config
     if locs:
         os.environ["EODAG_LOCS_CFG_FILE"] = locs
 
@@ -811,14 +809,6 @@ def deploy_wsgi_app(
         )
     )
     shutil.copy(os.path.join(webapp_src_path, "server.wsgi"), wsgi_path)
-    shutil.copy(
-        os.path.join(webapp_src_path, "description.md"),
-        os.path.join(webapp_dst_path, "description.md"),
-    )
-    shutil.copytree(
-        os.path.join(webapp_src_path, "templates"),
-        os.path.join(webapp_dst_path, "templates"),
-    )
 
     click.echo(
         "Overriding eodag HTTP server config with values: {}".format(server_config)

eodag/plugins/download/http.py

@@ -21,6 +21,7 @@
 import shutil
 import zipfile
 from cgi import parse_header
+from urllib.parse import urlparse
 
 import requests
 from requests import HTTPError, RequestException
@@ -305,7 +306,9 @@ def _download_assets(
                 asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
             ).headers.get("content-disposition", None)
             if asset_content_disposition:
-                asset_filename = parse_header(asset_content_disposition)[-1]["filename"]
+                asset_filename = parse_header(asset_content_disposition)[-1].get(
+                    "filename", None
+                )
             else:
                 asset_filename = None
 
@@ -342,15 +345,26 @@ def _download_assets(
                         logger.warning("Skipping %s" % asset_url)
                     error_messages.add(str(e))
                 else:
-                    if asset_filename is not None:
-                        asset_rel_path = asset_filename
-                    else:
-                        asset_rel_path = (
-                            asset_url.replace(product.location, "")
-                            .replace("https://", "")
-                            .replace("http://", "")
+                    asset_rel_path = urlparse(asset_url).path.strip("/")
+                    asset_rel_dir = os.path.dirname(asset_rel_path)
+
+                    if asset_filename is None:
+                        # try getting content-disposition header from GET if not in HEAD result
+                        asset_content_disposition = stream.headers.get(
+                            "content-disposition", None
                         )
-                    asset_abs_path = os.path.join(fs_dir_path, asset_rel_path)
+                        if asset_content_disposition:
+                            asset_filename = parse_header(asset_content_disposition)[
+                                -1
+                            ].get("filename", None)
+
+                    if asset_filename is None:
+                        # default filename extracted from path
+                        asset_filename = os.path.basename(asset_rel_path)
+
+                    asset_abs_path = os.path.join(
+                        fs_dir_path, asset_rel_dir, asset_filename
+                    )
                     asset_abs_path_dir = os.path.dirname(asset_abs_path)
                     if not os.path.isdir(asset_abs_path_dir):
                         os.makedirs(asset_abs_path_dir)

eodag/rest/templates/README

@@ -0,0 +1 @@
+Empty templates directory kept for compatibility reasons

pyproject.toml

@@ -3,4 +3,4 @@ requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
-fallback_version = "2.6.2.dev0"
+fallback_version = "2.6.3.dev0"

setup.cfg

@@ -59,7 +59,21 @@ install_requires =
     cdsapi
 
 [options.extras_require]
-dev = pytest; pytest-cov; pytest-html; tox; faker; moto; twine; wheel; flake8; pre-commit; responses
+dev =
+    pytest
+    pytest-cov
+    # pytest-html max version set and py requirement added
+    # until https://github.com/pytest-dev/pytest-html/issues/559 is fixed
+    py >= 1.8.2
+    pytest-html < 3.2.0
+    tox
+    faker
+    moto
+    twine
+    wheel
+    flake8
+    pre-commit
+    responses
 notebook = tqdm[notebook]
 tutorials =
     eodag-cube >= 0.2.0