Merge pull request #24 from georchestra/backend_search_distant_gn

implement and test search function

backend/maelstro/common/models.py

@@ -0,0 +1,9 @@
+from typing import Any, Optional
+from pydantic import BaseModel, Field
+
+
+class SearchQuery(BaseModel):
+    query: Optional[dict[str, Any]] = {"query_string": {"query": "*"}}
+    source_: Optional[list[str]] = Field([], alias="_source")
+    from_: Optional[int] = Field(0, alias="from")
+    size: Optional[int] = 20

backend/maelstro/main.py

@@ -4,12 +4,21 @@
 
 from io import BytesIO
 from typing import Annotated, Any
-from fastapi import FastAPI, HTTPException, status, Request, Response, Header
+from fastapi import (
+    FastAPI,
+    HTTPException,
+    status,
+    Request,
+    Response,
+    Header,
+    Body,
+)
 from fastapi.responses import PlainTextResponse
 from geonetwork import GnApi
 from maelstro.config import ConfigError, app_config as config
 from maelstro.metadata import Meta
 from maelstro.core import CloneDataset
+from maelstro.common.models import SearchQuery
 
 
 app = FastAPI(root_path="/maelstro-backend")
@@ -101,6 +110,17 @@ def get_destinations() -> list[dict[str, str]]:
     return config.get_destinations()
 
 
+@app.post("/search/{src_name}")
+def post_search(
+    src_name: str, search_query: Annotated[SearchQuery, Body()]
+) -> dict[str, Any]:
+    src_info = config.get_access_info(
+        is_src=True, is_geonetwork=True, instance_id=src_name
+    )
+    gn = GnApi(src_info["url"], src_info["auth"])
+    return gn.search(search_query.model_dump(by_alias=True, exclude_unset=True))
+
+
 @app.get("/sources/{src_name}/data/{uuid}/layers")
 def get_layers(src_name: str, uuid: str) -> list[dict[str, str]]:
     try:

backend/pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "lxml (>=5.3.0,<6.0.0)",
     "types-pyyaml (>=6.0.12.20241230,<7.0.0.0)",
     "lxml-stubs (>=0.5.1,<0.6.0)",
-    "geonetwork @ git+https://github.com/camptocamp/python-geonetwork",
+    "geonetwork @ git+https://github.com/camptocamp/python-geonetwork@93aafcb9700914114d94dc111a3c5e4b7b58b606",
     "geoservercloud @ git+https://github.com/camptocamp/python-geoservercloud",
 ]
 

backend/tests/test_API.py

@@ -1,3 +1,4 @@
+import pytest
 from fastapi.testclient import TestClient
 
 from maelstro.main import app
@@ -10,3 +11,41 @@ def test_read_main():
     response = client.get("/")
     assert response.status_code == 200
     assert response.json() == {'Hello': 'World'}
+
+
+@pytest.mark.skip("Search on demo server currently down")
+def test_search():
+    response = client.post("/search/GeonetworkDemo", json={})
+    assert response.json()['hits']['total'] == {'value': 8316, 'relation': 'eq'}
+
+
+@pytest.mark.skip("Search on demo server currently down")
+def test_search2():
+    response = client.post("/search/GeonetworkDemo", json={"query": {"wildcard": {"resourceTitleObject.default": {"value": "plan_*"}}}, "size": 1})
+    assert len(response.json()['hits']['hits']) == 1
+    assert response.json()['hits']['total'] == 2
+
+
+def test_search3():
+    response = client.post("/search/GeonetworkRennes", json={"query": {"multi_match": {"fields": ["resourceTitleObject.*"], "query": "plan", "type": "bool_prefix"}}})
+    assert len(response.json()['hits']['hits']) == 10
+    assert response.json()['hits']['total']['value'] == 22
+
+
+def test_search4():
+    response = client.post("/search/GeonetworkRennes", json={
+        "query": {"query_string": {"fields": ["resourceTitleObject.*"], "query": "plan", "type": "bool_prefix"}},
+        "size": 15,
+    })
+    assert len(response.json()['hits']['hits']) == 15
+    assert response.json()['hits']['total']['value'] == 22
+
+
+def test_search5():
+    response = client.post("/search/GeonetworkRennes", json={
+        "source": [],
+        "from_": 0,
+        "size": 5
+    })
+    assert len(response.json()['hits']['hits']) == 5
+    assert response.json()['hits']['total']['value'] == 388

backend/tests/test_config.py

@@ -1,6 +1,7 @@
 import os
 import pytest
 from maelstro.config import Config, ConfigError
+from maelstro.common.types import Credentials
 
 
 os.environ["CONFIG_PATH"] = os.path.join(os.path.dirname(__file__), "test_config.yaml")
@@ -14,6 +15,10 @@ def test_init():
                 {
                     "name": "GeonetworkMaster",
                     "api_url": "https://demo.georchestra.org/geonetwork/srv/api",
+                },
+                {
+                    "name": "GeonetworkRennes",
+                    "api_url": "https://public.sig.rennesmetropole.fr/geonetwork/srv/api",
                 }
             ],
             "geoserver_instances": [
@@ -35,7 +40,9 @@ def test_init():
         "destinations": {
             "CompoLocale": {
                 "geonetwork": {
-                    "api_url": "http://geonetwork:8080/geonetwork/srv/api",
+                    "api_url": "https://georchestra-127-0-0-1.nip.io/geonetwork/srv/api",
+                    "login": "testadmin",
+                    "password": "testadmin",
                 },
                 "geoserver": {
                     "url": "https://georchestra-127-0-0-1.nip.io/geoserver"
@@ -72,7 +79,7 @@ def test_init():
 
 
 def test_subst_env():
-    os.environ["DEMO_LOGIN"] = "demo"
+    os.environ["DEMO_CRD"] = "demo"
     os.environ["LOCAL_LOGIN"] = "test"
     conf = Config("CONFIG_PATH")
     conf.config["sources"]["geonetwork_instances"][0]["login"] == "demo"
@@ -82,19 +89,19 @@ def test_subst_env():
 
 
 def test_get_info():
-    os.environ.pop("DEMO_LOGIN")
-    os.environ["DEMO_LOGIN"] = "demo"
+    os.environ.pop("DEMO_CRD")
+    os.environ["DEMO_CRD"] = "demo"
     conf = Config("CONFIG_PATH")
     assert conf.get_access_info(True, True, "GeonetworkMaster") == {
-        "auth": ("demo", "demo"),
+        "auth": Credentials("demo", "demo"),
         "url": "https://demo.georchestra.org/geonetwork/srv/api",
     }
     assert conf.get_access_info(True, False, "https://mastergs.rennesmetropole.fr/geoserver-geofence/") == {
-        "auth": ("toto6", "Str0ng_passW0rd"),
+        "auth": Credentials("toto6", "Str0ng_passW0rd"),
         "url": "https://mastergs.rennesmetropole.fr/geoserver-geofence/",
     }
     assert conf.get_access_info(False, True, "PlateformeProfessionnelle") == {
-        "auth": ("toto", "passW0rd"),
+        "auth": Credentials("toto", "passW0rd"),
         "url": "https://portail.sig.rennesmetropole.fr/geonetwork/srv/api",
     }
     assert conf.get_access_info(False, False, "CompoLocale") == {

backend/tests/test_config.yaml

@@ -5,8 +5,10 @@ sources:
     - name: "GeonetworkMaster"
       # api_url: https://mastergn.rennesmetropole.fr/geonetwork/srv/api
       api_url: https://demo.georchestra.org/geonetwork/srv/api
-      login_env_var: DEMO_LOGIN
-      password_env_var: DEMO_LOGIN
+      login_env_var: DEMO_CRD
+      password_env_var: DEMO_CRD
+    - name: "GeonetworkRennes"
+      api_url: https://public.sig.rennesmetropole.fr/geonetwork/srv/api
   geoserver_instances: 
     - url: "https://mastergs.rennesmetropole.fr/geoserver/"
       login: test
@@ -24,9 +26,9 @@ destinations:
   # nom de l'instance geOrchestra destination, telle qu'il apparaitra dans la UI
   "CompoLocale":
     geonetwork:
-      api_url: http://geonetwork:8080/geonetwork/srv/api
+      api_url: https://georchestra-127-0-0-1.nip.io/geonetwork/srv/api
       login_env_var: LOCAL_LOGIN
-      password_env_var: LOCAL_LOGIN
+      password_env_var: LOCAL_PASSWORD
     geoserver:
       url: https://georchestra-127-0-0-1.nip.io/geoserver
   "PlateformeProfessionnelle":

docker-compose.yml

@@ -33,7 +33,8 @@ services:
       - georchestra_datadir:/etc/georchestra
     environment:
       MAELSTRO_CONFIG: /app/tests/test_config.yaml
-      LOCAL_LOGIN: admin
+      LOCAL_LOGIN: testadmin
+      LOCAL_PASSWORD: testadmin
     healthcheck:
       test: "health_check"
       interval: 10s
@@ -49,5 +50,9 @@ services:
     build:
       context: ./backend
       target: check
+    environment:
+      MAELSTRO_CONFIG: /app/tests/test_config.yaml
+      LOCAL_LOGIN: testadmin
+      LOCAL_PASSWORD: testadmin
     volumes:
       - ./backend:/app