Merge branch 'main' into rs/fix-fetch-unlabeled-detections

.github/labeler.yml

@@ -72,6 +72,12 @@
     - src/app/api/*/endpoints/organizations.py
     - src/app/crud/organizations.py
 
+'endpoint: webhooks':
+- changed-files:
+  - any-glob-to-any-file:
+    - src/app/api/*/endpoints/webhooks.py
+    - src/app/crud/webhooks.py
+
 
 'topic: build':
 - changed-files:

.github/workflows/push.yml

@@ -54,7 +54,6 @@ jobs:
           key: ${{ secrets.SSH_DEPLOY_DEV }}
           script: |
             # Ensure we have max disk space
-            docker rm -fv $(docker ps -aq)
             docker rmi -f $(docker images -f "dangling=true" -q)
             docker volume rm -f $(docker volume ls -f "dangling=true" -q)
             # Update the service
@@ -63,7 +62,6 @@ jobs:
             # Check update
             docker inspect -f '{{ .Created }}' $(docker compose images -q backend)
             # Clean up
-            docker rm -fv $(docker ps -aq)
             docker rmi -f $(docker images -f "dangling=true" -q)
             docker volume rm -f $(docker volume ls -f "dangling=true" -q)
       - name: Ping server

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
       - id: debug-statements
         language_version: python3
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.6.2'
+    rev: 'v0.6.6'
     hooks:
       - id: ruff
         args:

CONTRIBUTING.md

@@ -13,12 +13,17 @@ The back-end core feature is to interact with the metadata tables. For the servi
 
 - Users: stores the hashed credentials and access level for users.
 - Cameras: stores the camera metadata.
+- Organizations: scope the access to the API.
 
 #### Core worklow tables
 
 - Detection: association of a picture and a camera.
 
-![UML diagram](https://github.com/user-attachments/assets/04016451-6a67-4496-adf7-ad790722d74b)
+#### Client-related tables
+
+- Webhook: stores the webhook URLs.
+
+_The UML is versioned at [`scripts/dbdiagram.txt`](https://github.com/pyronear/pyro-api/blob/main/scripts/dbdiagram.txt) and the UML diagram is available on [DBDiagram](https://dbdiagram.io/d/Pyronear-UML-665a15d0b65d933879357b58)._
 
 ### What is the full detection workflow through the API
 

client/pyroclient/client.py

@@ -23,6 +23,7 @@
     # CAMERAS
     #################
     "cameras-heartbeat": "/cameras/heartbeat",
+    "cameras-image": "/cameras/image",
     "cameras-fetch": "/cameras",
     #################
     # DETECTIONS
@@ -103,6 +104,22 @@ def headers(self) -> Dict[str, str]:
         return {"Authorization": f"Bearer {self.token}"}
 
     # CAMERAS
+    def fetch_cameras(self) -> Response:
+        """List the cameras accessible to the authenticated user
+
+        >>> from pyroclient import client
+        >>> api_client = Client("MY_USER_TOKEN")
+        >>> response = api_client.fetch_cameras()
+
+        Returns:
+            HTTP response
+        """
+        return requests.get(
+            self.routes["cameras-fetch"],
+            headers=self.headers,
+            timeout=self.timeout,
+        )
+
     def heartbeat(self) -> Response:
         """Update the last ping of the camera
 
@@ -115,6 +132,24 @@ def heartbeat(self) -> Response:
         """
         return requests.patch(self.routes["cameras-heartbeat"], headers=self.headers, timeout=self.timeout)
 
+    def update_last_image(self, media: bytes) -> Response:
+        """Update the last image of the camera
+
+        >>> from pyroclient import Client
+        >>> api_client = Client("MY_CAM_TOKEN")
+        >>> with open("path/to/my/file.ext", "rb") as f: data = f.read()
+        >>> response = api_client.update_last_image(data)
+
+        Returns:
+            HTTP response containing the update device info
+        """
+        return requests.patch(
+            self.routes["cameras-image"],
+            headers=self.headers,
+            files={"file": ("logo.png", media, "image/png")},
+            timeout=self.timeout,
+        )
+
     # DETECTIONS
     def create_detection(
         self,
@@ -171,22 +206,6 @@ def label_detection(self, detection_id: int, is_wildfire: bool) -> Response:
             timeout=self.timeout,
         )
 
-    def fetch_cameras(self) -> Response:
-        """List the cameras accessible to the authenticated user
-
-        >>> from pyroclient import client
-        >>> api_client = Client("MY_USER_TOKEN")
-        >>> response = api_client.fetch_cameras()
-
-        Returns:
-            HTTP response
-        """
-        return requests.get(
-            self.routes["cameras-fetch"],
-            headers=self.headers,
-            timeout=self.timeout,
-        )
-
     def get_detection_url(self, detection_id: int) -> Response:
         """Retrieve the URL of the media linked to a detection
 

client/tests/test_client.py

@@ -26,7 +26,13 @@ def test_client_constructor(token, host, timeout, expected_error):
 @pytest.fixture(scope="session")
 def test_cam_workflow(cam_token, mock_img):
     cam_client = Client(cam_token, "http://localhost:5050", timeout=10)
-    assert cam_client.heartbeat().status_code == 200
+    response = cam_client.heartbeat()
+    assert response.status_code == 200
+    # Check that last_image gets changed
+    assert response.json()["last_image"] is None
+    response = cam_client.update_last_image(mock_img)
+    assert response.status_code == 200, response.__dict__
+    assert isinstance(response.json()["last_image"], str)
     # Check that adding bboxes works
     with pytest.raises(ValueError, match="bboxes must be a non-empty list of tuples"):
         cam_client.create_detection(mock_img, 123.2, None)

pyproject.toml

@@ -32,7 +32,7 @@ boto3 = "^1.26.0"
 optional = true
 
 [tool.poetry.group.quality.dependencies]
-ruff = "==0.6.2"
+ruff = "==0.6.6"
 mypy = "==1.10.0"
 types-requests = ">=2.0.0"
 types-python-dateutil = "^2.8.0"
@@ -158,5 +158,7 @@ module = [
     "botocore.*",
     "databases",
     "posthog",
+    "prometheus_fastapi_instrumentator",
+    "pydantic_settings",
 ]
 ignore_missing_imports = true

scripts/dbdiagram.txt

@@ -8,8 +8,8 @@ Table "User" as U {
   "id" int [not null]
   "organization_id" int [ref: > O.id, not null]
   "role" userrole [not null]
-  "login" str [not null]
-  "hashed_password" str [not null]
+  "login" text [not null]
+  "hashed_password" text [not null]
   "created_at" timestamp [not null]
   Indexes {
     (id, login) [pk]
@@ -19,14 +19,15 @@ Table "User" as U {
 Table "Camera" as C {
   "id" int [not null]
   "organization_id" int [ref: > O.id, not null]
-  "name" str [not null]
+  "name" text [not null]
   "angle_of_view" float [not null]
   "elevation" float [not null]
   "lat" float [not null]
   "lon" float [not null]
   "is_trustable" bool [not null]
   "created_at" timestamp [not null]
   "last_active_at" timestamp
+  "last_image" text
   Indexes {
     (id) [pk]
   }
@@ -36,8 +37,8 @@ Table "Detection" as D {
   "id" int [not null]
   "camera_id" int [ref: > C.id, not null]
   "azimuth" float [not null]
-  "bucket_key" str [not null]
-  "bboxes" str [not null]
+  "bucket_key" text [not null]
+  "bboxes" text [not null]
   "is_wildfire" bool
   "created_at" timestamp [not null]
   "updated_at" timestamp [not null]
@@ -48,7 +49,16 @@ Table "Detection" as D {
 
 Table "Organization" as O {
   "id" int [not null]
-  "name" str [not null]
+  "name" text [not null]
+  Indexes {
+    (id) [pk]
+  }
+}
+
+
+Table "Webhook" as W {
+  "id" int [not null]
+  "url" text [not null]
   Indexes {
     (id) [pk]
   }

src/app/api/api_v1/endpoints/cameras.py

@@ -6,15 +6,16 @@
 from datetime import datetime
 from typing import List, cast
 
-from fastapi import APIRouter, Depends, HTTPException, Path, Security, status
+from fastapi import APIRouter, Depends, File, HTTPException, Path, Security, UploadFile, status
 
 from app.api.dependencies import get_camera_crud, get_jwt
 from app.core.config import settings
 from app.core.security import create_access_token
 from app.crud import CameraCRUD
 from app.models import Camera, Role, UserRole
-from app.schemas.cameras import CameraCreate, LastActive
+from app.schemas.cameras import CameraCreate, LastActive, LastImage
 from app.schemas.login import Token, TokenPayload
+from app.services.storage import s3_service, upload_file
 from app.services.telemetry import telemetry_client
 
 router = APIRouter()
@@ -62,10 +63,26 @@ async def heartbeat(
     cameras: CameraCRUD = Depends(get_camera_crud),
     token_payload: TokenPayload = Security(get_jwt, scopes=[Role.CAMERA]),
 ) -> Camera:
-    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-heartbeat", properties={"camera_id": camera_id})
+    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-heartbeat")
     return await cameras.update(token_payload.sub, LastActive(last_active_at=datetime.utcnow()))
 
 
+@router.patch("/image", status_code=status.HTTP_200_OK, summary="Update last image of a camera")
+async def update_image(
+    file: UploadFile = File(..., alias="file"),
+    cameras: CameraCRUD = Depends(get_camera_crud),
+    token_payload: TokenPayload = Security(get_jwt, scopes=[Role.CAMERA]),
+) -> Camera:
+    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-image")
+    bucket_key = await upload_file(file, token_payload.organization_id, token_payload.sub)
+    # If the upload succeeds, delete the previous image
+    cam = cast(Camera, await cameras.get(token_payload.sub, strict=True))
+    if isinstance(cam.last_image, str):
+        s3_service.get_bucket(s3_service.resolve_bucket_name(token_payload.organization_id)).delete_file(cam.last_image)
+    # Update the DB entry
+    return await cameras.update(token_payload.sub, LastImage(last_image=bucket_key, last_active_at=datetime.utcnow()))
+
+
 @router.post("/{camera_id}/token", status_code=status.HTTP_200_OK, summary="Request an access token for the camera")
 async def create_camera_token(
     camera_id: int = Path(..., gt=0),