feat: adapt namespace endpoints

components/renku_data_services/base_api/pagination.py

@@ -3,10 +3,12 @@
 from collections.abc import Callable, Coroutine, Sequence
 from functools import wraps
 from math import ceil
-from typing import Any, Concatenate, NamedTuple, ParamSpec, cast
+from typing import Any, Concatenate, NamedTuple, ParamSpec, TypeVar, cast
 
 from sanic import Request, json
 from sanic.response import JSONResponse
+from sqlalchemy import Select
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from renku_data_services import errors
 
@@ -94,3 +96,36 @@ async def decorated_function(request: Request, *args: _P.args, **kwargs: _P.kwar
         return json(items, headers=pagination.as_header())
 
     return decorated_function
+
+
+_T = TypeVar("_T")
+
+
+async def paginate_queries(
+    req: PaginationRequest, session: AsyncSession, stmts: list[tuple[Select[tuple[_T]], int]]
+) -> list[_T]:
+    """Paginate several different queries as if they were part of a single table."""
+    # NOTE: We ignore the possibility that a count for a statement is not accurate. I.e. the count
+    # says that the statement should return 10 items but the statement truly returns 8 or vice-versa.
+    # To fully account for edge cases of inaccuracry in the expected number of results
+    # we would have to run every query passed in - even though the offset is so high that we would only need
+    # to run 1 or 2 queries out of a large list.
+    output: list[_T] = []
+    max_offset = 0
+    stmt_offset = 0
+    offset_discount = 0
+    for stmt, stmt_cnt in stmts:
+        max_offset += stmt_cnt
+        if req.offset >= max_offset:
+            offset_discount += stmt_cnt
+            continue
+        stmt_offset = req.offset - offset_discount if req.offset > 0 else 0
+        res_scalar = await session.scalars(stmt.offset(stmt_offset).limit(req.per_page))
+        res = res_scalar.all()
+        num_required = req.per_page - len(output)
+        if num_required >= len(res):
+            output.extend(res)
+        else:
+            output.extend(res[:num_required])
+            return output
+    return output

components/renku_data_services/data_connectors/blueprints.py

@@ -346,7 +346,7 @@ def _dump_data_connector(data_connector: models.DataConnector, validator: RClone
         return dict(
             id=str(data_connector.id),
             name=data_connector.name,
-            namespace=data_connector.namespace.slug,
+            namespace="/".join(data_connector.namespace.path),
             slug=data_connector.slug,
             storage=storage,
             # secrets=,

components/renku_data_services/namespace/api.spec.yaml

@@ -12,13 +12,9 @@ paths:
     get:
       summary: Get all groups
       parameters:
-        - in: query
-          description: query parameters
-          name: params
-          style: form
-          explode: true
-          schema:
-            $ref: "#/components/schemas/GroupsGetQuery"
+        - $ref: "#/components/parameters/PaginationRequestPage"
+        - $ref: "#/components/parameters/PaginationRequestPerPage"
+        - $ref: "#/components/parameters/OnlyDirectMember"
       responses:
         "200":
           description: List of groups
@@ -260,13 +256,10 @@ paths:
     get:
       summary: Get all namespaces
       parameters:
-        - in: query
-          description: query parameters
-          name: params
-          style: form
-          explode: true
-          schema:
-            $ref: "#/components/schemas/NamespaceGetQuery"
+        - $ref: "#/components/parameters/PaginationRequestPage"
+        - $ref: "#/components/parameters/PaginationRequestPerPage"
+        - $ref: "#/components/parameters/MinimumRole"
+        - $ref: "#/components/parameters/NamespaceKind"
       responses:
         "200":
           description: List of namespaces
@@ -411,6 +404,13 @@ components:
       # - cannot contain uppercase characters
       pattern: '^(?!.*\.git$|.*\.atom$|.*[\-._][\-._].*)[a-z0-9][a-z0-9\-_.]*$'
       example: "a-slug-example"
+    SlugPath:
+      description: A list of slugs that make up the path to a resource
+      example: ["group1", "project2"]
+      type: array
+      minItems: 1
+      items:
+        $ref: "#/components/schemas/Slug"
     CreationDate:
       description: The date and time the resource was created (in UTC and ISO-8601 format)
       type: string
@@ -500,6 +500,7 @@ components:
       enum:
         - group
         - user
+        - project
     NamespaceResponseList:
       description: A list of Renku namespaces
       type: array
@@ -521,17 +522,21 @@ components:
           $ref: "#/components/schemas/KeycloakId"
         namespace_kind:
           $ref: "#/components/schemas/NamespaceKind"
+        path:
+          $ref: "#/components/schemas/SlugPath"
       required:
         - "id"
         - "namespace_kind"
         - "slug"
+        - "path"
       example:
         id: "01AN4Z79ZS5XN0F25N3DB94T4R"
         name: "R-Project Group"
         slug: "r-project"
         created_by: "owner-keycloak-id"
         creation_date: "2024-03-04T13:04:45Z"
         namespace_kind: "group"
+        path: ["r-project"]
     UserId:
       type: string
       description: Keycloak user ID
@@ -543,23 +548,15 @@ components:
       example: John
       minLength: 1
       maxLength: 256
-    NamespaceGetQuery:
-      description: Query params for namespace get request
-      allOf:
-      - $ref: "#/components/schemas/PaginationRequest"
-      - properties:
-          minimum_role:
-            description: A minimum role to filter results by.
-            $ref: "#/components/schemas/GroupRole"
-    GroupsGetQuery:
-      description: Query params for namespace get request
-      allOf:
-      - $ref: "#/components/schemas/PaginationRequest"
-      - properties:
-          direct_member:
-            description: A flag to filter groups where the user is a direct member.
-            type: boolean
-            default: false
+    NamespaceGetQueryKind:
+      type: array
+      description: Which namespace kinds to include in the response
+      items:
+        $ref: "#/components/schemas/NamespaceKind"
+      default:
+        - user
+        - group
+      minItems: 1
     GroupPermissions:
       description: The set of permissions on a group
       type: object
@@ -573,21 +570,17 @@ components:
         change_membership:
           description: The user can manage group members
           type: boolean
-    PaginationRequest:
-      type: object
-      additionalProperties: false
-      properties:
-        page:
-          description: Result's page number starting from 1
-          type: integer
-          minimum: 1
-          default: 1
-        per_page:
-          description: The number of results per page
-          type: integer
-          minimum: 1
-          maximum: 100
-          default: 20
+    PaginationRequestPage:
+      description: Result's page number starting from 1
+      type: integer
+      minimum: 1
+      default: 1
+    PaginationRequestPerPage:
+      description: The number of results per page
+      type: integer
+      minimum: 1
+      maximum: 100
+      default: 20
     ErrorResponse:
       type: object
       properties:
@@ -610,6 +603,48 @@ components:
             - "message"
       required:
         - "error"
+  parameters:
+    PaginationRequestPage:
+      in: query
+      description: the current page in paginated response
+      name: page
+      style: form
+      explode: true
+      schema:
+        $ref: "#/components/schemas/PaginationRequestPage"
+    PaginationRequestPerPage:
+      in: query
+      description: the number of results per page in a paginated response
+      name: per_page
+      style: form
+      explode: true
+      schema:
+        $ref: "#/components/schemas/PaginationRequestPerPage"
+    MinimumRole:
+      in: query
+      description: The minimum role the user should have in the resources returned
+      name: minimum_role
+      style: form
+      explode: true
+      schema:
+        $ref: "#/components/schemas/GroupRole"
+    NamespaceKind:
+      in: query
+      description: environment kinds query parameter
+      name: kinds
+      style: form
+      explode: true
+      schema:
+        $ref: "#/components/schemas/NamespaceGetQueryKind"
+    OnlyDirectMember:
+      in: query
+      description: A flag to filter for where the user is a direct member.
+      name: direct_member
+      style: form
+      explode: true
+      schema:
+        type: boolean
+        default: false
   responses:
     Error:
       description: The schema for all 4xx and 5xx responses

components/renku_data_services/namespace/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-10-22T07:46:53+00:00
+#   timestamp: 2025-02-07T13:58:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,17 @@
 from renku_data_services.namespace.apispec_base import BaseAPISpec
 
 
+class Slug(RootModel[str]):
+    root: str = Field(
+        ...,
+        description="A command-line/url friendly name for a namespace",
+        example="a-slug-example",
+        max_length=99,
+        min_length=1,
+        pattern="^(?!.*\\.git$|.*\\.atom$|.*[\\-._][\\-._].*)[a-z0-9][a-z0-9\\-_.]*$",
+    )
+
+
 class GroupRole(Enum):
     owner = "owner"
     editor = "editor"
@@ -21,6 +32,7 @@ class GroupRole(Enum):
 class NamespaceKind(Enum):
     group = "group"
     user = "user"
+    project = "project"
 
 
 class NamespaceResponse(BaseAPISpec):
@@ -59,6 +71,12 @@ class NamespaceResponse(BaseAPISpec):
         pattern="^[A-Za-z0-9-]+$",
     )
     namespace_kind: NamespaceKind
+    path: List[Slug] = Field(
+        ...,
+        description="A list of slugs that make up the path to a resource",
+        example=["group1", "project2"],
+        min_length=1,
+    )
 
 
 class GroupPermissions(BaseAPISpec):
@@ -69,16 +87,6 @@ class GroupPermissions(BaseAPISpec):
     )
 
 
-class PaginationRequest(BaseAPISpec):
-    model_config = ConfigDict(
-        extra="forbid",
-    )
-    page: int = Field(1, description="Result's page number starting from 1", ge=1)
-    per_page: int = Field(
-        20, description="The number of results per page", ge=1, le=100
-    )
-
-
 class Error(BaseAPISpec):
     code: int = Field(..., example=1404, gt=0)
     detail: Optional[str] = Field(
@@ -91,6 +99,31 @@ class ErrorResponse(BaseAPISpec):
     error: Error
 
 
+class GroupsGetParametersQuery(BaseAPISpec):
+    page: Optional[int] = Field(
+        None, description="Result's page number starting from 1", ge=1
+    )
+    per_page: Optional[int] = Field(
+        None, description="The number of results per page", ge=1, le=100
+    )
+    direct_member: bool = False
+
+
+class NamespacesGetParametersQuery(BaseAPISpec):
+    page: Optional[int] = Field(
+        None, description="Result's page number starting from 1", ge=1
+    )
+    per_page: Optional[int] = Field(
+        None, description="The number of results per page", ge=1, le=100
+    )
+    minimum_role: Optional[GroupRole] = None
+    kinds: Optional[List[NamespaceKind]] = Field(
+        None,
+        description="Which namespace kinds to include in the response",
+        min_length=1,
+    )
+
+
 class GroupResponse(BaseAPISpec):
     id: str = Field(
         ...,
@@ -261,25 +294,5 @@ class NamespaceResponseList(RootModel[List[NamespaceResponse]]):
     root: List[NamespaceResponse] = Field(..., description="A list of Renku namespaces")
 
 
-class NamespaceGetQuery(PaginationRequest):
-    minimum_role: Optional[GroupRole] = Field(
-        None, description="A minimum role to filter results by."
-    )
-
-
-class GroupsGetQuery(PaginationRequest):
-    direct_member: bool = Field(
-        False, description="A flag to filter groups where the user is a direct member."
-    )
-
-
-class GroupsGetParametersQuery(BaseAPISpec):
-    params: Optional[GroupsGetQuery] = None
-
-
-class NamespacesGetParametersQuery(BaseAPISpec):
-    params: Optional[NamespaceGetQuery] = None
-
-
 class GroupResponseList(RootModel[List[GroupResponse]]):
     root: List[GroupResponse] = Field(..., description="A list of Renku groups")

components/renku_data_services/namespace/apispec_base.py

@@ -1,8 +1,16 @@
 """Base models for API specifications."""
 
+import pydantic
 from pydantic import BaseModel, field_validator
 from ulid import ULID
 
+# NOTE: We are monkeypatching the regex engine for the root model because
+# the datamodel code generator that makes classes from the API spec does not
+# support setting this for the root model and by default the root model is using
+# the rust regex create which does not support lookahead/behind regexs and we need
+# that functionality to parse slugs and prevent certain suffixes in slug names.
+pydantic.RootModel.model_config = {"regex_engine": "python-re"}
+
 
 class BaseAPISpec(BaseModel):
     """Base API specification."""