diff --git a/components/renku_data_services/migrations/versions/1e296d744eac_update_for_custom_environment_build.py b/components/renku_data_services/migrations/versions/1e296d744eac_update_for_custom_environment_build.py new file mode 100644 index 000000000..0115c11df --- /dev/null +++ b/components/renku_data_services/migrations/versions/1e296d744eac_update_for_custom_environment_build.py @@ -0,0 +1,70 @@ +"""Update for custom environment build + +Revision ID: 1e296d744eac +Revises: d71f0f795d30 +Create Date: 2025-02-03 23:09:31.954635 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "1e296d744eac" +down_revision = "d71f0f795d30" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "build_parameters", + sa.Column("id", sa.String(length=26), nullable=False), + sa.Column("repository", sa.String(length=500), nullable=False), + sa.Column("builder_variant", sa.String(length=99), nullable=False), + sa.Column("frontend_variant", sa.String(length=99), nullable=False), + sa.PrimaryKeyConstraint("id"), + schema="sessions", + ) + + op.execute("CREATE TYPE environmentimagesource AS ENUM ('image', 'build')") + + op.add_column( + "environments", + sa.Column( + "environment_image_source", + sa.Enum("image", "build", name="environmentimagesource"), + nullable=False, + server_default="image", + ), + schema="sessions", + ) + op.add_column( + "environments", + sa.Column("build_parameters_id", sa.String(length=26), nullable=True, server_default=None), + schema="sessions", + ) + op.create_foreign_key( + "environments_build_parameters_id_fk", + "environments", + "build_parameters", + ["build_parameters_id"], + ["id"], + ondelete="CASCADE", + source_schema="sessions", + referent_schema="sessions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("environments_build_parameters_id_fk", "environments", schema="sessions", type_="foreignkey") + op.drop_column("environments", "build_parameters_id", schema="sessions") + op.drop_column("environments", "environment_image_source", schema="sessions") + + op.execute("DROP TYPE environmentimagesource") + + op.drop_table("build_parameters", schema="sessions") + # ### end Alembic commands ### diff --git a/components/renku_data_services/notebooks/api/classes/k8s_client.py b/components/renku_data_services/notebooks/api/classes/k8s_client.py index c17353d08..e28bb6c99 100644 --- a/components/renku_data_services/notebooks/api/classes/k8s_client.py +++ b/components/renku_data_services/notebooks/api/classes/k8s_client.py @@ -467,7 +467,6 @@ async def get_server(self, name: str, safe_username: str) -> _SessionType | None If the request to the cache fails, fallback to the k8s API. """ - server = None try: server = await self.cache.get_server(name) except CacheError: diff --git a/components/renku_data_services/project/api.spec.yaml b/components/renku_data_services/project/api.spec.yaml index ac39de6a9..c6f923ce2 100644 --- a/components/renku_data_services/project/api.spec.yaml +++ b/components/renku_data_services/project/api.spec.yaml @@ -769,9 +769,8 @@ components: - https://github.com/SwissDataScienceCenter/project-1.git - git@github.com:SwissDataScienceCenter/project-2.git Repository: - description: A project's repository + description: A git repository URL type: string - example: git@github.com:SwissDataScienceCenter/project-1.git Visibility: description: Project's visibility levels type: string diff --git a/components/renku_data_services/project/apispec.py b/components/renku_data_services/project/apispec.py index cddce5ec8..841bfa6dd 100644 --- a/components/renku_data_services/project/apispec.py +++ b/components/renku_data_services/project/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-12-18T12:39:57+00:00 +# timestamp: 2025-02-12T08:08:38+00:00 from __future__ import annotations diff --git a/components/renku_data_services/session/api.spec.yaml b/components/renku_data_services/session/api.spec.yaml index 91fb40dbf..68a6830d3 100644 --- a/components/renku_data_services/session/api.spec.yaml +++ b/components/renku_data_services/session/api.spec.yaml @@ -260,7 +260,7 @@ components: type: array items: $ref: "#/components/schemas/Environment" - Environment: + EnvironmentWithoutContainerImage: description: A Renku 2.0 session environment type: object properties: @@ -272,8 +272,6 @@ components: $ref: "#/components/schemas/CreationDate" description: $ref: "#/components/schemas/Description" - container_image: - $ref: "#/components/schemas/ContainerImage" default_url: $ref: "#/components/schemas/DefaultUrl" uid: @@ -296,36 +294,57 @@ components: - id - name - creation_date - - container_image - port - uid - gid - default_url - example: - id: 01AN4Z79ZS6XX96588FDX0H099 - name: JupyterLab environment - creation_date: "2023-11-01T17:32:28Z" - description: JupyterLab session environment - container_image: renku-jupyter:latest - default_url: "/lab" - port: 8080 - working_directory: /home/jovyan/work - mount_directory: /home/jovyan/work - uid: 1000 - gid: 1000 - is_archive: false + Environment: + allOf: + - $ref: "#/components/schemas/EnvironmentWithoutContainerImage" + - type: object + properties: + container_image: + $ref: "#/components/schemas/ContainerImage" + required: + - container_image EnvironmentGetInLauncher: + oneOf: + - $ref: "#/components/schemas/EnvironmentWithImageGet" + - $ref: "#/components/schemas/EnvironmentWithBuildGet" + EnvironmentWithImageGet: allOf: - $ref: "#/components/schemas/Environment" - type: object properties: + environment_image_source: + $ref: "#/components/schemas/EnvironmentImageSourceImage" environment_kind: - $ref: "#/components/schemas/EnvironmentKind" + allOf: + - $ref: "#/components/schemas/EnvironmentKind" + default: custom required: + - environment_image_source - environment_kind - example: - environment_kind: global_environment - EnvironmentPostInLauncher: + EnvironmentWithBuildGet: + allOf: + - $ref: "#/components/schemas/EnvironmentWithoutContainerImage" + - type: object + properties: + container_image: + $ref: "#/components/schemas/ContainerImage" + build_parameters: + $ref: "#/components/schemas/BuildParameters" + environment_image_source: + $ref: "#/components/schemas/EnvironmentImageSourceBuild" + environment_kind: + allOf: + - $ref: "#/components/schemas/EnvironmentKind" + default: custom + required: + - build_parameters + - environment_image_source + - environment_kind + EnvironmentPostInLauncherHelper: allOf: - $ref: "#/components/schemas/EnvironmentPost" - type: object @@ -334,8 +353,10 @@ components: $ref: "#/components/schemas/EnvironmentKind" required: - environment_kind - example: - environment_kind: global_environment + EnvironmentPostInLauncher: + oneOf: + - $ref: "#/components/schemas/EnvironmentPostInLauncherHelper" + - $ref: "#/components/schemas/BuildParametersPost" EnvironmentPost: description: Data required to create a session environment type: object @@ -349,17 +370,14 @@ components: default_url: allOf: - $ref: "#/components/schemas/DefaultUrl" - - default: /lab default: /lab uid: allOf: - $ref: "#/components/schemas/EnvironmentUid" - - default: 1000 default: 1000 gid: allOf: - $ref: "#/components/schemas/EnvironmentGid" - - default: 1000 default: 1000 working_directory: $ref: "#/components/schemas/EnvironmentWorkingDirectory" @@ -368,18 +386,21 @@ components: port: allOf: - $ref: "#/components/schemas/EnvironmentPort" - - default: 8080 default: 8080 command: $ref: "#/components/schemas/EnvironmentCommand" args: $ref: "#/components/schemas/EnvironmentArgs" is_archived: - $ref: "#/components/schemas/IsArchived" + allOf: + - $ref: "#/components/schemas/IsArchived" default: false + environment_image_source: + $ref: "#/components/schemas/EnvironmentImageSourceImage" required: - name - container_image + - environment_image_source EnvironmentPatchInLauncher: allOf: - $ref: "#/components/schemas/EnvironmentPatch" @@ -387,6 +408,10 @@ components: properties: environment_kind: $ref: "#/components/schemas/EnvironmentKind" + environment_image_source: + $ref: "#/components/schemas/EnvironmentImageSource" + build_parameters: + $ref: "#/components/schemas/BuildParametersPatch" EnvironmentPatch: type: object description: Update a session environment @@ -506,6 +531,16 @@ components: minLength: 1 maxLength: 99 example: My Renku Session :) + BuilderVariant: + description: Type of virtual environment manager when building custom environments. + type: string + minLength: 1 + maxLength: 99 + FrontendVariant: + description: User's Frontend Choice. + type: string + minLength: 1 + maxLength: 99 EnvironmentIdOnlyPatch: type: object properties: @@ -519,12 +554,24 @@ components: required: - id EnvironmentKind: - description: Kind of environment to use + description: Kind of the environment type: string enum: - GLOBAL - CUSTOM - example: CUSTOM + EnvironmentImageSourceImage: + type: string + enum: + - image + EnvironmentImageSourceBuild: + type: string + enum: + - build + EnvironmentImageSource: + description: Source of the environment's image + oneOf: + - $ref: "#/components/schemas/EnvironmentImageSourceImage" + - $ref: "#/components/schemas/EnvironmentImageSourceBuild" EnvironmentId: description: Id of the environment to use type: string @@ -539,6 +586,40 @@ components: description: A description for the resource type: string maxLength: 500 + BuildParameters: + description: Build parameters + type: object + additionalProperties: false + properties: + repository: + $ref: "#/components/schemas/Repository" + builder_variant: + $ref: "#/components/schemas/BuilderVariant" + frontend_variant: + $ref: "#/components/schemas/FrontendVariant" + required: + - repository + - builder_variant + - frontend_variant + BuildParametersPost: + allOf: + - $ref: "#/components/schemas/BuildParameters" + - type: object + properties: + environment_image_source: + $ref: "#/components/schemas/EnvironmentImageSourceBuild" + required: + - environment_image_source + BuildParametersPatch: + description: Data for updating a build + type: object + properties: + repository: + $ref: "#/components/schemas/Repository" + builder_variant: + $ref: "#/components/schemas/BuilderVariant" + frontend_variant: + $ref: "#/components/schemas/FrontendVariant" ContainerImage: description: A container image type: string @@ -547,6 +628,9 @@ components: # based on https://github.com/opencontainers/distribution-spec/blob/main/spec.md pattern: "^[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*(\\/[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*)*(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}|@sha256:[a-fA-F0-9]{64}){0,1}$" example: renku/renkulab-py:3.10-0.18.1 + Repository: + description: A git repository URL + type: string DefaultUrl: description: The default path to open in a session type: string @@ -596,7 +680,9 @@ components: example: "/home/jovyan/work" EnvironmentMountDirectory: type: string - description: The location where the persistent storage for the session will be mounted, usually it should be identical to or a parent of the working directory, if left unset will default to the working directory. + description: + The location where the persistent storage for the session will be mounted, usually it should be identical to or + a parent of the working directory, if left unset will default to the working directory. minLength: 1 example: "/home/jovyan/work" EnvironmentMountDirectoryPatch: @@ -607,13 +693,13 @@ components: items: type: string description: The command that will be run i.e. will overwrite the image Dockerfile ENTRYPOINT, equivalent to command in Kubernetes - minLength: 1 + minItems: 1 EnvironmentArgs: type: array items: type: string description: The arguments that will follow the command, i.e. will overwrite the image Dockerfile CMD, equivalent to args in Kubernetes - minLength: 1 + minItems: 1 IsArchived: type: boolean description: Whether this environment is archived and not for use in new projects or not diff --git a/components/renku_data_services/session/apispec.py b/components/renku_data_services/session/apispec.py index 2376977b3..2437d12ae 100644 --- a/components/renku_data_services/session/apispec.py +++ b/components/renku_data_services/session/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2025-01-13T09:07:25+00:00 +# timestamp: 2025-02-12T08:08:39+00:00 from __future__ import annotations @@ -17,6 +17,14 @@ class EnvironmentKind(Enum): CUSTOM = "CUSTOM" +class EnvironmentImageSourceImage(Enum): + image = "image" + + +class EnvironmentImageSourceBuild(Enum): + build = "build" + + class Error(BaseAPISpec): code: int = Field(..., example=1404, gt=0) detail: Optional[str] = Field( @@ -42,7 +50,7 @@ class EnvironmentsGetParametersQuery(BaseAPISpec): get_environment_params: Optional[GetEnvironmentParams] = None -class Environment(BaseAPISpec): +class EnvironmentWithoutContainerImage(BaseAPISpec): id: str = Field( ..., description="ULID identifier", @@ -65,13 +73,6 @@ class Environment(BaseAPISpec): description: Optional[str] = Field( None, description="A description for the resource", max_length=500 ) - container_image: str = Field( - ..., - description="A container image", - example="renku/renkulab-py:3.10-0.18.1", - max_length=500, - pattern="^[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*(\\/[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*)*(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}|@sha256:[a-fA-F0-9]{64}){0,1}$", - ) default_url: str = Field( ..., description="The default path to open in a session", @@ -118,8 +119,19 @@ class Environment(BaseAPISpec): ) -class EnvironmentGetInLauncher(Environment): - environment_kind: EnvironmentKind +class Environment(EnvironmentWithoutContainerImage): + container_image: str = Field( + ..., + description="A container image", + example="renku/renkulab-py:3.10-0.18.1", + max_length=500, + pattern="^[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*(\\/[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*)*(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}|@sha256:[a-fA-F0-9]{64}){0,1}$", + ) + + +class EnvironmentWithImageGet(Environment): + environment_image_source: EnvironmentImageSourceImage + environment_kind: EnvironmentKind = "custom" class EnvironmentPost(BaseAPISpec): @@ -184,6 +196,7 @@ class EnvironmentPost(BaseAPISpec): False, description="Whether this environment is archived and not for use in new projects or not", ) + environment_image_source: EnvironmentImageSourceImage class EnvironmentPatch(BaseAPISpec): @@ -243,48 +256,6 @@ class EnvironmentPatch(BaseAPISpec): ) -class SessionLauncher(BaseAPISpec): - id: str = Field( - ..., - description="ULID identifier", - max_length=26, - min_length=26, - pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", - ) - project_id: str = Field( - ..., - description="ULID identifier", - max_length=26, - min_length=26, - pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", - ) - name: str = Field( - ..., - description="Renku session name", - example="My Renku Session :)", - max_length=99, - min_length=1, - ) - creation_date: datetime = Field( - ..., - description="The date and time the resource was created (in UTC and ISO-8601 format)", - example="2023-11-01T17:32:28Z", - ) - description: Optional[str] = Field( - None, description="A description for the resource", max_length=500 - ) - environment: EnvironmentGetInLauncher - resource_class_id: Optional[int] = Field( - ..., description="The identifier of a resource class" - ) - disk_storage: Optional[int] = Field( - None, - description="The size of disk storage for the session, in gigabytes", - example=8, - ge=1, - ) - - class EnvironmentIdOnlyPatch(BaseAPISpec): id: Optional[str] = Field( None, @@ -303,22 +274,66 @@ class EnvironmentIdOnlyPost(BaseAPISpec): ) +class BuildParameters(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + repository: str = Field(..., description="A git repository URL") + builder_variant: str = Field( + ..., + description="Type of virtual environment manager when building custom environments.", + max_length=99, + min_length=1, + ) + frontend_variant: str = Field( + ..., description="User's Frontend Choice.", max_length=99, min_length=1 + ) + + +class BuildParametersPost(BuildParameters): + environment_image_source: EnvironmentImageSourceBuild + + +class BuildParametersPatch(BaseAPISpec): + repository: Optional[str] = Field(None, description="A git repository URL") + builder_variant: Optional[str] = Field( + None, + description="Type of virtual environment manager when building custom environments.", + max_length=99, + min_length=1, + ) + frontend_variant: Optional[str] = Field( + None, description="User's Frontend Choice.", max_length=99, min_length=1 + ) + + class EnvironmentList(RootModel[List[Environment]]): root: List[Environment] = Field(..., description="A list of session environments") -class EnvironmentPostInLauncher(EnvironmentPost): +class EnvironmentWithBuildGet(EnvironmentWithoutContainerImage): + container_image: Optional[str] = Field( + None, + description="A container image", + example="renku/renkulab-py:3.10-0.18.1", + max_length=500, + pattern="^[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*(\\/[a-z0-9]+((\\.|_|__|-+)[a-z0-9]+)*)*(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}|@sha256:[a-fA-F0-9]{64}){0,1}$", + ) + build_parameters: BuildParameters + environment_image_source: EnvironmentImageSourceBuild + environment_kind: EnvironmentKind = "custom" + + +class EnvironmentPostInLauncherHelper(EnvironmentPost): environment_kind: EnvironmentKind class EnvironmentPatchInLauncher(EnvironmentPatch): environment_kind: Optional[EnvironmentKind] = None - - -class SessionLaunchersList(RootModel[List[SessionLauncher]]): - root: List[SessionLauncher] = Field( - ..., description="A list of Renku session launchers", min_length=0 - ) + environment_image_source: Optional[ + Union[EnvironmentImageSourceImage, EnvironmentImageSourceBuild] + ] = Field(None, description="Source of the environment's image") + build_parameters: Optional[BuildParametersPatch] = None class SessionLauncherPost(BaseAPISpec): @@ -351,7 +366,10 @@ class SessionLauncherPost(BaseAPISpec): example=8, ge=1, ) - environment: Union[EnvironmentPostInLauncher, EnvironmentIdOnlyPost] + environment: Union[ + EnvironmentIdOnlyPost, + Union[EnvironmentPostInLauncherHelper, BuildParametersPost], + ] class SessionLauncherPatch(BaseAPISpec): @@ -375,3 +393,51 @@ class SessionLauncherPatch(BaseAPISpec): environment: Optional[Union[EnvironmentPatchInLauncher, EnvironmentIdOnlyPatch]] = ( None ) + + +class SessionLauncher(BaseAPISpec): + id: str = Field( + ..., + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + project_id: str = Field( + ..., + description="ULID identifier", + max_length=26, + min_length=26, + pattern="^[0-7][0-9A-HJKMNP-TV-Z]{25}$", + ) + name: str = Field( + ..., + description="Renku session name", + example="My Renku Session :)", + max_length=99, + min_length=1, + ) + creation_date: datetime = Field( + ..., + description="The date and time the resource was created (in UTC and ISO-8601 format)", + example="2023-11-01T17:32:28Z", + ) + description: Optional[str] = Field( + None, description="A description for the resource", max_length=500 + ) + environment: Union[EnvironmentWithImageGet, EnvironmentWithBuildGet] + resource_class_id: Optional[int] = Field( + ..., description="The identifier of a resource class" + ) + disk_storage: Optional[int] = Field( + None, + description="The size of disk storage for the session, in gigabytes", + example=8, + ge=1, + ) + + +class SessionLaunchersList(RootModel[List[SessionLauncher]]): + root: List[SessionLauncher] = Field( + ..., description="A list of Renku session launchers", min_length=0 + ) diff --git a/components/renku_data_services/session/blueprints.py b/components/renku_data_services/session/blueprints.py index 9469fe300..2ee781dcd 100644 --- a/components/renku_data_services/session/blueprints.py +++ b/components/renku_data_services/session/blueprints.py @@ -7,7 +7,7 @@ from sanic_ext import validate from ulid import ULID -from renku_data_services import base_models +from renku_data_services import base_models, errors from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint from renku_data_services.base_api.misc import validate_query @@ -135,12 +135,22 @@ def patch(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @only_authenticated - @validate(json=apispec.SessionLauncherPatch) - async def _patch( - _: Request, user: base_models.APIUser, launcher_id: ULID, body: apispec.SessionLauncherPatch - ) -> JSONResponse: + async def _patch(request: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse: async with self.session_repo.session_maker() as session, session.begin(): current_launcher = await self.session_repo.get_launcher(user, launcher_id) + body = apispec.SessionLauncherPatch.model_validate(request.json) + + # NOTE: This is required to deal with the multiple possible types for the environment field: If some + # random fields are passed then the validation chooses the environment type to be EnvironmentIdOnlyPatch + # which might not be the case and would set the session's environment ID to None. + # TODO: Check how validation exactly works for Union types to see if we can do this in a clear way. + if isinstance(body.environment, apispec.EnvironmentIdOnlyPatch) and "id" not in request.json.get( + "environment", {} + ): + raise errors.ValidationError( + message="There are errors in the following fields, id: Input should be a valid string" + ) + launcher_patch = validate_session_launcher_patch(body, current_launcher) launcher = await self.session_repo.update_launcher( user=user, launcher_id=launcher_id, patch=launcher_patch, session=session diff --git a/components/renku_data_services/session/core.py b/components/renku_data_services/session/core.py index bd9319061..3bd435bbd 100644 --- a/components/renku_data_services/session/core.py +++ b/components/renku_data_services/session/core.py @@ -4,6 +4,7 @@ from ulid import ULID +from renku_data_services import errors from renku_data_services.base_models.core import RESET, ResetType from renku_data_services.session import apispec, models @@ -26,6 +27,63 @@ def validate_unsaved_environment( args=environment.args, command=environment.command, is_archived=environment.is_archived, + environment_image_source=models.EnvironmentImageSource.image, + ) + + +def validate_unsaved_build_parameters( + environment: apispec.BuildParameters | apispec.BuildParametersPatch, +) -> models.UnsavedBuildParameters: + """Validate an unsaved build parameters object.""" + if environment.builder_variant is None: + raise errors.ValidationError(message="The field 'builder_variant' is required") + if environment.frontend_variant is None: + raise errors.ValidationError(message="The field 'frontend_variant' is required") + if environment.repository is None: + raise errors.ValidationError(message="The field 'repository' is required") + if environment.builder_variant not in models.BuilderVariant: + raise errors.ValidationError( + message=( + f"Invalid value for the field 'builder_variant': {environment.builder_variant}: " + f"Valid values are {[e.value for e in models.BuilderVariant]}" + ) + ) + if environment.frontend_variant not in models.FrontendVariant: + raise errors.ValidationError( + message=( + f"Invalid value for the field 'frontend_variant': {environment.frontend_variant}: " + f"Valid values are {[e.value for e in models.FrontendVariant]}" + ) + ) + + return models.UnsavedBuildParameters( + repository=environment.repository, + builder_variant=environment.builder_variant, + frontend_variant=environment.frontend_variant, + ) + + +def validate_build_parameters_patch(environment: apispec.BuildParametersPatch) -> models.BuildParametersPatch: + """Validate an unsaved build parameters object.""" + if environment.builder_variant is not None and environment.builder_variant not in models.BuilderVariant: + raise errors.ValidationError( + message=( + f"Invalid value for the field 'builder_variant': {environment.builder_variant}: " + f"Valid values are {[e.value for e in models.BuilderVariant]}" + ) + ) + if environment.frontend_variant is not None and environment.frontend_variant not in models.FrontendVariant: + raise errors.ValidationError( + message=( + f"Invalid value for the field 'frontend_variant': {environment.frontend_variant}: " + f"Valid values are {[e.value for e in models.FrontendVariant]}" + ) + ) + + return models.BuildParametersPatch( + repository=environment.repository, + builder_variant=environment.builder_variant, + frontend_variant=environment.frontend_variant, ) @@ -64,6 +122,20 @@ def validate_environment_patch(patch: apispec.EnvironmentPatch) -> models.Enviro ) +def validate_environment_patch_in_launcher(patch: apispec.EnvironmentPatchInLauncher) -> models.EnvironmentPatch: + """Validate the update to a session environment inside a session launcher.""" + environment_patch = validate_environment_patch(patch) + environment_patch.environment_image_source = ( + None + if patch.environment_image_source is None + else models.EnvironmentImageSource(patch.environment_image_source.value) + ) + environment_patch.build_parameters = ( + None if patch.build_parameters is None else validate_build_parameters_patch(patch.build_parameters) + ) + return environment_patch + + def validate_unsaved_session_launcher(launcher: apispec.SessionLauncherPost) -> models.UnsavedSessionLauncher: """Validate an unsaved session launcher.""" return models.UnsavedSessionLauncher( @@ -73,9 +145,11 @@ def validate_unsaved_session_launcher(launcher: apispec.SessionLauncherPost) -> resource_class_id=launcher.resource_class_id, disk_storage=launcher.disk_storage, # NOTE: When you create an environment with a launcher the environment can only be custom - environment=validate_unsaved_environment(launcher.environment, models.EnvironmentKind.CUSTOM) - if isinstance(launcher.environment, apispec.EnvironmentPostInLauncher) - else launcher.environment.id, + environment=launcher.environment.id + if isinstance(launcher.environment, apispec.EnvironmentIdOnlyPost) + else validate_unsaved_build_parameters(launcher.environment) + if isinstance(launcher.environment, apispec.BuildParametersPost) + else validate_unsaved_environment(launcher.environment, models.EnvironmentKind.CUSTOM), ) @@ -84,37 +158,112 @@ def validate_session_launcher_patch( ) -> models.SessionLauncherPatch: """Validate the update to a session launcher.""" data_dict = patch.model_dump(exclude_unset=True, mode="json") - environment: str | models.EnvironmentPatch | models.UnsavedEnvironment | None = None - if ( - isinstance(patch.environment, apispec.EnvironmentPatchInLauncher) - and current_launcher is not None - and current_launcher.environment.environment_kind == models.EnvironmentKind.GLOBAL - and patch.environment.environment_kind == apispec.EnvironmentKind.CUSTOM - ): - # This means that the global environment is being swapped for a custom one, - # so we have to create a brand new environment, but we have to validate here. - validated_env = apispec.EnvironmentPostInLauncher.model_validate(data_dict["environment"]) - environment = models.UnsavedEnvironment( - name=validated_env.name, - description=validated_env.description, - container_image=validated_env.container_image, - default_url=validated_env.default_url, - port=validated_env.port, - working_directory=PurePosixPath(validated_env.working_directory) - if validated_env.working_directory - else None, - mount_directory=PurePosixPath(validated_env.mount_directory) if validated_env.mount_directory else None, - uid=validated_env.uid, - gid=validated_env.gid, - environment_kind=models.EnvironmentKind(validated_env.environment_kind.value), - args=validated_env.args, - command=validated_env.command, - ) - elif isinstance(patch.environment, apispec.EnvironmentPatchInLauncher): - environment = validate_environment_patch(patch.environment) + environment: str | models.EnvironmentPatch | models.UnsavedEnvironment | models.UnsavedBuildParameters | None = None + if isinstance(patch.environment, apispec.EnvironmentPatchInLauncher): + match current_launcher.environment.environment_kind, patch.environment.environment_kind: + case models.EnvironmentKind.GLOBAL, apispec.EnvironmentKind.CUSTOM: + # This means that the global environment is being swapped for a custom one, + # so we have to create a brand-new environment, but we have to validate here. + if ( + patch.environment.environment_image_source == apispec.EnvironmentImageSourceImage.image + or patch.environment.environment_image_source is None + ): + # NOTE: The custom environment is being created from an image. + validated_env = apispec.EnvironmentPostInLauncherHelper.model_validate(data_dict["environment"]) + environment = models.UnsavedEnvironment( + name=validated_env.name, + description=validated_env.description, + container_image=validated_env.container_image, + default_url=validated_env.default_url, + port=validated_env.port, + working_directory=PurePosixPath(validated_env.working_directory) + if validated_env.working_directory + else None, + mount_directory=PurePosixPath(validated_env.mount_directory) + if validated_env.mount_directory + else None, + uid=validated_env.uid, + gid=validated_env.gid, + environment_kind=models.EnvironmentKind(validated_env.environment_kind.value), + args=validated_env.args, + command=validated_env.command, + environment_image_source=models.EnvironmentImageSource.image, + ) + elif patch.environment.environment_image_source == apispec.EnvironmentImageSourceBuild.build: + # NOTE: The environment type is changed to be built, so, all required fields should be passed (as in + # a POST request). + validated_build_parameters = apispec.BuildParameters.model_validate( + data_dict.get("environment", {}).get("build_parameters", {}) + ) + environment = validate_unsaved_build_parameters(validated_build_parameters) + case models.EnvironmentKind.GLOBAL, None: + # Trying to patch a global environment with a custom environment patch. + raise errors.ValidationError( + message=( + "There are errors in the following fields, environment.environment_kind: Input should be " + "'custom'" + ) + ) + case _, apispec.EnvironmentKind.GLOBAL: + # This means that the custom environment is being swapped for a global one, but the patch is not valid. + raise errors.ValidationError( + message="There are errors in the following fields, environment.id: Input should be a valid string" + ) + case models.EnvironmentKind.CUSTOM, _: + # This means that the custom environment is being updated. + current = current_launcher.environment.environment_image_source.value + new = ( + patch.environment.environment_image_source.value + if patch.environment.environment_image_source + else None + ) + + if ( + new == "image" or (new is None and current == "image") + ) and patch.environment.build_parameters is not None: + raise errors.ValidationError( + message="There are errors in the following fields, environment.build_parameters: Must be null" + ) + elif ( + new == "build" or (new is None and current == "build") + ) and patch.environment.build_parameters is None: + raise errors.ValidationError( + message="There are errors in the following fields, environment.build_parameters: Must be set" + ) + if current == "image" and new == "build": + if not patch.environment.build_parameters: + raise errors.ValidationError( + message=( + "There are errors in the following fields, environment.build_parameters: Must be set" + ) + ) + environment = validate_unsaved_build_parameters(patch.environment.build_parameters) + elif current == "build" and new == "image": + validated_env = apispec.EnvironmentPostInLauncherHelper.model_validate(data_dict["environment"]) + environment = models.UnsavedEnvironment( + name=validated_env.name, + description=validated_env.description, + container_image=validated_env.container_image, + default_url=validated_env.default_url, + port=validated_env.port, + working_directory=PurePosixPath(validated_env.working_directory) + if validated_env.working_directory + else None, + mount_directory=PurePosixPath(validated_env.mount_directory) + if validated_env.mount_directory + else None, + uid=validated_env.uid, + gid=validated_env.gid, + environment_kind=models.EnvironmentKind(validated_env.environment_kind.value), + args=validated_env.args, + command=validated_env.command, + environment_image_source=models.EnvironmentImageSource.image, + ) + else: + environment = validate_environment_patch_in_launcher(patch.environment) elif isinstance(patch.environment, apispec.EnvironmentIdOnlyPatch): environment = patch.environment.id - resource_class_id: int | None | ResetType = None + resource_class_id: int | None | ResetType if "resource_class_id" in data_dict and data_dict["resource_class_id"] is None: # NOTE: This means that the resource class set in the DB should be removed so that the # default resource class currently set in the CRC will be used. diff --git a/components/renku_data_services/session/db.py b/components/renku_data_services/session/db.py index 0715e9ce5..0492d90b8 100644 --- a/components/renku_data_services/session/db.py +++ b/components/renku_data_services/session/db.py @@ -80,6 +80,7 @@ def __insert_environment( uid=new_environment.uid, gid=new_environment.gid, environment_kind=new_environment.environment_kind, + environment_image_source=new_environment.environment_image_source, command=new_environment.command, args=new_environment.args, creation_date=datetime.now(UTC).replace(microsecond=0), @@ -89,6 +90,46 @@ def __insert_environment( session.add(environment) return environment + def __insert_build_parameters_environment( + self, + user: base_models.APIUser, + session: AsyncSession, + launcher: schemas.SessionLauncherORM, + new_build_parameters_environment: models.UnsavedBuildParameters, + ) -> schemas.EnvironmentORM: + if user.id is None: + raise errors.UnauthorizedError( + message="You have to be authenticated to insert an environment in the DB.", quiet=True + ) + build_parameters_orm = schemas.BuildParametersORM( + builder_variant=new_build_parameters_environment.builder_variant, + frontend_variant=new_build_parameters_environment.frontend_variant, + repository=new_build_parameters_environment.repository, + ) + session.add(build_parameters_orm) + + environment_orm = schemas.EnvironmentORM( + name=launcher.name, + created_by_id=user.id, + description=f"Generated environment for {launcher.name}", + container_image="image:unknown-at-the-moment", # TODO: This should come from the build + default_url="/lab", # TODO: This should come from the build + port=8888, # TODO: This should come from the build + working_directory=None, # TODO: This should come from the build + mount_directory=None, # TODO: This should come from the build + uid=1000, # TODO: This should come from the build + gid=1000, # TODO: This should come from the build + environment_kind=models.EnvironmentKind.CUSTOM, + command=None, # TODO: This should come from the build + args=None, # TODO: This should come from the build + creation_date=datetime.now(UTC).replace(microsecond=0), + environment_image_source=models.EnvironmentImageSource.build, + build_parameters_id=build_parameters_orm.id, + build_parameters=build_parameters_orm, + ) + session.add(environment_orm) + return environment_orm + async def insert_environment( self, user: base_models.APIUser, environment: models.UnsavedEnvironment ) -> models.Environment: @@ -146,6 +187,53 @@ def __update_environment( if update.is_archived is not None: environment.is_archived = update.is_archived + async def __update_environment_build_parameters( + self, + environment: schemas.EnvironmentORM, + update: models.EnvironmentPatch, + session: AsyncSession, + launcher: schemas.SessionLauncherORM, + ) -> None: + if not update.build_parameters: + return + + build_parameters = update.build_parameters + + if build_parameters.repository is not None: + environment.build_parameters.repository = build_parameters.repository + if build_parameters.builder_variant is not None: + environment.build_parameters.builder_variant = build_parameters.builder_variant + if build_parameters.frontend_variant is not None: + environment.build_parameters.frontend_variant = build_parameters.frontend_variant + + build_parameters_orm = environment.build_parameters + created_by_id = environment.created_by_id + + environment.build_parameters_id = None + await session.delete(environment) + + environment_orm = schemas.EnvironmentORM( + name=launcher.name, + created_by_id=created_by_id, + description=f"Generated environment for {launcher.name}", + container_image="image:unknown-at-the-moment", # TODO: This should come from the build + default_url="/lab", # TODO: This should come from the build + port=8888, # TODO: This should come from the build + working_directory=None, # TODO: This should come from the build + mount_directory=None, # TODO: This should come from the build + uid=1000, # TODO: This should come from the build + gid=1000, # TODO: This should come from the build + environment_kind=models.EnvironmentKind.CUSTOM, + command=None, # TODO: This should come from the build + args=None, # TODO: This should come from the build + creation_date=datetime.now(UTC).replace(microsecond=0), + environment_image_source=models.EnvironmentImageSource.build, + build_parameters_id=build_parameters_orm.id, + build_parameters=build_parameters_orm, + ) + session.add(environment_orm) + launcher.environment = environment_orm + async def update_environment( self, user: base_models.APIUser, environment_id: ULID, patch: models.EnvironmentPatch ) -> models.Environment: @@ -279,6 +367,35 @@ async def insert_launcher( command=launcher.environment.command, args=launcher.environment.args, creation_date=datetime.now(UTC).replace(microsecond=0), + environment_image_source=models.EnvironmentImageSource.image, + ) + session.add(environment_orm) + elif isinstance(launcher.environment, models.UnsavedBuildParameters): + build_parameters_orm = schemas.BuildParametersORM( + builder_variant=launcher.environment.builder_variant, + frontend_variant=launcher.environment.frontend_variant, + repository=launcher.environment.repository, + ) + session.add(build_parameters_orm) + + environment_orm = schemas.EnvironmentORM( + name=launcher.name, + created_by_id=user.id, + description=f"Generated environment for {launcher.name}", + container_image="image:unknown-at-the-moment", # TODO: This should come from the build + default_url="/lab", # TODO: This should come from the build + port=8888, # TODO: This should come from the build + working_directory=None, # TODO: This should come from the build + mount_directory=None, # TODO: This should come from the build + uid=1000, # TODO: This should come from the build + gid=1000, # TODO: This should come from the build + environment_kind=models.EnvironmentKind.CUSTOM, + command=None, # TODO: This should come from the build + args=None, # TODO: This should come from the build + creation_date=datetime.now(UTC).replace(microsecond=0), + environment_image_source=models.EnvironmentImageSource.build, + build_parameters_id=build_parameters_orm.id, + build_parameters=build_parameters_orm, ) session.add(environment_orm) else: @@ -454,13 +571,13 @@ async def __update_launcher_environment( user: base_models.APIUser, launcher: schemas.SessionLauncherORM, session: AsyncSession, - update: models.EnvironmentPatch | models.UnsavedEnvironment | str, + update: models.EnvironmentPatch | models.UnsavedEnvironment | models.UnsavedBuildParameters | str, ) -> None: current_env_kind = launcher.environment.environment_kind match update, current_env_kind: case str() as env_id, _: # The environment in the launcher is set via ID, the new ID has to refer - # to an environment that is GLOBAL. + # to an environment that is global. old_environment = launcher.environment new_environment_id = ULID.from_str(env_id) res_env = await session.scalars( @@ -481,20 +598,30 @@ async def __update_launcher_environment( launcher.environment_id = new_environment_id launcher.environment = new_environment if old_environment.environment_kind == models.EnvironmentKind.CUSTOM: - # A custom environment exists but it is being updated to a global one + # A custom environment exists, but it is being updated to a global one # We remove the custom environment to avoid accumulating custom environments that are not associated # with any launchers. await session.delete(old_environment) case models.EnvironmentPatch(), models.EnvironmentKind.CUSTOM: # Custom environment being updated - self.__update_environment(launcher.environment, update) - case models.UnsavedEnvironment() as new_custom_environment, models.EnvironmentKind.GLOBAL if ( + if launcher.environment.environment_image_source == models.EnvironmentImageSource.build: + await self.__update_environment_build_parameters(launcher.environment, update, session, launcher) + else: + self.__update_environment(launcher.environment, update) + case models.UnsavedEnvironment() as new_custom_environment, _ if ( new_custom_environment.environment_kind == models.EnvironmentKind.CUSTOM ): # Global environment replaced by a custom one new_env = self.__insert_environment(user, session, new_custom_environment) launcher.environment = new_env await session.flush() + case models.UnsavedBuildParameters() as new_custom_build_parameters_environment, _: + # Global environment replaced by a custom one which will be built + new_env = self.__insert_build_parameters_environment( + user, session, launcher, new_custom_build_parameters_environment + ) + launcher.environment = new_env + await session.flush() case _: raise errors.ValidationError( message="Encountered an invalid payload for updating a launcher environment", quiet=True diff --git a/components/renku_data_services/session/models.py b/components/renku_data_services/session/models.py index 61019e8f5..2f7c140b6 100644 --- a/components/renku_data_services/session/models.py +++ b/components/renku_data_services/session/models.py @@ -25,6 +25,44 @@ class EnvironmentKind(StrEnum): CUSTOM = "CUSTOM" +class EnvironmentImageSource(StrEnum): + """The source of the environment image.""" + + image = "image" + build = "build" + + +class BuilderVariant(StrEnum): + """The type of environment builder.""" + + conda = "conda" + pip = "pip" + + +class FrontendVariant(StrEnum): + """The environment frontend choice.""" + + vscodium = "vscodium" + jupyterlab = "jupyterlab" + streamlit = "streamlit" + + +@dataclass(kw_only=True, frozen=True, eq=True) +class UnsavedBuildParameters: + """The parameters of a build.""" + + repository: str + builder_variant: str + frontend_variant: str + + +@dataclass(kw_only=True, frozen=True, eq=True) +class BuildParameters(UnsavedBuildParameters): + """BuildParameters saved in the database.""" + + id: ULID + + @dataclass(kw_only=True, frozen=True, eq=True) class UnsavedEnvironment: """Session environment model that has not been saved.""" @@ -39,6 +77,7 @@ class UnsavedEnvironment: uid: int = 1000 gid: int = 1000 environment_kind: EnvironmentKind + environment_image_source: EnvironmentImageSource args: list[str] | None = None command: list[str] | None = None is_archived: bool = False @@ -72,9 +111,20 @@ class Environment(UnsavedEnvironment): mount_directory: PurePosixPath | None uid: int gid: int + build_parameters: BuildParameters | None + build_parameters_id: ULID | None -@dataclass(frozen=True, eq=True, kw_only=True) +@dataclass(kw_only=True, frozen=True, eq=True) +class BuildParametersPatch: + """Patch for parameters of a build.""" + + repository: str | None = None + builder_variant: str | None = None + frontend_variant: str | None = None + + +@dataclass(eq=True, kw_only=True) class EnvironmentPatch: """Model for changes requested on a session environment.""" @@ -90,6 +140,8 @@ class EnvironmentPatch: args: list[str] | None | ResetType = None command: list[str] | None | ResetType = None is_archived: bool | None = None + build_parameters: BuildParametersPatch | None = None + environment_image_source: EnvironmentImageSource | None = None @dataclass(frozen=True, eq=True, kw_only=True) @@ -101,7 +153,7 @@ class UnsavedSessionLauncher: description: str | None resource_class_id: int | None disk_storage: int | None - environment: str | UnsavedEnvironment + environment: str | UnsavedEnvironment | UnsavedBuildParameters """When a string is passed for the environment it should be the ID of an existing environment.""" @@ -121,8 +173,8 @@ class SessionLauncherPatch: name: str | None = None description: str | None = None - # NOTE: When unsaved environment is used it means a brand new environment should be created for the + # NOTE: When unsaved environment is used it means a brand-new environment should be created for the # launcher with the update of the launcher. - environment: str | EnvironmentPatch | UnsavedEnvironment | None = None + environment: str | EnvironmentPatch | UnsavedEnvironment | UnsavedBuildParameters | None = None resource_class_id: int | None | ResetType = None disk_storage: int | None | ResetType = None diff --git a/components/renku_data_services/session/orm.py b/components/renku_data_services/session/orm.py index 0445698af..7164c7ccd 100644 --- a/components/renku_data_services/session/orm.py +++ b/components/renku_data_services/session/orm.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import PurePosixPath -from sqlalchemy import JSON, BigInteger, Boolean, DateTime, MetaData, String, false, func +from sqlalchemy import JSON, BigInteger, Boolean, DateTime, Enum, MetaData, String, false, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, relationship from sqlalchemy.schema import ForeignKey @@ -54,7 +54,13 @@ class EnvironmentORM(BaseORM): mount_directory: Mapped[PurePosixPath | None] = mapped_column("mount_directory", PurePosixPathType, nullable=True) uid: Mapped[int] = mapped_column("uid") gid: Mapped[int] = mapped_column("gid") - environment_kind: Mapped[models.EnvironmentKind] = mapped_column("environment_kind") + environment_kind: Mapped[models.EnvironmentKind] = mapped_column( + "environment_kind", Enum(models.EnvironmentKind, values_callable=lambda e: [v.value for v in e]) + ) + environment_image_source: Mapped[models.EnvironmentImageSource] = mapped_column( + "environment_image_source", server_default="image", nullable=False + ) + args: Mapped[list[str] | None] = mapped_column("args", JSONVariant, nullable=True) command: Mapped[list[str] | None] = mapped_column("command", JSONVariant, nullable=True) @@ -67,6 +73,15 @@ class EnvironmentORM(BaseORM): "is_archived", Boolean(), default=False, server_default=false(), nullable=False ) + build_parameters_id: Mapped[ULID | None] = mapped_column( + "build_parameters_id", + ForeignKey("build_parameters.id", ondelete="CASCADE", name="environments_build_parameters_id_fk"), + nullable=True, + server_default=None, + default=None, + ) + build_parameters: Mapped["BuildParametersORM"] = relationship(lazy="joined", default=None) + def dump(self) -> models.Environment: """Create a session environment model from the EnvironmentORM.""" return models.Environment( @@ -86,6 +101,9 @@ def dump(self) -> models.Environment: args=self.args, command=self.command, is_archived=self.is_archived, + environment_image_source=self.environment_image_source, + build_parameters=self.build_parameters.dump() if self.build_parameters else None, + build_parameters_id=self.build_parameters_id, ) @@ -163,3 +181,27 @@ def dump(self) -> models.SessionLauncher: disk_storage=self.disk_storage, environment=self.environment.dump(), ) + + +class BuildParametersORM(BaseORM): + """A Renku 2.0 session build parameters.""" + + __tablename__ = "build_parameters" + + id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False) + """Id of this session build parameters object.""" + + repository: Mapped[str] = mapped_column("repository", String(500)) + + builder_variant: Mapped[str] = mapped_column("builder_variant", String(99)) + + frontend_variant: Mapped[str] = mapped_column("frontend_variant", String(99)) + + def dump(self) -> models.BuildParameters: + """Create a session build parameters model from the BuildParametersORM.""" + return models.BuildParameters( + id=self.id, + repository=self.repository, + builder_variant=self.builder_variant, + frontend_variant=self.frontend_variant, + ) diff --git a/poetry.lock b/poetry.lock index bb7654029..6e144052c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -4208,6 +4208,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a52d48f4e7bf9005e8f0a89209bf9a73f7190ddf0489eee5eb51377385f59f2a"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"}, @@ -4216,6 +4217,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1492a6051dab8d912fc2adeef0e8c72216b24d57bd896ea607cb90bb0c4981d3"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"}, @@ -4224,6 +4226,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"}, @@ -4232,6 +4235,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f6f3eac23941b32afccc23081e1f50612bdbe4e982012ef4f5797986828cd01"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"}, @@ -4240,6 +4244,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2c59aa6170b990d8d2719323e628aaf36f3bfbc1c26279c0eeeb24d05d2d11c7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"}, {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, diff --git a/test/bases/renku_data_services/data_api/conftest.py b/test/bases/renku_data_services/data_api/conftest.py index eb5b2461e..0792c7042 100644 --- a/test/bases/renku_data_services/data_api/conftest.py +++ b/test/bases/renku_data_services/data_api/conftest.py @@ -6,7 +6,7 @@ import pytest_asyncio from authzed.api.v1 import Relationship, RelationshipUpdate, SubjectReference, WriteRelationshipsRequest from sanic import Sanic -from sanic_testing.testing import SanicASGITestClient +from sanic_testing.testing import SanicASGITestClient, TestingResponse from ulid import ULID from components.renku_data_services.utils.middleware import validate_null_byte @@ -211,6 +211,26 @@ async def sanic_client( return sanic_client_with_migrations +@pytest_asyncio.fixture +async def post(sanic_client, user_headers, admin_headers): + async def post_helper(url: str, admin: bool = False, payload: dict[str, Any] = None) -> TestingResponse: + headers = admin_headers if admin else user_headers + _, response = await sanic_client.post(url, headers=headers, json=payload) + return response + + return post_helper + + +@pytest_asyncio.fixture +async def patch(sanic_client, user_headers, admin_headers): + async def patch_helper(url: str, admin: bool = False, payload: Any | None = None) -> TestingResponse: + headers = admin_headers if admin else user_headers + _, response = await sanic_client.patch(url, headers=headers, json=payload) + return response + + return patch_helper + + @pytest_asyncio.fixture async def create_project(sanic_client, user_headers, admin_headers, regular_user, admin_user): async def create_project_helper( @@ -306,6 +326,7 @@ async def create_session_environment_helper(name: str, **payload) -> dict[str, A payload.update({"name": name}) payload["description"] = payload.get("description") or "A session environment." payload["container_image"] = payload.get("container_image") or "some_image:some_tag" + payload["environment_image_source"] = payload.get("environment_image_source") or "image" _, res = await sanic_client.post("/api/data/environments", headers=admin_headers, json=payload) @@ -327,6 +348,7 @@ async def create_session_launcher_helper(name: str, project_id: str, **payload) "environment_kind": "CUSTOM", "name": "Test", "container_image": "some_image:some_tag", + "environment_image_source": "image", } _, res = await sanic_client.post("/api/data/session_launchers", headers=user_headers, json=payload) diff --git a/test/bases/renku_data_services/data_api/test_sessions.py b/test/bases/renku_data_services/data_api/test_sessions.py index 6f68b7b25..dc85a4a79 100644 --- a/test/bases/renku_data_services/data_api/test_sessions.py +++ b/test/bases/renku_data_services/data_api/test_sessions.py @@ -107,6 +107,7 @@ async def test_post_session_environment(sanic_client: SanicASGITestClient, admin "name": "Environment 1", "description": "A session environment.", "container_image": image_name, + "environment_image_source": "image", } _, res = await sanic_client.post("/api/data/environments", headers=admin_headers, json=payload) @@ -150,6 +151,7 @@ async def test_post_session_environment_unauthorized(sanic_client: SanicASGITest "name": "Environment 1", "description": "A session environment.", "container_image": "some_image:some_tag", + "environment_image_source": "image", } _, res = await sanic_client.post("/api/data/environments", headers=user_headers, json=payload) @@ -379,19 +381,10 @@ async def test_get_project_launchers( @pytest.mark.asyncio -async def test_post_session_launcher( - sanic_client: SanicASGITestClient, - valid_resource_pool_payload: dict[str, Any], - user_headers, - admin_headers, - member_1_headers, - create_project, - create_resource_pool, -) -> None: +async def test_post_session_launcher(sanic_client, admin_headers, create_project, create_resource_pool) -> None: project = await create_project("Some project") - resource_pool_data = valid_resource_pool_payload - resource_pool = await create_resource_pool(admin=True, **resource_pool_data) + resource_pool = await create_resource_pool(admin=True) payload = { "name": "Launcher 1", @@ -403,6 +396,7 @@ async def test_post_session_launcher( "container_image": "some_image:some_tag", "name": "custom_name", "environment_kind": "CUSTOM", + "environment_image_source": "image", }, } @@ -414,13 +408,56 @@ async def test_post_session_launcher( assert res.json.get("project_id") == project["id"] assert res.json.get("description") == "A session launcher." environment = res.json.get("environment", {}) + assert environment.get("name") == "custom_name" assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("environment_image_source") == "image" assert environment.get("container_image") == "some_image:some_tag" assert environment.get("id") is not None assert res.json.get("resource_class_id") == resource_pool["classes"][0]["id"] assert res.json.get("disk_storage") == 2 +@pytest.mark.asyncio +async def test_post_session_launcher_with_environment_build( + sanic_client, + admin_headers, + create_project, + create_resource_pool, +) -> None: + project = await create_project("Some project") + + payload = { + "name": "Launcher 1", + "project_id": project["id"], + "description": "A session launcher.", + "environment": { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + "environment_image_source": "build", + }, + } + + _, response = await sanic_client.post("/api/data/session_launchers", headers=admin_headers, json=payload) + + assert response.status_code == 201, response.text + assert response.json is not None + assert response.json.get("name") == "Launcher 1" + assert response.json.get("project_id") == project["id"] + assert response.json.get("description") == "A session launcher." + environment = response.json.get("environment", {}) + assert environment.get("id") is not None + assert environment.get("name") == "Launcher 1" + assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("build_parameters") == { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + } + assert environment.get("environment_image_source") == "build" + assert environment.get("container_image") == "image:unknown-at-the-moment" + + @pytest.mark.asyncio async def test_post_session_launcher_unauthorized( sanic_client: SanicASGITestClient, @@ -489,6 +526,7 @@ async def test_patch_session_launcher( "container_image": "some_image:some_tag", "name": "custom_name", "environment_kind": "CUSTOM", + "environment_image_source": "image", }, } @@ -546,6 +584,7 @@ async def test_patch_session_launcher_environment( "container_image": "some_image:some_tag", "name": "custom_name", "environment_kind": "CUSTOM", + "environment_image_source": "image", }, } _, res = await sanic_client.post("/api/data/session_launchers", headers=user_headers, json=payload) @@ -556,19 +595,31 @@ async def test_patch_session_launcher_environment( assert environment.get("container_image") == "some_image:some_tag" assert environment.get("id") is not None + launcher_id = res.json["id"] + # Patch in a global environment patch_payload = { "environment": {"id": global_env["id"]}, } _, res = await sanic_client.patch( - f"/api/data/session_launchers/{res.json['id']}", headers=user_headers, json=patch_payload + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload ) assert res.status_code == 200, res.text assert res.json is not None - launcher_id = res.json["id"] global_env["environment_kind"] = "GLOBAL" + global_env["environment_image_source"] = "image" assert res.json["environment"] == global_env + # Trying to patch with some random fields should fail + patch_payload = { + "environment": {"random_field": "random_value"}, + } + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + assert res.status_code == 422, res.text + assert "There are errors in the following fields, id: Input should be a valid string" in res.text + # Trying to patch a field of the global environment should fail patch_payload = { "environment": {"container_image": "new_image"}, @@ -580,7 +631,12 @@ async def test_patch_session_launcher_environment( # Patching in a wholly new custom environment over the global is allowed patch_payload = { - "environment": {"container_image": "new_image", "name": "new_custom", "environment_kind": "CUSTOM"}, + "environment": { + "container_image": "new_image", + "name": "new_custom", + "environment_kind": "CUSTOM", + "environment_image_source": "image", + }, } _, res = await sanic_client.patch( f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload @@ -609,6 +665,166 @@ async def test_patch_session_launcher_environment( assert res.json["environment"].get("args") is None assert res.json["environment"].get("command") is None + # Should not be able to patch fields for the built environment + patch_payload = { + "environment": {"build_parameters": {"repository": "https://github.com/repo.get"}}, + } + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + assert res.status_code == 422, res.text + + # Should not be able to change the custom environment to be built from a repository + patch_payload = { + "environment": { + "environment_image_source": "build", + "build_parameters": { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + }, + }, + } + + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + + assert res.status_code == 200, res.text + assert res.json is not None + assert res.json.get("name") == "Launcher 1" + assert res.json.get("project_id") == project["id"] + assert res.json.get("description") == "A session launcher." + environment = res.json.get("environment", {}) + assert environment.get("id") is not None + assert environment.get("name") == "Launcher 1" + assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("build_parameters") == { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + } + assert environment.get("environment_image_source") == "build" + assert environment.get("container_image") == "image:unknown-at-the-moment" + + +@pytest.mark.asyncio +async def test_patch_session_launcher_environment_with_build_parameters( + sanic_client: SanicASGITestClient, + user_headers, + create_project, + create_resource_pool, + create_session_environment, +) -> None: + project = await create_project("Some project 1") + resource_pool = await create_resource_pool(admin=True) + global_env = await create_session_environment("Some environment") + + # Create a global environment with the launcher + payload = { + "name": "Launcher 1", + "project_id": project["id"], + "description": "A session launcher.", + "resource_class_id": resource_pool["classes"][0]["id"], + "environment": {"id": global_env["id"]}, + } + _, res = await sanic_client.post("/api/data/session_launchers", headers=user_headers, json=payload) + assert res.status_code == 201, res.text + assert res.json is not None + global_env["environment_kind"] = "GLOBAL" + global_env["environment_image_source"] = "image" + assert res.json["environment"] == global_env + + launcher_id = res.json["id"] + + patch_payload = { + "environment": { + "environment_kind": "CUSTOM", + "environment_image_source": "build", + "build_parameters": { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + }, + }, + } + + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + + assert res.status_code == 200, res.text + assert res.json is not None + assert res.json.get("name") == "Launcher 1" + assert res.json.get("project_id") == project["id"] + assert res.json.get("description") == "A session launcher." + environment = res.json.get("environment", {}) + assert environment.get("id") is not None + assert environment.get("name") == "Launcher 1" + assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("build_parameters") == { + "repository": "https://github.com/some/repo", + "builder_variant": "pip", + "frontend_variant": "jupyterlab", + } + assert environment.get("environment_image_source") == "build" + assert environment.get("container_image") == "image:unknown-at-the-moment" + + # Patch the build parameters + patch_payload = { + "environment": { + "build_parameters": { + "repository": "new_repo", + "builder_variant": "conda", + }, + }, + } + + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + + assert res.status_code == 200, res.text + assert res.json is not None + assert res.json.get("name") == "Launcher 1" + assert res.json.get("project_id") == project["id"] + assert res.json.get("description") == "A session launcher." + environment = res.json.get("environment", {}) + assert environment.get("id") is not None + assert environment.get("name") == "Launcher 1" + assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("build_parameters") == { + "repository": "new_repo", + "builder_variant": "conda", + "frontend_variant": "jupyterlab", + } + assert environment.get("environment_image_source") == "build" + assert environment.get("container_image") == "image:unknown-at-the-moment" + + # Back to a custom environment with image + patch_payload = { + "environment": { + "container_image": "new_image", + "name": "new_custom", + "environment_kind": "CUSTOM", + "environment_image_source": "image", + }, + } + _, res = await sanic_client.patch( + f"/api/data/session_launchers/{launcher_id}", headers=user_headers, json=patch_payload + ) + assert res.status_code == 200, res.text + assert res.json.get("name") == "Launcher 1" + assert res.json.get("project_id") == project["id"] + assert res.json.get("description") == "A session launcher." + environment = res.json.get("environment", {}) + assert environment.get("id") is not None + assert environment.get("name") == "new_custom" + assert environment.get("environment_kind") == "CUSTOM" + assert environment.get("build_parameters") is None + assert environment.get("environment_image_source") == "image" + assert environment.get("container_image") == "new_image" + @pytest.mark.asyncio async def test_patch_session_launcher_reset_fields( @@ -632,6 +848,7 @@ async def test_patch_session_launcher_reset_fields( "container_image": "some_image:some_tag", "name": "custom_name", "environment_kind": "CUSTOM", + "environment_image_source": "image", }, } @@ -674,6 +891,7 @@ async def test_patch_session_launcher_keeps_unset_values( "container_image": "some_image:some_tag", "environment_kind": "CUSTOM", "name": "custom_name", + "environment_image_source": "image", }, ) diff --git a/test/components/renku_pack_builder/test_environment_build.py b/test/components/renku_pack_builder/test_environment_build.py index f206ecc6d..a02f5ab09 100644 --- a/test/components/renku_pack_builder/test_environment_build.py +++ b/test/components/renku_pack_builder/test_environment_build.py @@ -1,9 +1,9 @@ -import pytest import subprocess import time - from pathlib import Path +import pytest + def kubectl_apply(namespace: str, manifest: str) -> subprocess.CompletedProcess: cmd = ["kubectl", "--namespace", namespace, "apply", "-f", manifest]