From 29871a6c2aa8f5ecdd89f35d284b4679c8b92d90 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 11 Feb 2025 08:42:23 +0100 Subject: [PATCH 1/2] fix: properly validate project slugs for pinned projects (#639) --- components/renku_data_services/users/api.spec.yaml | 2 +- components/renku_data_services/users/apispec.py | 6 +++--- .../renku_data_services/data_api/test_user_preferences.py | 6 ++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/renku_data_services/users/api.spec.yaml b/components/renku_data_services/users/api.spec.yaml index a70b59f9d..17c613b08 100644 --- a/components/renku_data_services/users/api.spec.yaml +++ b/components/renku_data_services/users/api.spec.yaml @@ -531,7 +531,7 @@ components: example: "user/my-project" # limitations based on allowed characters in project slugs from Gitlab from here: # https://docs.gitlab.com/ee/user/reserved_names.html - pattern: "[a-zA-Z0-9_.-/]" + pattern: "^[a-zA-Z0-9]([_.\\-/]?[a-zA-Z0-9]+)*[a-zA-Z0-9]$" AddPinnedProject: type: object additionalProperties: false diff --git a/components/renku_data_services/users/apispec.py b/components/renku_data_services/users/apispec.py index fc69d1a4b..3e59a7eeb 100644 --- a/components/renku_data_services/users/apispec.py +++ b/components/renku_data_services/users/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-12-02T09:12:47+00:00 +# timestamp: 2025-02-11T07:28:35+00:00 from __future__ import annotations @@ -44,7 +44,7 @@ class ProjectSlug(RootModel[str]): description="The slug used to identify a project", example="user/my-project", min_length=3, - pattern="[a-zA-Z0-9_.-/]", + pattern="^[a-zA-Z0-9]([_.\\-/]?[a-zA-Z0-9]+)*[a-zA-Z0-9]$", ) @@ -57,7 +57,7 @@ class AddPinnedProject(BaseAPISpec): description="The slug used to identify a project", example="user/my-project", min_length=3, - pattern="[a-zA-Z0-9_.-/]", + pattern="^[a-zA-Z0-9]([_.\\-/]?[a-zA-Z0-9]+)*[a-zA-Z0-9]$", ) diff --git a/test/bases/renku_data_services/data_api/test_user_preferences.py b/test/bases/renku_data_services/data_api/test_user_preferences.py index 8ab292ea5..f9e923346 100644 --- a/test/bases/renku_data_services/data_api/test_user_preferences.py +++ b/test/bases/renku_data_services/data_api/test_user_preferences.py @@ -71,6 +71,12 @@ async def test_post_user_preferences_pinned_projects( _, res = await create_user_preferences(sanic_client, valid_add_pinned_project_payload, api_user) assert res.status_code == 200 + _, res = await sanic_client.post( + "/api/data/user/preferences/pinned_projects", + headers={"Authorization": f"bearer {api_user.access_token}"}, + data=json.dumps(dict(project_slug="/user.2/second-project///")), + ) + assert res.status_code == 422 _, res = await sanic_client.post( "/api/data/user/preferences/pinned_projects", headers={"Authorization": f"bearer {api_user.access_token}"}, From 5bcbf81dea3fbe28e35d27a413b56735fd7f1476 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Wed, 12 Feb 2025 09:12:51 +0100 Subject: [PATCH 2/2] feat: allow creation of initial global environments on first start (#637) * feat: allow creation of initial global environments on first start * add test * change to contain env creation to migration only --- ...0996_create_initial_global_environments.py | 149 ++++++++++++++++++ .../data_api/test_migrations.py | 16 ++ .../data_api/test_sessions.py | 4 + 3 files changed, 169 insertions(+) create mode 100644 components/renku_data_services/migrations/versions/450ae3930996_create_initial_global_environments.py diff --git a/components/renku_data_services/migrations/versions/450ae3930996_create_initial_global_environments.py b/components/renku_data_services/migrations/versions/450ae3930996_create_initial_global_environments.py new file mode 100644 index 000000000..67983e0c5 --- /dev/null +++ b/components/renku_data_services/migrations/versions/450ae3930996_create_initial_global_environments.py @@ -0,0 +1,149 @@ +"""bootstrap initial global environments + +Mainly used for CI deployments so they have a envs for testing. + +Revision ID: 450ae3930996 +Revises: d71f0f795d30 +Create Date: 2025-02-07 02:34:53.408066 + +""" + +import logging +from dataclasses import dataclass + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +from renku_data_services.base_models.core import InternalServiceAdmin + +JSONVariant = sa.JSON().with_variant(JSONB(), "postgresql") +# revision identifiers, used by Alembic. +revision = "450ae3930996" +down_revision = "d71f0f795d30" +branch_labels = None +depends_on = None + + +@dataclass +class Environment: + name: str + container_image: str + default_url: str + port: int = 8888 + description: str = "" + working_directory: str | None = None + mount_directory: str | None = None + uid: int = 1000 + gid: int = 1000 + args: list[str] | None = None + command: list[str] | None = None + + +GLOBAL_ENVIRONMENTS = [ + Environment( + name="Python/Jupyter", + description="Standard python environment", + container_image="renku/renkulab-py:latest", + default_url="/lab", + working_directory="/home/jovyan/work", + mount_directory="/home/jovyan/work", + port=8888, + uid=1000, + gid=100, + command=["sh", "-c"], + args=[ + '/entrypoint.sh jupyter server --ServerApp.ip=0.0.0.0 --ServerApp.port=8888 --ServerApp.base_url=$RENKU_BASE_URL_PATH --ServerApp.token="" --ServerApp.password="" --ServerApp.allow_remote_access=true --ContentsManager.allow_hidden=true --ServerApp.allow_origin=* --ServerApp.root_dir="/home/jovyan/work"' + ], + ), + Environment( + name="Rstudio", + description="Standard R environment", + container_image="renku/renkulab-r:latest", + default_url="/rstudio", + working_directory="/home/jovyan/work", + mount_directory="/home/jovyan/work", + port=8888, + uid=1000, + gid=100, + command=["sh", "-c"], + args=[ + '/entrypoint.sh jupyter server --ServerApp.ip=0.0.0.0 --ServerApp.port=8888 --ServerApp.base_url=$RENKU_BASE_URL_PATH --ServerApp.token="" --ServerApp.password="" --ServerApp.allow_remote_access=true --ContentsManager.allow_hidden=true --ServerApp.allow_origin=* --ServerApp.root_dir="/home/jovyan/work"' + ], + ), +] + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + try: + connection = op.get_bind() + + logging.info("creating global environments") + env_stmt = sa.select(sa.column("id", type_=sa.String)).select_from(sa.table("environments", schema="sessions")) + existing_envs = connection.execute(env_stmt).all() + if existing_envs: + logging.info("skipping environment creation as there already are existing environments") + return + for env in GLOBAL_ENVIRONMENTS: + op.execute( + sa.text( + """INSERT INTO sessions.environments( + id, + name, description, + created_by_id, + creation_date, + container_image, + default_url, + port, + working_directory, + mount_directory, + uid, + gid, + args, + command, + environment_kind + )VALUES ( + generate_ulid(), + :name, + :description, + :created_by_id, + now(), + :container_image, + :default_url, + :port, + :working_directory, + :mount_directory, + :uid, + :gid, + :args, + :command, + 'GLOBAL' + )""" # nosec: B608 + ).bindparams( + sa.bindparam("name", value=env.name, type_=sa.Text), + sa.bindparam("description", value=env.description, type_=sa.Text), + sa.bindparam("created_by_id", value=InternalServiceAdmin.id, type_=sa.Text), + sa.bindparam("container_image", value=env.container_image, type_=sa.Text), + sa.bindparam("default_url", value=env.default_url, type_=sa.Text), + sa.bindparam("port", value=env.port, type_=sa.Integer), + sa.bindparam("working_directory", value=env.working_directory, type_=sa.Text), + sa.bindparam("mount_directory", value=env.mount_directory, type_=sa.Text), + sa.bindparam("uid", value=env.uid, type_=sa.Integer), + sa.bindparam("gid", value=env.gid, type_=sa.Integer), + sa.bindparam("args", value=env.args, type_=JSONVariant), + sa.bindparam("command", value=env.command, type_=JSONVariant), + ) + ) + logging.info(f"created global environment {env.name}") + + except Exception: + logging.exception("creation of intial global environments failed") + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/test/bases/renku_data_services/data_api/test_migrations.py b/test/bases/renku_data_services/data_api/test_migrations.py index 8068a3400..f4470c378 100644 --- a/test/bases/renku_data_services/data_api/test_migrations.py +++ b/test/bases/renku_data_services/data_api/test_migrations.py @@ -255,3 +255,19 @@ async def test_migration_to_1ef98b967767_and_086eb60b42c8(app_config_instance: C '--ServerApp.token="" --ServerApp.password="" --ServerApp.allow_remote_access=true ' '--ContentsManager.allow_hidden=true --ServerApp.allow_origin=* --ServerApp.root_dir="/home/jovyan/work"', ] + + +@pytest.mark.asyncio +async def test_migration_create_global_envs( + app_config_instance: Config, + sanic_client_no_migrations: SanicASGITestClient, + admin_headers: dict, + admin_user: UserInfo, + tmpdir_factory, + monkeysession, +) -> None: + run_migrations_for_app("common", "head") + envs = await app_config_instance.session_repo.get_environments() + assert len(envs) == 2 + assert any(e.name == "Python/Jupyter" for e in envs) + assert any(e.name == "Rstudio" for e in envs) diff --git a/test/bases/renku_data_services/data_api/test_sessions.py b/test/bases/renku_data_services/data_api/test_sessions.py index 6f68b7b25..e6c5203eb 100644 --- a/test/bases/renku_data_services/data_api/test_sessions.py +++ b/test/bases/renku_data_services/data_api/test_sessions.py @@ -57,6 +57,8 @@ async def test_get_all_session_environments( "Environment 1", "Environment 2", "Environment 3", + "Python/Jupyter", # environments added by bootstrap migration + "Rstudio", } _, res = await sanic_client.get("/api/data/environments?include_archived=true", headers=unauthorized_headers) @@ -68,6 +70,8 @@ async def test_get_all_session_environments( "Environment 2", "Environment 3", "Environment 4", + "Python/Jupyter", # environments added by bootstrap migration + "Rstudio", }