From 7d0dacfbf32349a549017b827f3838e10625b961 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Wed, 20 Nov 2024 14:52:57 +0100 Subject: [PATCH] feat: add kpack k8s resources --- .devcontainer/kpack/clusterstack.yaml | 10 + .devcontainer/kpack/clusterstore.yaml | 12 ++ .devcontainer/kpack/python-builder.yaml | 19 ++ Makefile | 8 +- .../renku_data_services/errors/errors.py | 16 ++ components/renku_data_services/session/crs.py | 178 ++++++++++++++++++ .../session/kpack_client.py | 154 +++++++++++++++ 7 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/kpack/clusterstack.yaml create mode 100644 .devcontainer/kpack/clusterstore.yaml create mode 100644 .devcontainer/kpack/python-builder.yaml create mode 100644 components/renku_data_services/session/crs.py create mode 100644 components/renku_data_services/session/kpack_client.py diff --git a/.devcontainer/kpack/clusterstack.yaml b/.devcontainer/kpack/clusterstack.yaml new file mode 100644 index 000000000..fc1beef04 --- /dev/null +++ b/.devcontainer/kpack/clusterstack.yaml @@ -0,0 +1,10 @@ +apiVersion: kpack.io/v1alpha2 +kind: ClusterStack +metadata: + name: base +spec: + id: "io.buildpacks.stacks.jammy" + buildImage: + image: "paketobuildpacks/build-jammy-base" + runImage: + image: "paketobuildpacks/run-jammy-base" diff --git a/.devcontainer/kpack/clusterstore.yaml b/.devcontainer/kpack/clusterstore.yaml new file mode 100644 index 000000000..10e8d7380 --- /dev/null +++ b/.devcontainer/kpack/clusterstore.yaml @@ -0,0 +1,12 @@ +apiVersion: kpack.io/v1alpha2 +kind: ClusterStore +metadata: + name: default +spec: + sources: + - image: gcr.io/paketo-buildpacks/java + - image: gcr.io/paketo-buildpacks/nodejs + - image: gcr.io/paketo-buildpacks/python + - image: gcr.io/paketo-buildpacks/procfile + - image: gcr.io/paketo-buildpacks/source-removal + - image: gcr.io/paketo-buildpacks/environment-variables diff --git a/.devcontainer/kpack/python-builder.yaml b/.devcontainer/kpack/python-builder.yaml new file mode 100644 index 000000000..2423bb901 --- /dev/null +++ b/.devcontainer/kpack/python-builder.yaml @@ -0,0 +1,19 @@ +apiVersion: kpack.io/v1alpha2 +kind: Builder +metadata: + name: python-builder-2 + namespace: default +spec: + tag: k3d-myregistry.localhost:12345/apps/python-builder + stack: + name: base + kind: ClusterStack + store: + name: default + kind: ClusterStore + order: + - group: + - id: paketo-buildpacks/python + - id: paketo-buildpacks/procfile + - id: paketo-buildpacks/source-removal + - id: paketo-buildpacks/environment-variables diff --git a/Makefile b/Makefile index b86b973ff..a4d6fe40e 100644 --- a/Makefile +++ b/Makefile @@ -156,13 +156,19 @@ help: ## Display this help. k3d_cluster: ## Creates a k3d cluster for testing k3d cluster delete - k3d cluster create --agents 1 --k3s-arg --disable=metrics-server@server:0 + k3d registry create myregistry.localhost --port 12345 + k3d cluster create --agents 1 --k3s-arg --disable=metrics-server@server:0 --registry-use k3d-myregistry.localhost:12345 install_amaltheas: ## Installs both version of amalthea in the. NOTE: It uses the currently active k8s context. helm repo add renku https://swissdatasciencecenter.github.io/helm-charts helm repo update helm upgrade --install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION) helm upgrade --install amalthea-se renku/amalthea-sessions --version ${AMALTHEA_SESSIONS_VERSION} +install_kpack: + curl -L https://github.com/buildpacks-community/kpack/releases/download/v0.15.0/release-0.15.0.yaml | kubectl apply -f - + kubectl apply -f .devcontainer/kpack/clusterstore.yaml + kubectl apply -f .devcontainer/kpack/clusterstack.yaml + kubectl apply -f .devcontainer/kpack/python-builder.yaml # TODO: Add the version variables from the top of the file here when the charts are fully published amalthea_schema: ## Updates generates pydantic classes from CRDs diff --git a/components/renku_data_services/errors/errors.py b/components/renku_data_services/errors/errors.py index 62589aaf7..f73593e93 100644 --- a/components/renku_data_services/errors/errors.py +++ b/components/renku_data_services/errors/errors.py @@ -154,3 +154,19 @@ class SecretCreationError(BaseError): code: int = 1511 message: str = "An error occurred creating secrets." status_code: int = 500 + + +@dataclass +class CannotStartBuildError(ProgrammingError): + """Raised when an image build couldn't be started.""" + + code: int = 1512 + message: str = "An error occurred creating an image build." "" + + +@dataclass +class DeleteBuildError(ProgrammingError): + """Raised when an image build couldn't be deleted.""" + + code: int = 1513 + message: str = "An error occurred deleting an image build." "" diff --git a/components/renku_data_services/session/crs.py b/components/renku_data_services/session/crs.py new file mode 100644 index 000000000..8ecd5c353 --- /dev/null +++ b/components/renku_data_services/session/crs.py @@ -0,0 +1,178 @@ +"""Custom Resources for environments, mainly kpack.""" + +from datetime import datetime +from typing import Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class Metadata(BaseModel): + """Basic k8s metadata spec.""" + + class Config: + """Do not exclude unknown properties.""" + + extra = "allow" + + name: str + namespace: str | None = None + labels: dict[str, str] = Field(default_factory=dict) + annotations: dict[str, str] = Field(default_factory=dict) + uid: str | None = None + creationTimestamp: datetime | None = None + deletionTimestamp: datetime | None = None + + +class EnvItem(BaseModel): + """Environment variable definition.""" + + name: str + value: str + + +class ResourceRequest(BaseModel): + """Resource request entry.""" + + cpu: str + memory: str + + +class K8sResourceRequest(BaseModel): + """K8s resource request.""" + + requests: ResourceRequest + limits: ResourceRequest + + +class ImagePullSecret(BaseModel): + """K8s image pull secret.""" + + name: str + + +class PersistentVolumeReference(BaseModel): + """Reference to a persistent volume claim.""" + + persistentVolumeClaimName: str + + +class KpackBuilderReference(BaseModel): + """Refernce to Kpack builder.""" + + name: str + kind: str = "Builder" + + +class DockerImage(BaseModel): + """Docker Image.""" + + image: str + + +class DockerImageWithSecret(DockerImage): + """Docker image with a pull secret.""" + + imagePullSecrets: list[ImagePullSecret] + + +class KpackGitSource(BaseModel): + """Git repository source.""" + + url: str + revision: str + + +class KpackBlobSource(BaseModel): + """Blob/file archive source.""" + + url: str + stripComponents: str + + +class KpackSource(BaseModel): + """Kpack files source resource.""" + + git: KpackGitSource | None = None + blob: KpackBlobSource | None = None + + @model_validator(mode="after") + def validate(self) -> Self: + """Validate mode data.""" + if bool(self.git) == bool(self.blob): + raise ValueError("'git' and 'blob' are mutually exclusive and one of them must be set.") + return self + + +class KpackBuildCustomization(BaseModel): + """Customization of a kpack build.""" + + env: list[EnvItem] + + +class KpackImageSpec(BaseModel): + """KPack image spec model.""" + + tag: str + additionalTags: list[str] + serviceAccountName: str + builder: KpackBuilderReference + source: KpackSource + build: KpackBuildCustomization + successBuildHistoryLimit: int = 1 + failedBuildHistoryLimit: int = 1 + + +class KpackImage(BaseModel): + """Kpack Image resource.""" + + model_config = ConfigDict( + extra="allow", + ) + kind: str = "Image" + apiVersion: str = "kpack.io/v1alpha2" + metadata: Metadata + spec: KpackImageSpec + + +class KpackVolumeCache(BaseModel): + """Persistent volume to serve as cache for kpack build.""" + + volume: PersistentVolumeReference + + +class ImageTagReference(BaseModel): + """Reference to an image tag.""" + + tag: str + + +class KpackCacheImage(BaseModel): + """Image definition to use as build cache.""" + + registry: ImageTagReference + + +class KpackBuildSpec(BaseModel): + """Spec for kpack build.""" + + builder: DockerImageWithSecret + cache: KpackVolumeCache | KpackCacheImage + env: list[EnvItem] + resources: K8sResourceRequest + runImage: DockerImage + serviceAccountName: str + source: KpackSource + tags: list[str] + activeDeadlineSeconds: int = 1800 + + +class KpackBuild(BaseModel): + """KPack build resource.""" + + model_config = ConfigDict( + extra="allow", + ) + kind: str = "Build" + apiVersion: str = "kpack.io/v1alpha2" + metadata: Metadata + spec: KpackBuildSpec diff --git a/components/renku_data_services/session/kpack_client.py b/components/renku_data_services/session/kpack_client.py new file mode 100644 index 000000000..3f3c87e06 --- /dev/null +++ b/components/renku_data_services/session/kpack_client.py @@ -0,0 +1,154 @@ +"""K8s client for kpack.""" + +import logging + +from kr8s import NotFoundError, ServerError +from kr8s.asyncio.objects import APIObject +from kubernetes.client import ApiClient + +from renku_data_services.errors.errors import CannotStartBuildError, DeleteBuildError +from renku_data_services.notebooks.errors.intermittent import IntermittentError +from renku_data_services.notebooks.util.retries import retry_with_exponential_backoff_async +from renku_data_services.session.crs import KpackBuild, KpackImage + + +class KpackImageV1Alpha2Kr8s(APIObject): + """Spec for kpack images used by the k8s client.""" + + kind: str = "Image" + version: str = "kpack.io/v1alpha2" + namespaced: bool = True + plural: str = "images" + singular: str = "image" + scalable: bool = False + endpoint: str = "image" + + +class KpackBuildV1Alpha2Kr8s(APIObject): + """Spec for kpack build used by the k8s client.""" + + kind: str = "Build" + version: str = "kpack.io/v1alpha2" + namespaced: bool = True + plural: str = "builds" + singular: str = "build" + scalable: bool = False + endpoint: str = "build" + + +class KpackClient: + """Client for creating kpack resources in kubernetes.""" + + def __init__(self, namespace: str) -> None: + self.namespace = namespace + self.sanitize = ApiClient().sanitize_for_serialization + + async def create_build(self, manifest: KpackBuild) -> KpackBuild: + """Create a new image build.""" + manifest.metadata.namespace = self.namespace + build = await KpackBuildV1Alpha2Kr8s(manifest.model_dump(exclude_none=True, mode="json")) + build_name = manifest.metadata.name + try: + await build.create() + except ServerError as e: + logging.exception(f"Cannot create the image build {build_name} because of {e}") + raise CannotStartBuildError(message=f"Cannot create the image build {build_name}") + await build.refresh() + build_resource = await retry_with_exponential_backoff_async(lambda x: x is None)(self.get_build)(build_name) + if build_resource is None: + raise CannotStartBuildError(message=f"Cannot create the image build {build_name}") + return build_resource + + async def get_build(self, name: str) -> KpackBuild | None: + """Get an image build.""" + try: + build = await KpackBuildV1Alpha2Kr8s.get(name=name, namespace=self.namespace) + except NotFoundError: + return None + except ServerError as e: + if e.status not in [400, 404]: + logging.exception(f"Cannot get the build {name} because of {e}") + raise IntermittentError(f"Cannot get build {name} from the k8s API.") + return None + return KpackBuild.model_validate(build.to_dict()) + + async def list_builds(self, label_selector: str | None = None) -> list[KpackBuild]: + """Get a list of kpack builds.""" + try: + builds = await KpackBuildV1Alpha2Kr8s.list(namespace=self.namespace, label_selector=label_selector) + except ServerError as e: + if e.status not in [400, 404]: + logging.exception(f"Cannot list builds because of {e}") + raise IntermittentError("Cannot list builds") + return [] + output: list[KpackBuild] + if isinstance(builds, APIObject): + output = [KpackBuild.model_validate(builds.to_dict())] + else: + output = [KpackBuild.model_validate(b.to_dict()) for b in builds] + return output + + async def delete_build(self, name: str) -> None: + """Delete a kpack build.""" + build = await KpackBuildV1Alpha2Kr8s(dict(metadata=dict(name=name, namespace=self.namespace))) + try: + await build.delete(propagation_policy="Foreground") + except ServerError as e: + logging.exception(f"Cannot delete build {name} because of {e}") + raise DeleteBuildError() + return None + + async def create_image(self, manifest: KpackImage) -> KpackImage: + """Create a new image image.""" + manifest.metadata.namespace = self.namespace + image = await KpackImageV1Alpha2Kr8s(manifest.model_dump(exclude_none=True, mode="json")) + image_name = manifest.metadata.name + try: + await image.create() + except ServerError as e: + logging.exception(f"Cannot create the image image {image_name} because of {e}") + raise CannotStartBuildError(message=f"Cannot create the kpack image {image_name}") + await image.refresh() + image_resource = await retry_with_exponential_backoff_async(lambda x: x is None)(self.get_image)(image_name) + if image_resource is None: + raise CannotStartBuildError(message=f"Cannot create the kpack image {image_name}") + return image_resource + + async def get_image(self, name: str) -> KpackImage | None: + """Get an image image.""" + try: + image = await KpackImageV1Alpha2Kr8s.get(name=name, namespace=self.namespace) + except NotFoundError: + return None + except ServerError as e: + if e.status not in [400, 404]: + logging.exception(f"Cannot get the image {name} because of {e}") + raise IntermittentError(f"Cannot get image {name} from the k8s API.") + return None + return KpackImage.model_validate(image.to_dict()) + + async def list_images(self, label_selector: str | None = None) -> list[KpackImage]: + """Get a list of kpack images.""" + try: + images = await KpackImageV1Alpha2Kr8s.list(namespace=self.namespace, label_selector=label_selector) + except ServerError as e: + if e.status not in [400, 404]: + logging.exception(f"Cannot list images because of {e}") + raise IntermittentError("Cannot list images") + return [] + output: list[KpackImage] + if isinstance(images, APIObject): + output = [KpackImage.model_validate(images.to_dict())] + else: + output = [KpackImage.model_validate(b.to_dict()) for b in images] + return output + + async def delete_image(self, name: str) -> None: + """Delete a kpack image.""" + image = await KpackImageV1Alpha2Kr8s(dict(metadata=dict(name=name, namespace=self.namespace))) + try: + await image.delete(propagation_policy="Foreground") + except ServerError as e: + logging.exception(f"Cannot delete image {name} because of {e}") + raise DeleteBuildError() + return None