Merge branch 'upstream-master' into sync/upstream_12172024_MC

.github/workflows/e2e-test.yml

@@ -513,7 +513,8 @@ jobs:
     needs:
       [
         kserve-image-build,
-        predictor-runtime-build
+        predictor-runtime-build,
+        explainer-runtime-build
       ]
     steps:
       - name: Checkout
@@ -941,3 +942,63 @@ jobs:
         if: always()
         run: |
           ./test/scripts/gh-actions/status-check.sh
+
+  test-huggingface-server-vllm:
+    runs-on: ubuntu-22.04
+    needs:
+      [ kserve-image-build, predictor-runtime-build]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Setup Minikube
+        uses: ./.github/actions/minikube-setup
+
+      - name: KServe dependency setup
+        uses: ./.github/actions/kserve-dep-setup
+
+      - name: Download base images
+        uses: ./.github/actions/base-download
+
+      - name: Download huggingface server image
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.HUGGINGFACE_IMG }}-${{ github.sha }}
+          path: ./tmp
+
+      - name: Load docker images
+        uses: ./.github/actions/load-docker-images
+        with:
+          directory: ./tmp
+
+      - name: Install Poetry and version plugin
+        run: ./test/scripts/gh-actions/setup-poetry.sh
+
+      - name: Install KServe
+        run: |
+          ./test/scripts/gh-actions/setup-kserve.sh
+
+          kubectl get pods -n kserve
+          kubectl describe pods -n kserve
+
+      - name: Run E2E tests
+        timeout-minutes: 30
+        run: |
+          ./test/scripts/gh-actions/run-e2e-tests.sh "vllm" "1"
+
+      - name: Check system status
+        if: always()
+        run: |
+          ./test/scripts/gh-actions/status-check.sh

.github/workflows/huggingface-cpu-docker-publish.yml

@@ -0,0 +1,101 @@
+name: Huggingface CPU Docker Publisher
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: huggingfaceserver 
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run tests
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64
+          context: python
+          file: python/huggingface_server_cpu_openvino.Dockerfile
+          push: false
+          # https://github.com/docker/buildx/issues/1533
+          provenance: false
+
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    # Ensure test job passes before pushing image.
+    needs: test
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Export version variable
+        run: |
+          IMAGE_ID=kserve/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          # [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo VERSION=$VERSION >> $GITHUB_ENV
+          echo IMAGE_ID=$IMAGE_ID >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64
+          context: python
+          file: python/huggingface_server_cpu_openvino.Dockerfile
+          push: true
+          tags: ${{ env.IMAGE_ID }}:${{ env.VERSION }}
+          # https://github.com/docker/buildx/issues/1533
+          provenance: false

.github/workflows/huggingface-docker-publish.yml

@@ -86,6 +86,9 @@ jobs:
           # Use Docker `latest` tag convention
           [ "$VERSION" == "master" ] && VERSION=latest
 
+          # Add "-gpu" suffix to the version
+          VERSION="${VERSION}-gpu"
+
           echo VERSION=$VERSION >> $GITHUB_ENV
           echo IMAGE_ID=$IMAGE_ID >> $GITHUB_ENV
 

.github/workflows/scheduled-image-scan.yml

@@ -24,6 +24,7 @@ jobs:
               file: python/storage-initializer.Dockerfile,
             },
             { name: router, file: router.Dockerfile },
+            { name: kserve-localmodel-controller, file: localmodel.Dockerfile },
           ]
 
     steps:
@@ -70,6 +71,8 @@ jobs:
             { name: xgbserver, file: python/xgb.Dockerfile },
             { name: pmmlserver, file: python/pmml.Dockerfile },
             { name: paddleserver, file: python/paddle.Dockerfile },
+            { name: lgbserver, file: python/lgb.Dockerfile },
+            { name: huggingfaceserver, file: python/huggingface_server.Dockerfile },
           ]
 
     steps:

.gitignore

@@ -44,10 +44,8 @@ venv/
 .openapi-generator/
 
 # Ignore files for local deployment
-config/overlays/development/manager_image_patch.yaml
 config/overlays/development/configmap/inferenceservice_patch.yaml
-config/overlays/development/manager_auth_proxy_patch.yaml
-config/overlays/development/localmodel_image_patch.yaml
+config/overlays/development/*_patch.yaml
 config/overlays/dev-image-config/inferenceservice_patch.yaml
 
 .ko.yaml

Makefile

@@ -87,7 +87,7 @@ deploy-dev: manifests
 	cd config/default && if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then \
 	echo > ../certmanager/certificate.yaml; \
 	else git checkout HEAD -- ../certmanager/certificate.yaml; fi;
-	kubectl apply --server-side=true -k config/overlays/development
+	kubectl apply --server-side=true --force-conflicts -k config/overlays/development
 	if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then ./hack/self-signed-ca.sh; fi;
 	# TODO: Add runtimes as part of default deployment
 	kubectl wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s
@@ -137,13 +137,18 @@ manifests: controller-gen
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths=./pkg/apis/serving/... output:crd:dir=config/crd/full
 	$(CONTROLLER_GEN) rbac:roleName=kserve-manager-role paths={./pkg/controller/v1alpha1/inferencegraph,./pkg/controller/v1alpha1/trainedmodel,./pkg/controller/v1beta1/...} output:rbac:artifacts:config=config/rbac
 	$(CONTROLLER_GEN) rbac:roleName=kserve-localmodel-manager-role paths=./pkg/controller/v1alpha1/localmodel output:rbac:artifacts:config=config/rbac/localmodel
+	$(CONTROLLER_GEN) rbac:roleName=kserve-localmodelnode-agent-role paths=./pkg/controller/v1alpha1/localmodelnode output:rbac:artifacts:config=config/rbac/localmodelnode
 	# Copy the cluster role to the helm chart
 	cp config/rbac/auth_proxy_role.yaml charts/kserve-resources/templates/clusterrole.yaml
 	cat config/rbac/role.yaml >> charts/kserve-resources/templates/clusterrole.yaml
 	# Copy the local model role with Helm chart while keeping the Helm template condition
 	echo '{{- if .Values.kserve.localmodel.enabled }}' > charts/kserve-resources/templates/localmodel/role.yaml
 	cat config/rbac/localmodel/role.yaml >> charts/kserve-resources/templates/localmodel/role.yaml
 	echo '{{- end }}' >> charts/kserve-resources/templates/localmodel/role.yaml
+	# Copy the local model node role with Helm chart while keeping the Helm template condition
+	echo '{{- if .Values.kserve.localmodel.enabled }}'> charts/kserve-resources/templates/localmodelnode/role.yaml
+	cat config/rbac/localmodelnode/role.yaml >> charts/kserve-resources/templates/localmodelnode/role.yaml
+	echo '{{- end }}' >> charts/kserve-resources/templates/localmodelnode/role.yaml
 
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths=./pkg/apis/serving/v1alpha1
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths=./pkg/apis/serving/v1beta1
@@ -198,8 +203,8 @@ generate: controller-gen
 	hack/update-helm-docs.sh
 
 bump-version:
-	# TBA
-	echo "bumping version numbers for this release"
+	@echo "bumping version numbers for this release"
+	@hack/prepare-for-release.sh $(PRIOR_VERSION) $(NEW_VERSION)
 
 # Build the docker image
 docker-build: test

README.md

@@ -4,6 +4,7 @@
 [![Releases](https://img.shields.io/github/release-pre/opendatahub-io/kserve.svg?sort=semver)](https://github.com/opendatahub-io/kserve/releases)
 [![LICENSE](https://img.shields.io/github/license/opendatahub-io/kserve.svg)](https://github.com/opendatahub-io/kserve/blob/master/LICENSE)
 [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://github.com/kserve/community/blob/main/README.md#questions-and-issues)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20KServe%20Guru-006BFF)](https://gurubase.io/g/kserve)
 
 KServe provides a Kubernetes [Custom Resource Definition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) for serving predictive and generative machine learning (ML) models. It aims to solve production model serving use cases by providing high abstraction interfaces for Tensorflow, XGBoost, ScikitLearn, PyTorch, Huggingface Transformer/LLM models using standardized data plane protocols.
 

charts/kserve-crd-minimal/templates/serving.kserve.io_clusterlocalmodels.yaml → charts/kserve-crd-minimal/templates/serving.kserve.io_localmodelcaches.yaml

@@ -3,14 +3,14 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.16.2
-  name: clusterlocalmodels.serving.kserve.io
+  name: localmodelcaches.serving.kserve.io
 spec:
   group: serving.kserve.io
   names:
-    kind: ClusterLocalModel
-    listKind: ClusterLocalModelList
-    plural: clusterlocalmodels
-    singular: clusterlocalmodel
+    kind: LocalModelCache
+    listKind: LocalModelCacheList
+    plural: localmodelcaches
+    singular: localmodelcache
   scope: Cluster
   versions:
   - name: v1alpha1

config/crd/full/serving.kserve.io_clusterlocalmodels.yaml → charts/kserve-crd/templates/serving.kserve.io_localmodelcaches.yaml

@@ -4,14 +4,14 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.16.2
-  name: clusterlocalmodels.serving.kserve.io
+  name: localmodelcaches.serving.kserve.io
 spec:
   group: serving.kserve.io
   names:
-    kind: ClusterLocalModel
-    listKind: ClusterLocalModelList
-    plural: clusterlocalmodels
-    singular: clusterlocalmodel
+    kind: LocalModelCache
+    listKind: LocalModelCacheList
+    plural: localmodelcaches
+    singular: localmodelcache
   scope: Cluster
   versions:
   - name: v1alpha1
@@ -73,9 +73,6 @@ spec:
                   - NodeDownloading
                   - NodeDownloaded
                   - NodeDownloadError
-                  - NodeDeleting
-                  - NodeDeletionError
-                  - NodeDeleted
                   type: string
                 type: object
             type: object

charts/kserve-resources/README.md

@@ -55,6 +55,10 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.14.0
 | kserve.controller.tag | string | `"v0.14.0"` | KServe controller contrainer image tag. |
 | kserve.controller.tolerations | list | `[]` | A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core).  For example:   tolerations:   - key: foo.bar.com/role     operator: Equal     value: master     effect: NoSchedule |
 | kserve.controller.topologySpreadConstraints | list | `[]` | A list of Kubernetes TopologySpreadConstraints, if required. For more information, see [Topology spread constraint v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core  For example:   topologySpreadConstraints:   - maxSkew: 2     topologyKey: topology.kubernetes.io/zone     whenUnsatisfiable: ScheduleAnyway     labelSelector:       matchLabels:         app.kubernetes.io/instance: kserve-controller-manager         app.kubernetes.io/component: controller |
+| kserve.localmodel.agent.hostPath | string | `"/mnt/models"` |  |
+| kserve.localmodel.agent.image | string | `"kserve/kserve-localmodelnode-agent"` |  |
+| kserve.localmodel.agent.nodeSelector | object | `{}` |  |
+| kserve.localmodel.agent.tag | string | `"v0.14.0"` |  |
 | kserve.localmodel.controller.image | string | `"kserve/kserve-localmodel-controller"` |  |
 | kserve.localmodel.controller.tag | string | `"v0.14.0"` |  |
 | kserve.localmodel.enabled | bool | `false` |  |

charts/kserve-resources/templates/clusterrole.yaml

@@ -145,13 +145,6 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - serving.kserve.io
-  resources:
-  - clusterlocalmodels
-  verbs:
-  - get
-  - list
 - apiGroups:
   - serving.kserve.io
   resources:
@@ -185,3 +178,11 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - serving.kserve.io
+  resources:
+  - localmodelcaches
+  verbs:
+  - get
+  - list
+  - watch

charts/kserve-resources/templates/clusterstoragecontainer.yaml

@@ -21,6 +21,7 @@ spec:
     - prefix: gs://
     - prefix: s3://
     - prefix: hdfs://
+    - prefix: hf://
     - prefix: webhdfs://
     - regex: "https://(.+?).blob.core.windows.net/(.+)"
     - regex: "https://(.+?).file.core.windows.net/(.+)"