From f13c7da0fb62ec53678ee70718127cef76b6cd51 Mon Sep 17 00:00:00 2001 From: ffilippopoulos Date: Thu, 6 Mar 2025 11:19:47 +0000 Subject: [PATCH] Deploy Ceph CSI driver from upstream templates --- ceph-csi-rbd/Makefile | 37 +++ ceph-csi-rbd/README.md | 2 + ceph-csi-rbd/cluster/csi-nodeplugin-rbac.yaml | 41 +++ .../cluster/csi-provisioner-rbac.yaml | 83 +++++ ceph-csi-rbd/cluster/csidriver.yaml | 17 ++ ceph-csi-rbd/cluster/kustomization.yaml | 6 + .../namespaced/csi-nodeplugin-rbac.yaml | 7 + .../namespaced/csi-provisioner-rbac.yaml | 37 +++ .../namespaced/csi-rbdplugin-provisioner.yaml | 288 ++++++++++++++++++ ceph-csi-rbd/namespaced/csi-rbdplugin.yaml | 229 ++++++++++++++ ceph-csi-rbd/namespaced/kustomization.yaml | 7 + 11 files changed, 754 insertions(+) create mode 100644 ceph-csi-rbd/Makefile create mode 100644 ceph-csi-rbd/README.md create mode 100644 ceph-csi-rbd/cluster/csi-nodeplugin-rbac.yaml create mode 100644 ceph-csi-rbd/cluster/csi-provisioner-rbac.yaml create mode 100644 ceph-csi-rbd/cluster/csidriver.yaml create mode 100644 ceph-csi-rbd/cluster/kustomization.yaml create mode 100644 ceph-csi-rbd/namespaced/csi-nodeplugin-rbac.yaml create mode 100644 ceph-csi-rbd/namespaced/csi-provisioner-rbac.yaml create mode 100644 ceph-csi-rbd/namespaced/csi-rbdplugin-provisioner.yaml create mode 100644 ceph-csi-rbd/namespaced/csi-rbdplugin.yaml create mode 100644 ceph-csi-rbd/namespaced/kustomization.yaml diff --git a/ceph-csi-rbd/Makefile b/ceph-csi-rbd/Makefile new file mode 100644 index 00000000..35d389ad --- /dev/null +++ b/ceph-csi-rbd/Makefile @@ -0,0 +1,37 @@ +.DEFAULT_GOAL := get-upstream + +VERSION=v3.13.0 + +.PHONY: get-upstream +get-upstream: + mkdir -p cluster + mkdir -p namespaced + # csi-nodeplugin-rbac.yaml + curl -Ls https://raw.githubusercontent.com/ceph/ceph-csi/refs/tags/$(VERSION)/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml > /tmp/csi-nodeplugin-rbac.yaml + sd "namespace: default" "namespace: sys-ceph" /tmp/csi-nodeplugin-rbac.yaml + yq 'select(.kind=="ClusterRole")' /tmp/csi-nodeplugin-rbac.yaml > cluster/csi-nodeplugin-rbac.yaml + echo "---" >> cluster/csi-nodeplugin-rbac.yaml + yq 'select(.kind=="ClusterRoleBinding")' /tmp/csi-nodeplugin-rbac.yaml >> cluster/csi-nodeplugin-rbac.yaml + yq 'select(.kind=="ServiceAccount")' /tmp/csi-nodeplugin-rbac.yaml > namespaced/csi-nodeplugin-rbac.yaml + rm /tmp/csi-nodeplugin-rbac.yaml + # csi-provisioner-rbac.yaml + curl -Ls https://raw.githubusercontent.com/ceph/ceph-csi/refs/tags/$(VERSION)/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml > /tmp/csi-provisioner-rbac.yaml + sd "namespace: default" "namespace: sys-ceph" /tmp/csi-provisioner-rbac.yaml + yq 'select(.kind=="ClusterRole")' /tmp/csi-provisioner-rbac.yaml > cluster/csi-provisioner-rbac.yaml + echo "---" >> cluster/csi-provisioner-rbac.yaml + yq 'select(.kind=="ClusterRoleBinding")' /tmp/csi-provisioner-rbac.yaml >> cluster/csi-provisioner-rbac.yaml + echo "---" >> cluster/csi-provisioner-rbac.yaml + yq 'select(.kind=="ServiceAccount")' /tmp/csi-provisioner-rbac.yaml > namespaced/csi-provisioner-rbac.yaml + echo "---" >> namespaced/csi-provisioner-rbac.yaml + yq 'select(.kind=="Role")' /tmp/csi-provisioner-rbac.yaml >> namespaced/csi-provisioner-rbac.yaml + echo "---" >> namespaced/csi-provisioner-rbac.yaml + yq 'select(.kind=="RoleBinding")' /tmp/csi-provisioner-rbac.yaml >> namespaced/csi-provisioner-rbac.yaml + rm /tmp/csi-provisioner-rbac.yaml + # csi-rbdplugin-provisioner.yaml + curl -Ls https://raw.githubusercontent.com/ceph/ceph-csi/refs/tags/$(VERSION)/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml > namespaced/csi-rbdplugin-provisioner.yaml + sd "namespace: default" "namespace: sys-ceph" namespaced/csi-rbdplugin-provisioner.yaml + # csi-rbdplugin.yaml + curl -Ls https://raw.githubusercontent.com/ceph/ceph-csi/refs/tags/v3.13.0/deploy/rbd/kubernetes/csi-rbdplugin.yaml > namespaced/csi-rbdplugin.yaml + sd "namespace: default" "namespace: sys-ceph" namespaced/csi-rbdplugin.yaml + # csidriver.yaml + curl -Ls https://raw.githubusercontent.com/ceph/ceph-csi/refs/tags/v3.13.0/deploy/rbd/kubernetes/csidriver.yaml > cluster/csidriver.yaml diff --git a/ceph-csi-rbd/README.md b/ceph-csi-rbd/README.md new file mode 100644 index 00000000..02108866 --- /dev/null +++ b/ceph-csi-rbd/README.md @@ -0,0 +1,2 @@ +Deploys ceph-csi from templates found at: +https://github.com/ceph/ceph-csi/tree/devel/deploy/rbd/kubernetes diff --git a/ceph-csi-rbd/cluster/csi-nodeplugin-rbac.yaml b/ceph-csi-rbd/cluster/csi-nodeplugin-rbac.yaml new file mode 100644 index 00000000..acd14a2a --- /dev/null +++ b/ceph-csi-rbd/cluster/csi-nodeplugin-rbac.yaml @@ -0,0 +1,41 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + # allow to read Vault Token and connection options from the Tenants namespace + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["list", "get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rbd-csi-nodeplugin + # replace with non-default namespace name + namespace: sys-ceph +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/ceph-csi-rbd/cluster/csi-provisioner-rbac.yaml b/ceph-csi-rbd/cluster/csi-provisioner-rbac.yaml new file mode 100644 index 00000000..028bb418 --- /dev/null +++ b/ceph-csi-rbd/cluster/csi-provisioner-rbac.yaml @@ -0,0 +1,83 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch", "create"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["get", "list", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: sys-ceph +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- diff --git a/ceph-csi-rbd/cluster/csidriver.yaml b/ceph-csi-rbd/cluster/csidriver.yaml new file mode 100644 index 00000000..1d1ae5b1 --- /dev/null +++ b/ceph-csi-rbd/cluster/csidriver.yaml @@ -0,0 +1,17 @@ +# +# /!\ DO NOT MODIFY THIS FILE +# +# This file has been automatically generated by Ceph-CSI yamlgen. +# The source for the contents can be found in the api/deploy directory, make +# your modifications there. +# +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: "rbd.csi.ceph.com" +spec: + attachRequired: true + podInfoOnMount: false + seLinuxMount: true + fsGroupPolicy: File diff --git a/ceph-csi-rbd/cluster/kustomization.yaml b/ceph-csi-rbd/cluster/kustomization.yaml new file mode 100644 index 00000000..3a29cc1f --- /dev/null +++ b/ceph-csi-rbd/cluster/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - csidriver.yaml + - csi-nodeplugin-rbac.yaml + - csi-provisioner-rbac.yaml diff --git a/ceph-csi-rbd/namespaced/csi-nodeplugin-rbac.yaml b/ceph-csi-rbd/namespaced/csi-nodeplugin-rbac.yaml new file mode 100644 index 00000000..231ba54b --- /dev/null +++ b/ceph-csi-rbd/namespaced/csi-nodeplugin-rbac.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-nodeplugin + # replace with non-default namespace name + namespace: sys-ceph diff --git a/ceph-csi-rbd/namespaced/csi-provisioner-rbac.yaml b/ceph-csi-rbd/namespaced/csi-provisioner-rbac.yaml new file mode 100644 index 00000000..0f49e2aa --- /dev/null +++ b/ceph-csi-rbd/namespaced/csi-provisioner-rbac.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: sys-ceph +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: sys-ceph + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: sys-ceph +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: sys-ceph +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/ceph-csi-rbd/namespaced/csi-rbdplugin-provisioner.yaml b/ceph-csi-rbd/namespaced/csi-rbdplugin-provisioner.yaml new file mode 100644 index 00000000..38824a2e --- /dev/null +++ b/ceph-csi-rbd/namespaced/csi-rbdplugin-provisioner.yaml @@ -0,0 +1,288 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: csi-rbdplugin-provisioner + # replace with non-default namespace name + namespace: sys-ceph + labels: + app: csi-metrics +spec: + selector: + app: csi-rbdplugin-provisioner + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8680 + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin-provisioner + # replace with non-default namespace name + namespace: sys-ceph +spec: + replicas: 3 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-rbdplugin-provisioner + topologyKey: "kubernetes.io/hostname" + serviceAccountName: rbd-csi-provisioner + priorityClassName: system-cluster-critical + containers: + - name: csi-rbdplugin + image: quay.io/cephcsi/cephcsi:v3.13.0 + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--controllerserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--pidlimit=-1" + - "--rbdhardmaxclonedepth=8" + - "--rbdsoftmaxclonedepth=4" + - "--enableprofiling=false" + - "--setmetadata=true" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # - name: KMS_CONFIGMAP_NAME + # value: encryptionConfig + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + - name: CSI_ADDONS_ENDPOINT + value: unix:///csi/csi-addons.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-config + mountPath: /etc/ceph/ + - name: oidc-token + mountPath: /run/secrets/tokens + readOnly: true + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v5.0.1 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--retry-interval-start=500ms" + - "--leader-election=true" + - "--feature-gates=HonorPVReclaimPolicy=true" + - "--prevent-volume-mode-conversion=true" + # if fstype is not specified in storageclass, ext4 is default + - "--default-fstype=ext4" + - "--extra-create-metadata=true" + - "--immediate-topology=false" + - "--http-endpoint=$(POD_IP):8090" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8090 + name: http-endpoint + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--leader-election=true" + - "--extra-create-metadata=true" + - "--enable-volume-group-snapshots=true" + - "--http-endpoint=$(POD_IP):8092" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8092 + name: http-endpoint + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.6.1 + args: + - "--v=1" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--default-fstype=ext4" + - "--http-endpoint=$(POD_IP):8093" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8093 + name: http-endpoint + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.11.1 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--leader-election" + - "--retry-interval-start=500ms" + - "--handle-volume-inuse-error=false" + - "--feature-gates=RecoverVolumeExpansionFailure=true" + - "--http-endpoint=$(POD_IP):8091" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8091 + name: http-endpoint + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin-controller + image: quay.io/cephcsi/cephcsi:v3.13.0 + args: + - "--type=controller" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--drivernamespace=$(DRIVER_NAMESPACE)" + - "--setmetadata=true" + env: + - name: DRIVER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-config + mountPath: /etc/ceph/ + - name: liveness-prometheus + image: quay.io/cephcsi/cephcsi:v3.13.0 + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport=8680" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: 8680 + name: http-metrics + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + emptyDir: { + medium: "Memory" + } + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: ceph-csi-encryption-kms-config + configMap: + name: ceph-csi-encryption-kms-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 3600 + audience: ceph-csi-kms diff --git a/ceph-csi-rbd/namespaced/csi-rbdplugin.yaml b/ceph-csi-rbd/namespaced/csi-rbdplugin.yaml new file mode 100644 index 00000000..1d642693 --- /dev/null +++ b/ceph-csi-rbd/namespaced/csi-rbdplugin.yaml @@ -0,0 +1,229 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin + # replace with non-default namespace name + namespace: sys-ceph +spec: + selector: + matchLabels: + app: csi-rbdplugin + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccountName: rbd-csi-nodeplugin + hostNetwork: true + hostPID: true + priorityClassName: system-node-critical + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/cephcsi/cephcsi:v3.13.0 + args: + - "--nodeid=$(NODE_ID)" + - "--pluginpath=/var/lib/kubelet/plugins" + - "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/" + - "--type=rbd" + - "--nodeserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)" + - "--v=5" + - "--drivername=rbd.csi.ceph.com" + - "--enableprofiling=false" + # If topology based provisioning is desired, configure required + # node labels representing the nodes topology domain + # and pass the label names below, for CSI to consume and advertise + # its equivalent topology domain + # - "--domainlabels=failure-domain/region,failure-domain/zone" + # + # Options to enable read affinity. + # If enabled Ceph CSI will fetch labels from kubernetes node and + # pass `read_from_replica=localize,crush_location=type:value` during + # rbd map command. refer: + # https://docs.ceph.com/en/latest/man/8/rbd/#kernel-rbd-krbd-options + # for more details. + # - "--enable-read-affinity=true" + # - "--crush-location-labels=topology.io/zone,topology.io/rack" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # - name: KMS_CONFIGMAP_NAME + # value: encryptionConfig + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: CSI_ADDONS_ENDPOINT + value: unix:///csi/csi-addons.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /run/mount + name: host-mount + - mountPath: /etc/selinux + name: etc-selinux + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: ceph-csi-encryption-kms-config + mountPath: /etc/ceph-csi-encryption-kms-config/ + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: ceph-logdir + mountPath: /var/log/ceph + - name: ceph-config + mountPath: /etc/ceph/ + - name: oidc-token + mountPath: /run/secrets/tokens + readOnly: true + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 + args: + - "--v=1" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-prometheus + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: quay.io/cephcsi/cephcsi:v3.13.0 + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metricsport=8680" + - "--metricspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: ceph-logdir + hostPath: + path: /var/log/ceph + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: etc-selinux + hostPath: + path: /etc/selinux + - name: host-mount + hostPath: + path: /run/mount + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: ceph-csi-encryption-kms-config + configMap: + name: ceph-csi-encryption-kms-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 3600 + audience: ceph-csi-kms +--- +# This is a service to expose the liveness metrics +apiVersion: v1 +kind: Service +metadata: + name: csi-metrics-rbdplugin + # replace with non-default namespace name + namespace: sys-ceph + labels: + app: csi-metrics +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8680 + selector: + app: csi-rbdplugin diff --git a/ceph-csi-rbd/namespaced/kustomization.yaml b/ceph-csi-rbd/namespaced/kustomization.yaml new file mode 100644 index 00000000..37fb4640 --- /dev/null +++ b/ceph-csi-rbd/namespaced/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - csi-nodeplugin-rbac.yaml + - csi-provisioner-rbac.yaml + - csi-rbdplugin-provisioner.yaml + - csi-rbdplugin.yaml