diff --git a/csi/alluxio/controllerserver.go b/csi/alluxio/controllerserver.go index 586e28cfe..a5d9cb8e8 100644 --- a/csi/alluxio/controllerserver.go +++ b/csi/alluxio/controllerserver.go @@ -36,11 +36,6 @@ type controllerServer struct { func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { volumeID := sanitizeVolumeID(req.GetName()) - if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { - glog.V(3).Infof("invalid create volume req: %v", req) - return nil, err - } - // Check arguments if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "Name missing in request") @@ -77,38 +72,6 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return &csi.DeleteVolumeResponse{}, nil } -func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if req.GetVolumeCapabilities() == nil { - return nil, status.Error(codes.InvalidArgument, "Volume capabilities missing in request") - } - - // We currently only support RWO - supportedAccessMode := &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - } - - for _, cap := range req.VolumeCapabilities { - if cap.GetAccessMode().GetMode() != supportedAccessMode.GetMode() { - return &csi.ValidateVolumeCapabilitiesResponse{Message: "Only single node writer is supported"}, nil - } - } - - return &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessMode: supportedAccessMode, - }, - }, - }, - }, nil -} - func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { return &csi.ControllerExpandVolumeResponse{}, status.Error(codes.Unimplemented, "ControllerExpandVolume is not implemented") } diff --git a/csi/alluxio/nodeserver.go b/csi/alluxio/nodeserver.go index 0857bf75b..0e85a0582 100644 --- a/csi/alluxio/nodeserver.go +++ b/csi/alluxio/nodeserver.go @@ -40,6 +40,8 @@ type nodeServer struct { mutex sync.Mutex } +const alluxioFuseHostPath = "/mnt/alluxio/fuse" + /* * When there is no app pod using the pv, the first app pod using the pv would trigger NodeStageVolume(). * Only after a successful return, NodePublishVolume() is called. @@ -55,8 +57,7 @@ type nodeServer struct { func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { targetPath := req.GetTargetPath() - stagingPath := req.GetStagingTargetPath() - + stagingPath := fmt.Sprintf("%s-%s", alluxioFuseHostPath, req.VolumeId) notMnt, err := ensureMountPoint(targetPath) if err != nil { glog.V(3).Infof("Error checking mount point: %+v.", err) @@ -109,7 +110,8 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol glog.V(3).Infof("Error creating CSI Fuse pod. %+v", err) return nil, status.Error(codes.Internal, err.Error()) } - if err := checkIfMountPointReady(req.GetStagingTargetPath()); err != nil { + stagingPath := fmt.Sprintf("%s-%s", alluxioFuseHostPath, req.VolumeId) + if err := checkIfMountPointReady(stagingPath); err != nil { glog.V(3).Infof("Mount point is not ready, or error occurs. %+v", err) return nil, status.Error(codes.Internal, err.Error()) } @@ -126,26 +128,20 @@ func getAndCompleteFusePodObj(ns *nodeServer, req *csi.NodeStageVolumeRequest) ( return nil, errors.Wrap(err, "Error getting Fuse pod object from template.") } - // Append extra information to pod name for uniqueness but not exceed maximum - csiFusePodObj.Name = getFusePodName(alluxioNamespacedName.Name, ns.nodeId, req.GetVolumeId())[:64] + // Append extra information to pod name for uniqueness + csiFusePodObj.Name = getFusePodName(alluxioNamespacedName.Name, ns.nodeId, req.GetVolumeId()) csiFusePodObj.Namespace = alluxioNamespacedName.Namespace // Set node name for scheduling csiFusePodObj.Spec.NodeName = ns.nodeId - // Set fuse mount point - csiFusePodObj.Spec.Containers[0].Args[2] = req.GetStagingTargetPath() + // Use unique mount path + stagingPath := fmt.Sprintf("%s-%s", alluxioFuseHostPath, req.VolumeId) + csiFusePodObj.Spec.InitContainers[0].Command[2] = stagingPath + csiFusePodObj.Spec.Containers[0].Args[0] = strings.ReplaceAll(csiFusePodObj.Spec.Containers[0].Args[0], alluxioFuseHostPath, stagingPath) + csiFusePodObj.Spec.Containers[0].Lifecycle.PreStop.Exec.Command[2] = stagingPath - // Set pre-stop command (umount) in pod lifecycle - lifecycle := &v1.Lifecycle{ - PreStop: &v1.Handler{ - Exec: &v1.ExecAction{ - Command: []string{"/opt/alluxio/integration/fuse/bin/alluxio-fuse", "unmount", req.GetStagingTargetPath()}, - }, - }, - } - csiFusePodObj.Spec.Containers[0].Lifecycle = lifecycle return csiFusePodObj, nil } @@ -187,12 +183,6 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag podName := getFusePodName(alluxioNamespacedName.Name, ns.nodeId, req.GetVolumeId()) if err := ns.client.CoreV1().Pods(alluxioNamespacedName.Namespace).Delete(podName, &metav1.DeleteOptions{}); err != nil { if strings.Contains(err.Error(), "not found") { - // Pod not found. Try to clean up the mount point. - command := exec.Command("umount", req.GetStagingTargetPath()) - _, err := command.CombinedOutput() - if err != nil { - glog.V(3).Infof("Error running command %v: %+v", command, err) - } return &csi.NodeUnstageVolumeResponse{}, nil } glog.V(3).Infof("Error deleting pod with name %v. %+v.", podName, err) @@ -268,5 +258,9 @@ func getFusePodObj(ns *nodeServer, alluxioNamespacedName types.NamespacedName) ( func getFusePodName(clusterName, nodeId, volumeId string) string { volumeIdParts := strings.Split(volumeId, "-") - return strings.Join([]string{clusterName, nodeId, volumeIdParts[len(volumeIdParts)-1]}, "-") + fusePodName := strings.Join([]string{clusterName, "fuse", nodeId, volumeIdParts[len(volumeIdParts)-1]}, "-") + if len(fusePodName) > 64 { + fusePodName = fusePodName[:64] + } + return fusePodName } diff --git a/deploy/charts/alluxio-csi/templates/nodeplugin.yaml b/deploy/charts/alluxio-csi/templates/nodeplugin.yaml index 47d959a75..bd4bb5ad4 100644 --- a/deploy/charts/alluxio-csi/templates/nodeplugin.yaml +++ b/deploy/charts/alluxio-csi/templates/nodeplugin.yaml @@ -12,6 +12,7 @@ {{- $name := include "alluxio-csi.name" . }} {{- $fullName := include "alluxio-csi.fullname" . }} {{- $chart := include "alluxio-csi.chart" . }} +{{- $hostMountPath := "/mnt/alluxio" }} kind: DaemonSet apiVersion: apps/v1 @@ -120,6 +121,9 @@ spec: volumeMounts: - name: plugin-dir mountPath: /plugin + - name: alluxio-fuse-mount + mountPath: {{ $hostMountPath }} + mountPropagation: "HostToContainer" - name: pods-mount-dir mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" @@ -128,6 +132,10 @@ spec: hostPath: path: /var/lib/kubelet/plugins/csi-alluxio-plugin type: DirectoryOrCreate + - name: alluxio-fuse-mount + hostPath: + path: {{ $hostMountPath }} + type: DirectoryOrCreate - name: pods-mount-dir hostPath: path: /var/lib/kubelet diff --git a/deploy/charts/alluxio/templates/csi/csi-fuse.yaml b/deploy/charts/alluxio/templates/csi/csi-fuse.yaml index a5a3d9677..e4273ec14 100644 --- a/deploy/charts/alluxio/templates/csi/csi-fuse.yaml +++ b/deploy/charts/alluxio/templates/csi/csi-fuse.yaml @@ -12,7 +12,10 @@ {{- $name := include "alluxio.name" . }} {{- $fullName := include "alluxio.fullname" . }} {{- $chart := include "alluxio.chart" . }} +{{- $hostMountPath := include "alluxio.mount.basePath" "" }} {{- $alluxioFuseMountPoint := include "alluxio.mount.basePath" "/fuse" }} +{{- $alluxioFuseLogDir := include "alluxio.basePath" "/logs"}} +{{- $alluxioFuseLogVolumeName := include "alluxio.getVolumeName" (dict "prefix" $fullName "component" "fuse-log") }} --- kind: ConfigMap @@ -37,9 +40,11 @@ data: role: alluxio-fuse spec: securityContext: - runAsUser: 0 # required for mounting to csi designated path - runAsGroup: 0 - fsGroup: 0 + runAsUser: {{ .Values.fuse.user }} + runAsGroup: {{ .Values.fuse.group }} + fsGroup: {{ .Values.fsGroup }} + hostNetwork: {{ .Values.hostNetwork }} + dnsPolicy: {{ .Values.dnsPolicy | default (.Values.hostNetwork | ternary "ClusterFirstWithHostNet" "ClusterFirst") }} {{- if .Values.serviceAccountName }} serviceAccountName: {{ .Values.serviceAccountName }} {{- end }} @@ -47,6 +52,14 @@ data: {{ include "alluxio.imagePullSecrets" . | indent 2 }} {{- end}} initContainers: + - name: create-alluxio-fuse-dir + image: {{ .Values.image }}:{{ .Values.imageTag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + command: [ "mkdir", "-p", {{ $alluxioFuseMountPoint }}] + volumeMounts: + - name: alluxio-fuse-mount + mountPath: {{ $hostMountPath }} + {{- if .Values.master.enabled }} - name: wait-master image: {{ .Values.image }}:{{ .Values.imageTag }} command: ["/bin/sh", "-c"] @@ -54,33 +67,23 @@ data: - until nslookup {{ $fullName }}-master-0; do sleep 2; done - volumeMounts: - - name: {{ $fullName }}-alluxio-conf - mountPath: /opt/alluxio/conf + {{- end }} containers: - name: alluxio-fuse image: {{ .Values.image }}:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.imagePullPolicy }} {{- if .Values.fuse.resources }} - resources: - {{- if .Values.fuse.resources.limits }} - limits: - cpu: {{ .Values.fuse.resources.limits.cpu }} - memory: {{ .Values.fuse.resources.limits.memory }} - {{- end }} - {{- if .Values.fuse.resources.requests }} - cpu: {{ .Values.fuse.resources.requests.cpu }} - memory: {{ .Values.fuse.resources.requests.memory }} - {{- end }} +{{ include "alluxio.resources" .Values.fuse.resources | indent 10 }} {{- end }} - command: [ "/entrypoint.sh" ] + command: ["/bin/sh", "-c"] args: - - fuse - - {{ required "The path of the dataset must be set." .Values.dataset.path }} - - {{ $alluxioFuseMountPoint }} - {{- range .Values.fuse.mountOptions }} - - -o {{ . }} - {{- end }} + - umount -l {{ $alluxioFuseMountPoint }}; + {{- $mountTableSource := get .Values.properties "alluxio.mount.table.source" }} + {{- if or (eq $mountTableSource "ETCD") (eq $mountTableSource "STATIC_FILE") }} + /entrypoint.sh fuse {{ $alluxioFuseMountPoint }} {{- range .Values.fuse.mountOptions }} -o {{ . }} {{- end }} + {{- else }} + /entrypoint.sh fuse {{ required "The path of the dataset must be set." .Values.dataset.path }} {{ $alluxioFuseMountPoint }} {{- range .Values.fuse.mountOptions }} -o {{ . }} {{- end }} + {{- end }} env: {{- range $key, $value := .Values.fuse.env }} - name: "{{ $key }}" @@ -88,12 +91,20 @@ data: {{- end }} securityContext: privileged: true # required by bidirectional mount + lifecycle: + preStop: + exec: + command: ["fusermount", "-u", {{ $alluxioFuseMountPoint }}] volumeMounts: - name: {{ $fullName }}-alluxio-conf mountPath: /opt/alluxio/conf - - name: pods-mount-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" + - name: alluxio-fuse-mount + mountPath: {{ $hostMountPath }} + mountPropagation: Bidirectional + {{- if .Values.hostPathForLogging }} + - name: {{ $alluxioFuseLogVolumeName }} + mountPath: {{ $alluxioFuseLogDir }} + {{- end }} {{- if .Values.secrets }} {{- include "alluxio.volumeMounts" (dict "volumeMounts" .Values.secrets.fuse "readOnly" true) | indent 8 }} {{- end }} @@ -105,13 +116,19 @@ data: {{- end }} restartPolicy: Always volumes: - - name: pods-mount-dir + - name: alluxio-fuse-mount hostPath: - path: /var/lib/kubelet - type: Directory + path: {{ $hostMountPath }} + type: DirectoryOrCreate - name: {{ $fullName }}-alluxio-conf configMap: name: {{ $fullName }}-alluxio-conf + {{- if .Values.hostPathForLogging }} + - name: {{ $alluxioFuseLogVolumeName }} + hostPath: + path: {{ .Values.fuse.hostPathForLogs }} + type: DirectoryOrCreate + {{- end }} {{- if .Values.secrets }} {{- include "alluxio.secretVolumes" .Values.secrets.fuse | indent 4 }} {{- end }} diff --git a/deploy/charts/alluxio/templates/csi/pvc.yaml b/deploy/charts/alluxio/templates/csi/pvc.yaml index c47314d37..88c241a96 100644 --- a/deploy/charts/alluxio/templates/csi/pvc.yaml +++ b/deploy/charts/alluxio/templates/csi/pvc.yaml @@ -16,8 +16,6 @@ kind: PersistentVolumeClaim metadata: name: {{ $fullName }}-alluxio-csi-fuse-pvc spec: - accessModes: - - ReadWriteOnce storageClassName: {{ $fullName }}-csi-storage-class resources: requests: diff --git a/deploy/charts/alluxio/templates/csi/storageClass.yaml b/deploy/charts/alluxio/templates/csi/storageClass.yaml index 2669963be..a2449de2d 100644 --- a/deploy/charts/alluxio/templates/csi/storageClass.yaml +++ b/deploy/charts/alluxio/templates/csi/storageClass.yaml @@ -16,7 +16,7 @@ kind: StorageClass metadata: name: {{ $fullName }}-csi-storage-class provisioner: alluxio -volumeBindingMode: Immediate +volumeBindingMode: WaitForFirstConsumer parameters: alluxioClusterName: {{ $fullName }} alluxioClusterNamespace: {{ .Release.Namespace }} diff --git a/deploy/charts/alluxio/templates/fuse/daemonset.yaml b/deploy/charts/alluxio/templates/fuse/daemonset.yaml index 0c6863643..274b29ab6 100644 --- a/deploy/charts/alluxio/templates/fuse/daemonset.yaml +++ b/deploy/charts/alluxio/templates/fuse/daemonset.yaml @@ -119,9 +119,6 @@ spec: - until nslookup {{ $fullName }}-master-0; do sleep 2; done - volumeMounts: - - name: {{ $fullName }}-alluxio-conf - mountPath: /opt/alluxio/conf {{- end }} containers: - name: alluxio-fuse diff --git a/tests/helm/expectedTemplates/csi/csi-fuse.yaml b/tests/helm/expectedTemplates/csi/csi-fuse.yaml index d8e93c7d8..514a6c221 100644 --- a/tests/helm/expectedTemplates/csi/csi-fuse.yaml +++ b/tests/helm/expectedTemplates/csi/csi-fuse.yaml @@ -22,14 +22,23 @@ data: role: alluxio-fuse spec: securityContext: - runAsUser: 0 # required for mounting to csi designated path - runAsGroup: 0 - fsGroup: 0 + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + hostNetwork: false + dnsPolicy: dummyDnsPolicy serviceAccountName: dummyServiceAccountName imagePullSecrets: - name: dummySecret1 - name: dummySecret2 initContainers: + - name: create-alluxio-fuse-dir + image: dummy/dummy:dummy + imagePullPolicy: IfNotPresent + command: [ "mkdir", "-p", /mnt/alluxio/fuse] + volumeMounts: + - name: alluxio-fuse-mount + mountPath: /mnt/alluxio - name: wait-master image: dummy/dummy:dummy command: ["/bin/sh", "-c"] @@ -37,9 +46,6 @@ data: - until nslookup dummy-alluxio-master-0; do sleep 2; done - volumeMounts: - - name: dummy-alluxio-alluxio-conf - mountPath: /opt/alluxio/conf containers: - name: alluxio-fuse image: dummy/dummy:dummy @@ -48,16 +54,13 @@ data: limits: cpu: 4 memory: 4Gi + requests: cpu: 0.5 memory: 1Gi - command: [ "/entrypoint.sh" ] + command: ["/bin/sh", "-c"] args: - - fuse - - /dummy/dataset/path - - /mnt/alluxio/fuse - - -o allow_other - - -o entry_timeout=3600 - - -o attr_timeout=3600 + - umount -l /mnt/alluxio/fuse; + /entrypoint.sh fuse /dummy/dataset/path /mnt/alluxio/fuse -o allow_other -o entry_timeout=3600 -o attr_timeout=3600 env: - name: "fuseEnvKey1" value: "fuseEnvVal1" @@ -65,12 +68,18 @@ data: value: "fuseEnvVal2" securityContext: privileged: true # required by bidirectional mount + lifecycle: + preStop: + exec: + command: ["fusermount", "-u", /mnt/alluxio/fuse] volumeMounts: - name: dummy-alluxio-alluxio-conf mountPath: /opt/alluxio/conf - - name: pods-mount-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" + - name: alluxio-fuse-mount + mountPath: /mnt/alluxio + mountPropagation: Bidirectional + - name: dummy-alluxio-fuse-log-volume + mountPath: /opt/alluxio/logs - name: dummySecretFuse1-volume mountPath: /dummyPath1 readOnly: true @@ -91,13 +100,17 @@ data: readOnly: false restartPolicy: Always volumes: - - name: pods-mount-dir + - name: alluxio-fuse-mount hostPath: - path: /var/lib/kubelet - type: Directory + path: /mnt/alluxio + type: DirectoryOrCreate - name: dummy-alluxio-alluxio-conf configMap: - name: dummy-alluxio-alluxio-conf + name: dummy-alluxio-alluxio-conf + - name: dummy-alluxio-fuse-log-volume + hostPath: + path: /mnt/alluxio/logs/fuse + type: DirectoryOrCreate - name: dummySecretFuse1-volume secret: secretName: dummySecretFuse1 diff --git a/tests/helm/expectedTemplates/csi/pvc.yaml b/tests/helm/expectedTemplates/csi/pvc.yaml index f3498ad37..fda60c547 100644 --- a/tests/helm/expectedTemplates/csi/pvc.yaml +++ b/tests/helm/expectedTemplates/csi/pvc.yaml @@ -16,8 +16,6 @@ kind: PersistentVolumeClaim metadata: name: dummy-alluxio-alluxio-csi-fuse-pvc spec: - accessModes: - - ReadWriteOnce storageClassName: dummy-alluxio-csi-storage-class resources: requests: diff --git a/tests/helm/expectedTemplates/csi/storageClass.yaml b/tests/helm/expectedTemplates/csi/storageClass.yaml index 7779b7f38..b2ccc5a13 100644 --- a/tests/helm/expectedTemplates/csi/storageClass.yaml +++ b/tests/helm/expectedTemplates/csi/storageClass.yaml @@ -16,7 +16,7 @@ kind: StorageClass metadata: name: dummy-alluxio-csi-storage-class provisioner: alluxio -volumeBindingMode: Immediate +volumeBindingMode: WaitForFirstConsumer parameters: alluxioClusterName: dummy-alluxio alluxioClusterNamespace: default diff --git a/tests/helm/expectedTemplates/fuse/daemonset.yaml b/tests/helm/expectedTemplates/fuse/daemonset.yaml index 7a8355b2d..b1d4ac9b2 100644 --- a/tests/helm/expectedTemplates/fuse/daemonset.yaml +++ b/tests/helm/expectedTemplates/fuse/daemonset.yaml @@ -126,9 +126,6 @@ spec: - until nslookup dummy-alluxio-master-0; do sleep 2; done - volumeMounts: - - name: dummy-alluxio-alluxio-conf - mountPath: /opt/alluxio/conf containers: - name: alluxio-fuse image: dummy/dummy:dummy