Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rbd: support QoS based on capacity for rbd volume #5016

Open
wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

rbd: support QoS based on capacity for rbd volume #5016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/rbd/deploy.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ make image-cephcsi
| `stripeUnit` | no | stripe unit in bytes |
| `stripeCount` | no | objects to stripe over before looping |
| `objectSize` | no | object size in bytes |
| `BaseReadIops` | no | the base limit of read operations per second |
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can all these operations can be supported for krbd?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, these operations can be supported for krbd, but we need to set QoS by cgroup after get QoS limit result by calcQosBasedOnCapacity. if we support burst QoS, krbd can not be support.

| `BaseWriteIops` | no | the base limit of write operations per second |
| `BaseReadBytesPerSecond` | no | the base limit of read bytes per second |
| `BaseWriteBytesPerSecond` | no | the base limit of write bytes per second |
| `ReadIopsPerGiB` | no | the limit of read operations per GiB |
| `WriteIopsPerGiB` | no | the limit of write operations per GiB |
| `ReadBpsPerGiB` | no | the limit of read bytes per GiB |
| `WriteBpsPerGiB` | no | the limit of write bytes per GiB |
| `BaseVolSizeBytes` | no | the min size of volume what use to calculate qos beased on capacity |
| `extraDeploy` | no | array of extra objects to deploy with the release |

**NOTE:** An accompanying CSI configuration file, needs to be provided to the
Expand Down
231 changes: 231 additions & 0 deletions e2e/rbd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -4643,6 +4644,236 @@ var _ = Describe("RBD", func() {
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})

By("validate rbd image qos", func() {
var (
baseReadIops = "2000"
baseWriteIops = "1000"
baseReadBytesPerSecond = "209715200"
baseWriteBytesPerSecond = "104857600"
readIopsPerGiB = "20"
writeIopsPerGiB = "10"
readBpsPerGiB = "2097152"
writeBpsPerGiB = "1048576"
baseVolSizeBytes = "21474836480"
)
qosParameters := map[string]string{
"BaseReadIops": baseReadIops,
"BaseWriteIops": baseWriteIops,
"BaseReadBytesPerSecond": baseReadBytesPerSecond,
"BaseWriteBytesPerSecond": baseWriteBytesPerSecond,
}
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}

err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
}()

// 1.1 create PVC
pvc, err := loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)

// 1.2 validate rbd image qos
wants := map[string]string{
"rbd_qos_read_iops_limit": baseReadIops,
"rbd_qos_write_iops_limit": baseWriteIops,
"rbd_qos_read_bps_limit": baseReadBytesPerSecond,
"rbd_qos_write_bps_limit": baseWriteBytesPerSecond,
}
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}

// 1.3 delete pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}

qosParameters = map[string]string{
"BaseReadIops": baseReadIops,
"BaseWriteIops": baseWriteIops,
"BaseReadBytesPerSecond": baseReadBytesPerSecond,
"BaseWriteBytesPerSecond": baseWriteBytesPerSecond,
"ReadIopsPerGiB": readIopsPerGiB,
"WriteIopsPerGiB": writeIopsPerGiB,
"ReadBpsPerGiB": readBpsPerGiB,
"WriteBpsPerGiB": writeBpsPerGiB,
"BaseVolSizeBytes": baseVolSizeBytes,
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}

err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}

// 2.1 create PVC
pvc, err = loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)

// 2.2 validate rbd image qos
wants = map[string]string{
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what will happen for cloned PVC or PVC created from snapshot, are they going to get the same values or different values?

Can you please add a test case of that as well

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cloned PVC or PVC created from snapshot get the same values after my testing on the ceph.

"rbd_qos_read_iops_limit": "3600",
"rbd_qos_write_iops_limit": "1800",
"rbd_qos_read_bps_limit": "377487360",
"rbd_qos_write_bps_limit": "188743680",
}
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}

// 3.1 create snapshot
err = createRBDSnapshotClass(f)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteRBDSnapshotClass()
if err != nil {
framework.Failf("failed to delete VolumeSnapshotClass: %v", err)
}
}()

snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
err = createSnapshot(&snap, deployTimeout)
if err != nil {
framework.Failf("failed to create snapshot: %v", err)
}
// validate created backend rbd images
// parent PVC + snapshot
totalImages := 2
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)

// 3.2 create pvc from snapshot
pvcClone, err := loadPVC(pvcClonePath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC: %v", err)
}
// validate created backend rbd images
// parent pvc + snapshot + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)

// 3.3 validate rbd image qos
err = validateQOS(f, pvcClone, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}

// 3.4 delete snapshot
err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
framework.Failf("failed to delete snapshot: %v", err)
}
// delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}

// 4.1 create pvc from pvc
pvcSmartClone, err := loadPVC(pvcSmartClonePath)
if err != nil {
framework.Failf("failed to load pvcSmartClone: %v", err)
}
pvcSmartClone.Namespace = f.UniqueName
pvcSmartClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to create pvc: %v", err)
}
// validate created backend rbd images
// parent pvc + temp clone + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)

// 4.2 validate rbd image qos
err = validateQOS(f, pvcSmartClone, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}

// 4.3 delete parent pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

validate the omap/imageCount etc to avoid leaks

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

// delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}

// END: validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})

By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() {
err := createRBDSnapshotClass(f)
if err != nil {
Expand Down
25 changes: 25 additions & 0 deletions e2e/rbd_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1166,3 +1166,28 @@ func validateStripe(f *framework.Framework,

return nil
}

func validateQOS(f *framework.Framework,
pvc *v1.PersistentVolumeClaim,
wants map[string]string,
) error {
metadataConfPrefix := "conf_"

imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}

rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
for k, v := range wants {
qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+k, f)
if err != nil {
return err
}
if qosVal != v {
return fmt.Errorf("%s: %s does not match expected %s", k, qosVal, v)
}
}

return nil
}
32 changes: 32 additions & 0 deletions examples/rbd/storageclass.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,38 @@ parameters:
# stripeCount: <>
# (optional) The object size in bytes.
# objectSize: <>

# rbd volume QoS.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not use https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/#the-volumeattributesclass-api as these values are not immutable as anything mentioned in the SC is mutable?

# QoS provides settings for rbd volume read/write iops
# and read/write bandwidth. There are 4 base qos parameters
# among them, when users apply for a volume capacity equal
# to or less than BaseVolSizebytes, use base qos limit.
# For the portion of capacity exceeding BaseVolSizebytes,
# QoS will be increased in steps set per GiB. If the step
# size parameter per GiB is not provided, only base QoS limit
# will be used and not associated with capacity size.
#
# note: currently supports rbd-nbd mounter.
#
# For more details
# (optional) the base limit of read operations per second.
# BaseReadIops: <>
# (optional) the base limit of write operations per second.
# BaseWriteIops: <>
# (optional) the base limit of read bytes per second.
# BaseReadBytesPerSecond: <>
# (optional) the base limit of write bytes per second.
# BaseWriteBytesPerSecond: <>
# (optional) the limit of read operations per GiB.
# ReadIopsPerGiB: <>
# (optional) the limit of write operations per GiB.
# WriteIopsPerGiB: <>
# (optional) the limit of read bytes per GiB.
# ReadBpsPerGiB: <>
# (optional) the limit of write bytes per GiB.
# WriteBpsPerGiB: <>
# (optional) min size of volume what use to calc qos beased on capacity.
# BaseVolSizeBytes:<>
reclaimPolicy: Delete
allowVolumeExpansion: true

Expand Down
31 changes: 30 additions & 1 deletion internal/rbd/controllerserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error())
}

// Get QosParameters from SC if qos configuration existing in SC
YiteGu marked this conversation as resolved.
Show resolved Hide resolved
err = rbdVol.SetQOS(ctx, req.GetParameters())
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}

err = rbdVol.Connect(cr)
if err != nil {
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
Expand Down Expand Up @@ -415,7 +421,7 @@ func (cs *ControllerServer) CreateVolume(
}
}()

err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap)
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap, req.GetParameters())
if err != nil {
if errors.Is(err, ErrFlattenInProgress) {
return nil, status.Error(codes.Aborted, err.Error())
Expand Down Expand Up @@ -712,6 +718,7 @@ func (cs *ControllerServer) createBackingImage(
secrets map[string]string,
rbdVol, parentVol *rbdVolume,
rbdSnap *rbdSnapshot,
scParams map[string]string,
) error {
var err error

Expand Down Expand Up @@ -766,6 +773,21 @@ func (cs *ControllerServer) createBackingImage(
return status.Error(codes.Internal, err.Error())
}

// Apply Qos parameters to rbd image.
err = rbdVol.ApplyQOS(ctx)
if err != nil {
log.ErrorLog(ctx, "failed apply QOS for rbd image: %v", err)

return status.Error(codes.Internal, err.Error())
}
// Save Qos parameters from SC in Image medatate, we will use it while resize volume.
err = rbdVol.SaveQOS(ctx, scParams)
if err != nil {
log.ErrorLog(ctx, "failed save QOS for rbd image: %v", err)

return status.Error(codes.Internal, err.Error())
}

return nil
}

Expand Down Expand Up @@ -1605,6 +1627,13 @@ func (cs *ControllerServer) ControllerExpandVolume(
if err != nil {
log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err)

return nil, status.Error(codes.Internal, err.Error())
}
// adjust rbd qos after resize volume.
err = rbdVol.AdjustQOS(ctx)
if err != nil {
log.ErrorLog(ctx, "failed adjust QOS for rbd image")

return nil, status.Error(codes.Internal, err.Error())
}
}
Expand Down
Loading
Loading