Skip to content

Commit

Permalink
test/e2e: multik8s: make sure backup succeeds if backup-transfer-part…
Browse files Browse the repository at this point in the history
…-size is changed during the backup

Signed-off-by: Ryotaro Banno <[email protected]>
  • Loading branch information
ushitora-anqou committed Feb 14, 2025
1 parent 030bf97 commit ff16d2d
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 0 deletions.
59 changes: 59 additions & 0 deletions test/e2e/multik8s/replication/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,65 @@ func replicationTestSuite() { //nolint:gocyclo
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
})

It("should succeed to back up if backup-transfer-part-size is changed", func(ctx SpecContext) {
namespace := util.GetUniqueName("ns-")
pvcName := util.GetUniqueName("pvc-")
backupName := util.GetUniqueName("mb-")
restoreName := util.GetUniqueName("mr-")

SetupEnvironment(namespace)

// Pause the object storage to make upload Jobs fail.
PauseObjectStorage(ctx)
defer ResumeObjectStorage(ctx)

// Create MantleBackup M0.
CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName)
writtenDataHash := WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName)
CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName, backupName)

// Wait until an upload Job is created.
WaitUploadJobCreated(ctx, PrimaryK8sCluster, namespace, backupName, 0)

// Get the expected number of the backup parts before changing backup-transfer-part-size.
pvc, err := GetPVC(PrimaryK8sCluster, namespace, pvcName)
Expect(err).NotTo(HaveOccurred())
numParts, err := GetNumberOfBackupParts(pvc.Spec.Resources.Requests.Storage())
Expect(err).NotTo(HaveOccurred())

// Change backup-transfer-part-size
originalBackupTransferPartSize, err := GetBackupTransferPartSize()
Expect(err).NotTo(HaveOccurred())
Expect(originalBackupTransferPartSize.String()).To(Equal("3Mi"))
ChangeBackupTransferPartSize("7Mi")
defer ChangeBackupTransferPartSize(originalBackupTransferPartSize.String())

// Resume the process.
ResumeObjectStorage(ctx)

// Wait for MB to be synced.
WaitMantleBackupSynced(namespace, backupName)

// .status.LargestCompleted{Export,Upload}PartNum should be correct.
Eventually(ctx, func(g Gomega) {
primaryMB, err := GetMB(PrimaryK8sCluster, namespace, backupName)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(primaryMB.Status.LargestCompletedExportPartNum).NotTo(BeNil())
g.Expect(primaryMB.Status.LargestCompletedUploadPartNum).NotTo(BeNil())
g.Expect(*primaryMB.Status.LargestCompletedExportPartNum).To(Equal(int(numParts - 1)))
g.Expect(*primaryMB.Status.LargestCompletedUploadPartNum).To(Equal(int(numParts - 1)))

secondaryMB, err := GetMB(SecondaryK8sCluster, namespace, backupName)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(secondaryMB.Status.LargestCompletedImportPartNum).NotTo(BeNil())
g.Expect(*secondaryMB.Status.LargestCompletedImportPartNum).To(Equal(int(numParts - 1)))
}).Should(Succeed())

// Make sure backups are correct.
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
})

It("should get metrics from the controller pod in the primary cluster", func(ctx SpecContext) {
metrics := []string{
`mantle_backup_creation_duration_seconds_count`,
Expand Down
49 changes: 49 additions & 0 deletions test/e2e/multik8s/testutil/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -901,3 +901,52 @@ func GetBackupTransferPartSize() (*resource.Quantity, error) {

return &qtyParsed, nil
}

func GetNumberOfBackupParts(snapshotSize *resource.Quantity) (int64, error) {
backupTransferPartSize, err := GetBackupTransferPartSize()
if err != nil {
return 0, fmt.Errorf("failed to get backup transfer part size :%w", err)
}
var expectedNumOfParts int64 = 1
if snapshotSize.Cmp(*backupTransferPartSize) > 0 { // pvcSize > backupTransferPartSize
backupTransferPartSize, ok := backupTransferPartSize.AsInt64()
if !ok {
return 0, errors.New("failed to convert backup transfer part size to i64")
}
pvcSizeI64, ok := snapshotSize.AsInt64()
if !ok {
return 0, errors.New("failed to convert pvc size to i64")
}
expectedNumOfParts = pvcSizeI64 / backupTransferPartSize
if pvcSizeI64%backupTransferPartSize != 0 {
expectedNumOfParts++
}
}
return expectedNumOfParts, nil
}

func ChangeBackupTransferPartSize(size string) {
GinkgoHelper()

deployMC, err := GetDeploy(PrimaryK8sCluster, CephClusterNamespace, MantleControllerDeployName)
Expect(err).NotTo(HaveOccurred())

args := deployMC.Spec.Template.Spec.Containers[0].Args
backupTransferPartSizeIndex := slices.IndexFunc(
args,
func(arg string) bool { return strings.HasPrefix(arg, "--backup-transfer-part-size=") },
)
Expect(backupTransferPartSizeIndex).NotTo(Equal(-1))

_, _, err = Kubectl(
PrimaryK8sCluster, nil,
"patch", "deploy", "-n", CephClusterNamespace, MantleControllerDeployName, "--type=json",
fmt.Sprintf(
`-p=[{"op": "replace", "path": "/spec/template/spec/containers/0/args/%d", `+
`"value":"--backup-transfer-part-size=%s"}]`,
backupTransferPartSizeIndex,
size,
),
)
Expect(err).NotTo(HaveOccurred())
}

0 comments on commit ff16d2d

Please sign in to comment.