From 35a0b610327b507a9a17d02c40fd29c762400c83 Mon Sep 17 00:00:00 2001 From: Yuji Ito Date: Tue, 21 Jan 2025 09:37:00 +0000 Subject: [PATCH] add tests for custom rbd export-diff command Signed-off-by: Yuji Ito --- .gitignore | 1 + Makefile | 4 +- ceph/Makefile | 12 +- ceph/export-diff.patch | 37 +- ceph/test/cluster/block.go | 73 +++ ceph/test/cluster/filesystem.go | 105 ++++ ceph/test/cluster/kubectl.go | 292 +++++++++++ ceph/test/cluster/rbd.go | 244 ++++++++++ .../cluster/template/deployment_block.yaml | 26 + .../template/deployment_filesystem.yaml | 26 + ceph/test/cluster/template/pool.yaml | 10 + ceph/test/cluster/template/pvc.yaml | 13 + ceph/test/cluster/template/sc.yaml | 19 + ceph/test/content_test.go | 452 ++++++++++++++++++ ceph/test/extend_test.go | 427 +++++++++++++++++ ceph/test/options_test.go | 153 ++++++ ceph/test/regression_test.go | 315 ++++++++++++ ceph/test/rollback_test.go | 242 ++++++++++ ceph/test/snapshot_name_test.go | 428 +++++++++++++++++ ceph/test/suite_test.go | 40 +- ceph/test/util.go | 16 + 21 files changed, 2915 insertions(+), 20 deletions(-) create mode 100644 ceph/test/cluster/block.go create mode 100644 ceph/test/cluster/filesystem.go create mode 100644 ceph/test/cluster/kubectl.go create mode 100644 ceph/test/cluster/rbd.go create mode 100644 ceph/test/cluster/template/deployment_block.yaml create mode 100644 ceph/test/cluster/template/deployment_filesystem.yaml create mode 100644 ceph/test/cluster/template/pool.yaml create mode 100644 ceph/test/cluster/template/pvc.yaml create mode 100644 ceph/test/cluster/template/sc.yaml create mode 100644 ceph/test/content_test.go create mode 100644 ceph/test/extend_test.go create mode 100644 ceph/test/options_test.go create mode 100644 ceph/test/regression_test.go create mode 100644 ceph/test/rollback_test.go create mode 100644 ceph/test/snapshot_name_test.go create mode 100644 ceph/test/util.go diff --git a/.gitignore b/.gitignore index 0d07a8d2..432be252 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ Dockerfile.cross *.swo *~ +ceph/ceph-custom.tar ceph/go.mod ceph/go.sum ceph/packages/ diff --git a/Makefile b/Makefile index bd016fa9..e1c45fdb 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,9 @@ mock: mockgen .PHONY: test test: manifests generate fmt vet envtest mock ## Run tests. # adding -p 1 -v to stream logs. see https://github.com/golang/go/issues/46959 - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out -p 1 -v + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --bin-dir $(LOCALBIN) -p path)" \ + SKIP_CEPH_CMD_TEST=1 \ + go test ./... -coverprofile cover.out -p 1 -v GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint golangci-lint: diff --git a/ceph/Makefile b/ceph/Makefile index f13bd556..214b3953 100644 --- a/ceph/Makefile +++ b/ceph/Makefile @@ -15,19 +15,17 @@ export MINIKUBE_HOME .PHONY: build build: ceph-custom.tar packages/touch -ceph-custom.tar: build-docker +ceph-custom.tar: Dockerfile $(PATCH_FILES) + docker build -f Dockerfile -t ceph-custom . docker save ceph-custom -o $@ -packages/touch: build-docker +packages/touch: ceph-custom.tar rm -rf packages docker create --name ceph-proc ceph-custom docker cp ceph-proc:/packages . docker rm ceph-proc touch $@ -build-docker: Dockerfile $(PATCH_FILES) - docker build -f Dockerfile -t ceph-custom . - .PHONY: test test: ceph-custom.tar $(MAKE) -C ${E2E_DIR} launch-minikube MINIKUBE_PROFILE=minikube @@ -35,7 +33,9 @@ test: ceph-custom.tar $(MAKE) -C ${E2E_DIR} install-rook-ceph-cluster1 $(MINIKUBE_CMD) image load ceph-custom.tar $(KUBECTL_CMD) apply -k ./manifests/ - go test -v -count=1 ./test/ + env \ + KUBECTL=$(KUBECTL_CMD) \ + go test -v -count=1 -timeout 1h ./test/ .PHONY: setup setup: diff --git a/ceph/export-diff.patch b/ceph/export-diff.patch index 56a31516..55383394 100644 --- a/ceph/export-diff.patch +++ b/ceph/export-diff.patch @@ -11,7 +11,7 @@ // encryption arguments static const std::string ENCRYPTION_FORMAT("encryption-format"); diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc -index ddcf0f2c30c..2ba0be2ebb1 100644 +index ddcf0f2c30c..1b72db2bde8 100644 --- a/src/tools/rbd/action/Export.cc +++ b/src/tools/rbd/action/Export.cc @@ -123,16 +123,27 @@ private: @@ -112,8 +112,8 @@ index ddcf0f2c30c..2ba0be2ebb1 100644 "snapshot starting point") - (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object"); + (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object") -+ (at::READ_OFFSET.c_str(), po::value(), "offset in bytes") -+ (at::READ_LENGTH.c_str(), po::value(), "length in bytes") ++ (at::READ_OFFSET.c_str(), po::value(), "offset in bytes") ++ (at::READ_LENGTH.c_str(), po::value(), "length in bytes") + (at::MID_SNAP_PREFIX.c_str(), po::value(), + "the prefix of snapshot name in output diff when specifying offset and length"); at::add_no_progress_option(options); @@ -159,7 +159,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644 int execute_diff(const po::variables_map &vm, const std::vector &ceph_global_init_args) { size_t arg_index = 0; -@@ -290,6 +345,26 @@ int execute_diff(const po::variables_map &vm, +@@ -290,6 +345,45 @@ int execute_diff(const po::variables_map &vm, from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as(); } @@ -170,23 +170,42 @@ index ddcf0f2c30c..2ba0be2ebb1 100644 + + uint64_t offset = 0; + if (vm.count(at::READ_OFFSET)) { -+ offset = vm[at::READ_OFFSET].as(); ++ // When passing a negative value as an argument for uint64_t typed arg using boost::program_options, ++ // it is casted without causing an error, so check logic was necessary. ++ int64_t s_offset = vm[at::READ_OFFSET].as(); ++ if (s_offset < 0) { ++ std::cerr << "rbd: offset must be greater than or equal to 0" << std::endl; ++ return -EINVAL; ++ } ++ offset = static_cast(s_offset); + } + + uint64_t length = 0; + if (vm.count(at::READ_LENGTH)) { -+ length = vm[at::READ_LENGTH].as(); ++ // When passing a negative value as an argument for uint64_t typed arg using boost::program_options, ++ // it is casted without causing an error, so check logic was necessary. ++ int64_t s_length = vm[at::READ_LENGTH].as(); ++ if (s_length < 0) { ++ std::cerr << "rbd: length must be greater than or equal to 0" << std::endl; ++ return -EINVAL; ++ } ++ length = static_cast(s_length); + } + + std::string mid_snap_prefix("mid-snap"); + if (vm.count(at::MID_SNAP_PREFIX)) { + mid_snap_prefix = vm[at::MID_SNAP_PREFIX].as(); ++ r = utils::validate_snapshot_name(at::ARGUMENT_MODIFIER_SOURCE, mid_snap_prefix, ++ utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_SNAP); ++ if (r < 0) { ++ return r; ++ } + } + librados::Rados rados; librados::IoCtx io_ctx; librbd::Image image; -@@ -299,9 +374,28 @@ int execute_diff(const po::variables_map &vm, +@@ -299,9 +393,28 @@ int execute_diff(const po::variables_map &vm, return r; } @@ -215,7 +234,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644 vm[at::WHOLE_OBJECT].as(), path.c_str(), vm[at::NO_PROGRESS].as()); if (r < 0) { -@@ -501,7 +595,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd +@@ -501,7 +614,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd const char *last_snap = NULL; for (size_t i = 0; i < snaps.size(); ++i) { utils::snap_set(image, snaps[i].name.c_str()); @@ -225,7 +244,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644 if (r < 0) { return r; } -@@ -509,7 +604,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd +@@ -509,7 +623,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd last_snap = snaps[i].name.c_str(); } utils::snap_set(image, std::string("")); diff --git a/ceph/test/cluster/block.go b/ceph/test/cluster/block.go new file mode 100644 index 00000000..41098a35 --- /dev/null +++ b/ceph/test/cluster/block.go @@ -0,0 +1,73 @@ +package cluster + +import ( + "fmt" + "log" + "os" + "os/exec" + "path" + "strings" + + "github.com/cybozu-go/mantle/test/util" +) + +func DiscardBlock(namespace, deployName string) error { + _, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--", + "blkdiscard", "/dev/rbd-device") + if err != nil { + return fmt.Errorf("failed to discard volume: %w", err) + } + return nil +} + +func WriteRandomBlock(namespace, deployName string, offset, size uint64) error { + _, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--", + "dd", "if=/dev/urandom", "of=/dev/rbd-device", "bs=1K", + fmt.Sprintf("seek=%d", offset/1024), fmt.Sprintf("count=%d", size/1024), "oflag=direct,dsync") + if err != nil { + return fmt.Errorf("failed to write random block: %w", err) + } + return nil +} + +func GetBlockAsFile(namespace, deployName, filename string) error { + const workFilename = "/tmp/work.bin" + + _, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--", + "dd", "if=/dev/rbd-device", "of="+workFilename, "bs=1K") + if err != nil { + return fmt.Errorf("failed to get block as file: %w", err) + } + + podName, err := GetPodNameByDeploy(namespace, deployName) + if err != nil { + return err + } + _, err = Kubectl("cp", namespace+"/"+podName+":"+workFilename, path.Join(workDir, filename)) + if err != nil { + return fmt.Errorf("failed to copy file to pod: %w", err) + } + + return RemoveFileByPod(namespace, deployName, workFilename) +} + +func CompareBlockWithFile(namespace, deployName, filename string) error { + workFilename := util.GetUniqueName("compare-file-") + defer func() { + _ = os.Remove(path.Join(workDir, workFilename)) + }() + + if err := GetBlockAsFile(namespace, deployName, workFilename); err != nil { + return err + } + + args := []string{path.Join(workDir, filename), path.Join(workDir, workFilename)} + log.Printf("📂 cmp %s", strings.Join(args, " ")) + _, err := exec.Command("cmp", args...).CombinedOutput() + if err != nil { + showMD5Sum(path.Join(workDir, filename)) + showMD5Sum(path.Join(workDir, workFilename)) + return fmt.Errorf("the devices having differences: %w", err) + } + return nil +} diff --git a/ceph/test/cluster/filesystem.go b/ceph/test/cluster/filesystem.go new file mode 100644 index 00000000..a6196194 --- /dev/null +++ b/ceph/test/cluster/filesystem.go @@ -0,0 +1,105 @@ +package cluster + +import ( + "fmt" + "log" + "os" + "os/exec" + "path" + "strings" + + "github.com/cybozu-go/mantle/test/util" +) + +var workDir string + +func MakeRandomFile(filename string, size int) error { + args := []string{"if=/dev/urandom", "of=" + path.Join(workDir, filename), "bs=1K", fmt.Sprintf("count=%d", size/1024)} + log.Printf("📂 dd %s", strings.Join(args, " ")) + command := exec.Command("dd", args...) + command.Stdout = os.Stdout + command.Stderr = os.Stderr + defer showMD5Sum(path.Join(workDir, filename)) + return command.Run() +} + +func PushFileToPod(filename, namespace, deployName, dst string) error { + podName, err := GetPodNameByDeploy(namespace, deployName) + if err != nil { + return err + } + _, err = Kubectl("cp", path.Join(workDir, filename), namespace+"/"+podName+":"+dst) + if err != nil { + return fmt.Errorf("failed to copy file to pod: %w", err) + } + + _, err = Kubectl("exec", "-n", namespace, podName, "--", "sync") + return err +} + +func RemoveFileByPod(namespace, deployName, target string) error { + podName, err := GetPodNameByDeploy(namespace, deployName) + if err != nil { + return err + } + _, err = Kubectl("exec", "-n", namespace, podName, "--", "rm", "-f", target) + if err != nil { + return fmt.Errorf("failed to remove file in pod: %w", err) + } + + _, err = Kubectl("exec", "-n", namespace, podName, "--", "sync") + return err +} + +// CompareFilesInPod compares the file in the host(expected) with in the pod. +func CompareFilesInPod(filename, namespace, deployName, target string) error { + workFilename := util.GetUniqueName("compare-file-") + defer func() { + _ = os.Remove(path.Join(workDir, workFilename)) + }() + + podName, err := GetPodNameByDeploy(namespace, deployName) + if err != nil { + return err + } + _, err = Kubectl("cp", namespace+"/"+podName+":"+target, path.Join(workDir, workFilename)) + if err != nil { + return err + } + + args := []string{path.Join(workDir, filename), path.Join(workDir, workFilename)} + log.Printf("📂 diff %s", strings.Join(args, " ")) + _, err = exec.Command("diff", args...).CombinedOutput() + if err != nil { + showMD5Sum(path.Join(workDir, filename)) + showMD5Sum(path.Join(workDir, workFilename)) + return fmt.Errorf("the files having differences: %w", err) + } + return nil +} + +func RemoveWorkDir() { + if err := os.RemoveAll(workDir); err != nil { + log.Fatalf("failed to remove workDir: %v", err) + } +} + +func showMD5Sum(filename string) { + args := []string{filename} + log.Printf("📂 md5sum %s", strings.Join(args, " ")) + command := exec.Command("md5sum", args...) + command.Stdout = os.Stdout + command.Stderr = os.Stderr + err := command.Run() + if err != nil { + log.Fatalf("failed to run md5sum: %v", err) + } +} + +func init() { + dir, err := os.MkdirTemp("", "test-mantle-ceph-") + if err != nil { + log.Fatalf("failed to create workDir: %v", err) + } + workDir = dir +} diff --git a/ceph/test/cluster/kubectl.go b/ceph/test/cluster/kubectl.go new file mode 100644 index 00000000..cad1658c --- /dev/null +++ b/ceph/test/cluster/kubectl.go @@ -0,0 +1,292 @@ +package cluster + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "regexp" + "strings" + "time" + + storagev1 "k8s.io/api/storage/v1" +) + +const ( + ROOK_NAMESPACE = "rook-ceph" + RBD_PROVISIONER = "rook-ceph.rbd.csi.ceph.com" +) + +var ( + //go:embed template/deployment_filesystem.yaml + deploymentFilesystemTemplate string + //go:embed template/deployment_block.yaml + deploymentBlockTemplate string + //go:embed template/pool.yaml + poolTemplate string + //go:embed template/pvc.yaml + pvcTemplate string + //go:embed template/sc.yaml + scTemplate string +) + +var kubectlCmd = os.Getenv("KUBECTL") +var cephMatcher = regexp.MustCompile(` (ceph|rbd) `) + +func Kubectl(args ...string) ([]byte, error) { + return KubectlWithInput(nil, args...) +} + +func KubectlWithInput(stdin []byte, args ...string) ([]byte, error) { + if len(kubectlCmd) == 0 { + return nil, fmt.Errorf("KUBECTL environment variable should be set") + } + + icon := "⚓" + if cephMatcher.MatchString(strings.Join(args, " ")) { + icon = "🐙" + } + log.Printf("%s kubectl %s", icon, strings.Join(args, " ")) + var stdout bytes.Buffer + command := exec.Command(kubectlCmd, args...) + if stdin != nil { + command.Stdin = bytes.NewReader(stdin) + } + command.Stdout = &stdout + command.Stderr = os.Stderr + + err := command.Run() + return stdout.Bytes(), err +} + +func GetObjectList[T any](kind, namespace string) (*T, error) { + var stdout []byte + var err error + if namespace == "" { + stdout, err = Kubectl("get", kind, "-o", "json") + } else { + stdout, err = Kubectl("get", kind, "-n", namespace, "-o", "json") + } + if err != nil { + return nil, fmt.Errorf("failed to get %s list: %w", kind, err) + } + + var objList T + if err := json.Unmarshal(stdout, &objList); err != nil { + return nil, fmt.Errorf("failed to unmarshal %s list: %w", kind, err) + } + + return &objList, nil +} + +func DeleteObject(kind, namespace, name string) error { + var err error + if len(namespace) == 0 { + _, err = Kubectl("delete", kind, name) + } else { + _, err = Kubectl("delete", kind, "-n", namespace, name) + } + if err != nil { + return fmt.Errorf("failed to delete %s %s/%s: %w", kind, namespace, name, err) + } + return nil +} + +func DeleteAllObjects(kind, namespace string) error { + _, err := Kubectl("delete", kind, "-n", namespace, "--all", "--wait=false") + if err != nil { + return fmt.Errorf("failed to delete %s in %s: %w", kind, namespace, err) + } + return nil +} + +func CreateNamespace(namespace string) error { + _, err := Kubectl("create", "namespace", namespace) + if err != nil { + return fmt.Errorf("failed to create namespace: %w", err) + } + return nil +} + +type VolumeMode string + +const ( + VolumeModeFilesystem VolumeMode = "Filesystem" + VolumeModeBlock VolumeMode = "Block" +) + +func CreateDeployment(namespace, deployName, pvcName string, volumeMode VolumeMode) error { + template := deploymentFilesystemTemplate + if volumeMode == VolumeModeBlock { + template = deploymentBlockTemplate + } + manifest := fmt.Sprintf(template, namespace, deployName, deployName, deployName, pvcName) + _, err := KubectlWithInput([]byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + _, err = Kubectl("wait", "--for=condition=available", "-n", namespace, "deploy/"+deployName, "--timeout=3m") + if err != nil { + return fmt.Errorf("failed to wait for deployment: %w", err) + } + return nil +} + +func CreatePool(poolName string) error { + manifest := fmt.Sprintf(poolTemplate, poolName) + _, err := KubectlWithInput([]byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create pool: %w", err) + } + return nil +} + +func CreatePVC(namespace, pvcName, scName, size string, volumeMode VolumeMode) error { + manifest := fmt.Sprintf(pvcTemplate, namespace, pvcName, volumeMode, size, scName) + _, err := KubectlWithInput([]byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + return nil +} + +func ResizePVC(namespace, pvcName, size string) error { + before, err := Kubectl("get", "-n", namespace, "pvc", pvcName, "-o", "jsonpath={.status.capacity.storage}") + if err != nil { + return fmt.Errorf("failed to get PVC size: %w", err) + } + + _, err = Kubectl("patch", "-n", namespace, "pvc", pvcName, + "-p", fmt.Sprintf(`{"spec":{"resources":{"requests":{"storage":"%s"}}}}`, size)) + if err != nil { + return fmt.Errorf("failed to patch PVC: %w", err) + } + + for i := 0; i < 30; i++ { + after, err := Kubectl("get", "-n", namespace, "pvc", pvcName, "-o", "jsonpath={.status.capacity.storage}") + if err != nil { + return fmt.Errorf("failed to get PVC size: %w", err) + } + if string(after) != string(before) { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("PVC size is not changed") +} + +func CreateSC(scName, poolName string) error { + manifest := fmt.Sprintf(scTemplate, scName, poolName) + _, err := KubectlWithInput([]byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create SC: %w", err) + } + return nil +} + +func CleanupNamespace(namespace string) error { + err := DeleteAllObjects("pod", namespace) + if err != nil { + return err + } + + err = DeleteAllObjects("pvc", namespace) + if err != nil { + return err + } + + _, err = Kubectl("delete", "namespace", namespace, "--wait=false") + if err != nil { + return fmt.Errorf("failed to delete namespace: %w", err) + } + return nil +} + +func CleanupGlobal() error { + scs, err := GetObjectList[storagev1.StorageClassList]("sc", "") + if err != nil { + return err + } + for _, sc := range scs.Items { + if sc.Provisioner == RBD_PROVISIONER { + _, err = Kubectl("delete", "sc", sc.Name) + if err != nil { + return fmt.Errorf("failed to delete SC: %w", err) + } + } + } + + err = DeleteAllObjects("cephblockpool", ROOK_NAMESPACE) + if err != nil { + return err + } + + return nil +} + +func GetImageNameByPVC(namespace, pvcName string) (string, error) { + _, err := Kubectl("wait", "--for=jsonpath={.status.phase}=Bound", "-n", namespace, "pvc", pvcName, "--timeout=3m") + if err != nil { + return "", fmt.Errorf("failed to wait for pvc bound: %w", err) + } + + stdout, err := Kubectl("get", "-n", namespace, "pvc", pvcName, "-o", "jsonpath={.spec.volumeName}") + if err != nil { + return "", fmt.Errorf("failed to get volume name by PVC: %w", err) + } + volumeName := string(stdout) + + stdout, err = Kubectl("get", "pv", volumeName, "-o", "jsonpath={.spec.csi.volumeAttributes.imageName}") + if err != nil { + return "", fmt.Errorf("failed to get image name by volume: %w", err) + } + + return string(stdout), nil +} + +func ScaleDeployment(namespace, deployName string, replicas int) error { + _, err := Kubectl("scale", "deploy", "-n", namespace, deployName, fmt.Sprintf("--replicas=%d", replicas)) + if err != nil { + return fmt.Errorf("failed to scale deployment: %w", err) + } + if replicas == 0 { + _, err = Kubectl("wait", "--for=delete", "pod", "-n", namespace, "--selector=app="+deployName, "--timeout=3m") + if err != nil { + return fmt.Errorf("failed to wait for pod deletion: %w", err) + } + } else { + _, err = Kubectl("wait", "--for=condition=available", "deploy", "-n", namespace, deployName, "--timeout=3m") + if err != nil { + return fmt.Errorf("failed to wait for deployment available: %w", err) + } + } + return nil +} + +func GetPodNameByDeploy(namespace, deployName string) (string, error) { + stdout, err := Kubectl("get", "pod", "-n", namespace, "-l", "app="+deployName, + "-o", "jsonpath={.items[0].metadata.name}") + if err != nil { + return "", err + } + return string(stdout), nil +} + +// RunWithStopPod runs the function with stopping the pod. +// If deployName is empty, it just runs the function. +func RunWithStopPod(namespace, deployName string, f func() error) error { + if len(deployName) == 0 { + return f() + } + + if err := ScaleDeployment(namespace, deployName, 0); err != nil { + return err + } + if err := f(); err != nil { + return err + } + return ScaleDeployment(namespace, deployName, 1) +} diff --git a/ceph/test/cluster/rbd.go b/ceph/test/cluster/rbd.go new file mode 100644 index 00000000..19089907 --- /dev/null +++ b/ceph/test/cluster/rbd.go @@ -0,0 +1,244 @@ +package cluster + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/cybozu-go/mantle/test/util" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Rbd(args ...string) ([]byte, error) { + return Kubectl(append([]string{"exec", "-n", ROOK_NAMESPACE, "deploy/rook-ceph-tools", "--", "rbd"}, args...)...) +} + +func ExportDiff(filename string, args ...string) error { + _, err := Kubectl("exec", "-n", ROOK_NAMESPACE, "deploy/rook-ceph-tools", "--", "rm", "-f", filename) + if err != nil { + return err + } + args = append([]string{"export-diff"}, args...) + args = append(args, filename) + stdout, err := Rbd(args...) + if err != nil { + return fmt.Errorf("failed to run rbd export-diff command: %w, %s", err, string(stdout)) + } + return nil +} + +func ImportDiff(filename, pool, image, rollbackTo, namespace, deployName, pvcName string) error { + return RunWithStopPod(namespace, deployName, func() error { + if rollbackTo == "" { + if len(pvcName) == 0 { + return fmt.Errorf("rollbackTo or pvcName must be specified") + } + err := discardVolume(namespace, pvcName) + if err != nil { + return fmt.Errorf("failed to discard volume: %w", err) + } + } else { + stdout, err := Rbd("snap", "rollback", pool+"/"+image+"@"+rollbackTo) + if err != nil { + return fmt.Errorf("failed to rollback snapshot: %w, %s", err, string(stdout)) + } + } + + stdout, err := Kubectl("exec", "-n", ROOK_NAMESPACE, "deploy/rook-ceph-tools", "--", + "sh", "-c", fmt.Sprintf("cat %s | rbd import-diff -p %s - %s", filename, pool, image)) + if err != nil { + return fmt.Errorf("failed to import diff: %w, %s", err, string(stdout)) + } + return nil + }) +} + +var mtxDiscardVolume = &sync.Mutex{} + +func discardVolume(namespace, pvcName string) error { + mtxDiscardVolume.Lock() + defer mtxDiscardVolume.Unlock() + + origPVCRaw, err := Kubectl("get", "-n", namespace, "pvc", pvcName, "-o", "json") + if err != nil { + return fmt.Errorf("failed to get PVC: %w", err) + } + var origPVC corev1.PersistentVolumeClaim + if err := json.Unmarshal(origPVCRaw, &origPVC); err != nil { + return fmt.Errorf("failed to unmarshal PVC: %w", err) + } + + origPVRaw, err := Kubectl("get", "pv", origPVC.Spec.VolumeName, "-o", "json") + if err != nil { + return fmt.Errorf("failed to get PV: %w", err) + } + var origPV corev1.PersistentVolume + if err := json.Unmarshal(origPVRaw, &origPV); err != nil { + return fmt.Errorf("failed to unmarshal PV: %w", err) + } + + discardPVName := util.GetUniqueName("discard-pv-") + discardPVCName := util.GetUniqueName("discard-pvc-") + discardDeployName := util.GetUniqueName("discard-pod-") + + volumeMode := corev1.PersistentVolumeBlock + discardPV := corev1.PersistentVolume{ + TypeMeta: origPV.TypeMeta, + ObjectMeta: v1.ObjectMeta{ + Name: discardPVName, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Capacity: origPV.Spec.Capacity, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + VolumeMode: &volumeMode, + StorageClassName: "", + }, + } + discardPV.Spec.CSI = &corev1.CSIPersistentVolumeSource{ + Driver: origPV.Spec.CSI.Driver, + ControllerExpandSecretRef: origPV.Spec.CSI.ControllerExpandSecretRef, + NodeStageSecretRef: origPV.Spec.CSI.NodeStageSecretRef, + VolumeAttributes: map[string]string{ + "clusterID": origPV.Spec.CSI.VolumeAttributes["clusterID"], + "imageFeatures": origPV.Spec.CSI.VolumeAttributes["imageFeatures"], + "imageFormat": origPV.Spec.CSI.VolumeAttributes["imageFormat"], + "pool": origPV.Spec.CSI.VolumeAttributes["pool"], + "staticVolume": "true", + }, + VolumeHandle: origPV.Spec.CSI.VolumeAttributes["imageName"], + } + discardPVRaw, err := json.Marshal(discardPV) + if err != nil { + return fmt.Errorf("failed to marshal PV: %w", err) + } + _, err = KubectlWithInput(discardPVRaw, "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create PV: %w", err) + } + + sc := "" + discardPVC := corev1.PersistentVolumeClaim{ + TypeMeta: origPVC.TypeMeta, + ObjectMeta: v1.ObjectMeta{ + Name: discardPVCName, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &sc, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: origPVC.Spec.Resources, + VolumeMode: &volumeMode, + VolumeName: discardPVName, + }, + } + discardPVCRaw, err := json.Marshal(discardPVC) + if err != nil { + return fmt.Errorf("failed to marshal PVC: %w", err) + } + _, err = KubectlWithInput(discardPVCRaw, "apply", "-f", "-") + if err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + + err = CreateDeployment(namespace, discardDeployName, discardPVCName, VolumeModeBlock) + if err != nil { + return fmt.Errorf("failed to create pod: %w", err) + } + + if err := DiscardBlock(namespace, discardDeployName); err != nil { + return err + } + + err = DeleteObject("deployment", namespace, discardDeployName) + if err != nil { + return fmt.Errorf("failed to delete pod: %w", err) + } + + err = DeleteObject("pvc", namespace, discardPVCName) + if err != nil { + return fmt.Errorf("failed to delete PVC: %w", err) + } + + err = DeleteObject("pv", "", discardPVName) + if err != nil { + return fmt.Errorf("failed to delete PV: %w", err) + } + + return nil +} + +func SnapCreate(pool, image, snap string) error { + stdout, err := Rbd("snap", "create", pool+"/"+image+"@"+snap) + if err != nil { + return fmt.Errorf("failed to create snapshot: %w, %s", err, string(stdout)) + } + return nil +} + +func SnapRemove(pool, image string, snaps []string) error { + for _, snap := range snaps { + stdout, err := Rbd("snap", "rm", pool+"/"+image+"@"+snap) + if err != nil { + return fmt.Errorf("failed to remove snapshot: %w, %s", err, string(stdout)) + } + } + return nil +} + +func SnapRemoveAll(pool, image string) error { + stdout, err := Rbd("snap", "ls", pool+"/"+image, "--format", "json") + if err != nil { + return fmt.Errorf("failed to list snapshots: %w, %s", err, string(stdout)) + } + + var snaps []SnapLsEntry + if err := json.Unmarshal(stdout, &snaps); err != nil { + return fmt.Errorf("failed to unmarshal snapshots: %w", err) + } + + snapNames := make([]string, 0, len(snaps)) + for _, s := range snaps { + snapNames = append(snapNames, s.Name) + } + + return SnapRemove(pool, image, snapNames) +} + +func SnapRollback(pool, image, snap, namespace, deployName string) error { + return RunWithStopPod(namespace, deployName, func() error { + stdout, err := Rbd("snap", "rollback", pool+"/"+image+"@"+snap) + if err != nil { + return fmt.Errorf("failed to rollback snapshot: %w, %s", err, string(stdout)) + } + return nil + }) +} + +type SnapLsEntry struct { + Name string `json:"name"` +} + +func SnapExists(pool, image, snap string) (bool, error) { + stdout, err := Rbd("snap", "ls", pool+"/"+image, "--format", "json") + if err != nil { + return false, fmt.Errorf("failed to list snapshots: %w, %s", err, string(stdout)) + } + + var snaps []SnapLsEntry + if err := json.Unmarshal(stdout, &snaps); err != nil { + return false, fmt.Errorf("failed to unmarshal snapshots: %w", err) + } + + for _, s := range snaps { + if s.Name == snap { + return true, nil + } + } + return false, nil +} diff --git a/ceph/test/cluster/template/deployment_block.yaml b/ceph/test/cluster/template/deployment_block.yaml new file mode 100644 index 00000000..2b22ac73 --- /dev/null +++ b/ceph/test/cluster/template/deployment_block.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: "%s" + name: "%s" +spec: + replicas: 1 + selector: + matchLabels: + app: "%s" + template: + metadata: + labels: + app: "%s" + spec: + containers: + - name: ubuntu + image: ghcr.io/cybozu/ubuntu:22.04 + command: ["pause"] + volumeDevices: + - name: rbd-device + devicePath: /dev/rbd-device + volumes: + - name: rbd-device + persistentVolumeClaim: + claimName: "%s" diff --git a/ceph/test/cluster/template/deployment_filesystem.yaml b/ceph/test/cluster/template/deployment_filesystem.yaml new file mode 100644 index 00000000..5563fafd --- /dev/null +++ b/ceph/test/cluster/template/deployment_filesystem.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: "%s" + name: "%s" +spec: + replicas: 1 + selector: + matchLabels: + app: "%s" + template: + metadata: + labels: + app: "%s" + spec: + containers: + - name: ubuntu + image: ghcr.io/cybozu/ubuntu:22.04 + command: ["pause"] + volumeMounts: + - name: rbd-volume + mountPath: /mnt + volumes: + - name: rbd-volume + persistentVolumeClaim: + claimName: "%s" diff --git a/ceph/test/cluster/template/pool.yaml b/ceph/test/cluster/template/pool.yaml new file mode 100644 index 00000000..911e2720 --- /dev/null +++ b/ceph/test/cluster/template/pool.yaml @@ -0,0 +1,10 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: "%s" + namespace: rook-ceph +spec: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false diff --git a/ceph/test/cluster/template/pvc.yaml b/ceph/test/cluster/template/pvc.yaml new file mode 100644 index 00000000..642726ce --- /dev/null +++ b/ceph/test/cluster/template/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: "%s" + name: "%s" +spec: + accessModes: + - ReadWriteOnce + volumeMode: "%s" + resources: + requests: + storage: "%s" + storageClassName: "%s" diff --git a/ceph/test/cluster/template/sc.yaml b/ceph/test/cluster/template/sc.yaml new file mode 100644 index 00000000..50900749 --- /dev/null +++ b/ceph/test/cluster/template/sc.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "%s" +provisioner: rook-ceph.rbd.csi.ceph.com +parameters: + clusterID: rook-ceph + pool: "%s" + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 +allowVolumeExpansion: true +reclaimPolicy: Delete diff --git a/ceph/test/content_test.go b/ceph/test/content_test.go new file mode 100644 index 00000000..39b626a9 --- /dev/null +++ b/ceph/test/content_test.go @@ -0,0 +1,452 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type contentTest struct { + namespace string + poolName string + scName string + + srcDeployName string + srcPVCName string + srcImageName string // will be set in setupEnv after creating PVC + + dstDeployName string + dstPVCName string + dstImageName string // will be set in setupEnv after creating PVC + + snapshots []string +} + +func testContent() { + test := &contentTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + + srcDeployName: util.GetUniqueName("pod-"), + srcPVCName: util.GetUniqueName("src-pvc-"), + dstDeployName: util.GetUniqueName("pod-"), + dstPVCName: util.GetUniqueName("dst-pvc-"), + + snapshots: []string{ + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + }, + } + + Describe("setup environment", test.setupEnv) + Describe("create snapshot", test.createSnapshot) + Describe("test main", test.test) + Describe("teardown environment", test.teardownEnv) +} + +//nolint:dupl +func (t *contentTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.srcPVCName, t.scName, "3Mi", cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.srcDeployName, t.srcPVCName, cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.srcPVCName) + Expect(err).NotTo(HaveOccurred()) + t.srcImageName = imageName + + err = cluster.CreatePVC(t.namespace, t.dstPVCName, t.scName, "3Mi", cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.dstDeployName, t.dstPVCName, cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + imageName, err = cluster.GetImageNameByPVC(t.namespace, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + t.dstImageName = imageName + }) +} + +func (t *contentTest) createSnapshot() { + It("create snapshot", func() { + // snapshot0 has 0.5MiB data + err := cluster.WriteRandomBlock(t.namespace, t.srcDeployName, 0, Quantity2Int("512Ki")) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[0]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[0]) + Expect(err).NotTo(HaveOccurred()) + + // snapshot1 has the same data as snapshot0 + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[1]) + Expect(err).NotTo(HaveOccurred()) + + // snapshot2 has additional 1.0MiB data + err = cluster.WriteRandomBlock(t.namespace, t.srcDeployName, Quantity2Int("512Ki"), Quantity2Int("1Mi")) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[2]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[2]) + Expect(err).NotTo(HaveOccurred()) + + // snapshot3 has different data from snapshot2 + err = cluster.WriteRandomBlock(t.namespace, t.srcDeployName, 0, Quantity2Int("512Ki")+Quantity2Int("1Mi")) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[3]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[3]) + Expect(err).NotTo(HaveOccurred()) + + // snapshot4 has discard volume + err = cluster.DiscardBlock(t.namespace, t.srcDeployName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[4]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[4]) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *contentTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *contentTest) test() { + It("checks that the data exported with the offset and length options is correct, "+ + " by import, rollback, and comparing it with the original data.", func() { + err := cluster.ExportDiff("/tmp/snapshot0-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot0-offset-1Mi.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot0.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Mi.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot2-offset-1Mi.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[1], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot2.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot3-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot3", + "--from-snap", t.snapshots[2], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[3]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot3.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[3]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot4-offset-1Mi.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot4", + "--from-snap", t.snapshots[3], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[4]), + ) + Expect(err).NotTo(HaveOccurred()) + + rollbackMap := map[string]string{ + "/tmp/snapshot0-offset-1Ki.bin": "", + "/tmp/snapshot0-offset-1Mi.bin": "", + "/tmp/snapshot0.bin": "", + "/tmp/snapshot1-offset-1Ki.bin": t.snapshots[0], + "/tmp/snapshot1-offset-1Mi.bin": t.snapshots[0], + "/tmp/snapshot1.bin": "", + "/tmp/snapshot2-offset-1Mi.bin": t.snapshots[1], + "/tmp/snapshot2.bin": "", + "/tmp/snapshot3-offset-1Ki.bin": t.snapshots[2], + "/tmp/snapshot3.bin": "", + "/tmp/snapshot4-offset-1Mi.bin": t.snapshots[3], + } + + tests := []struct { + description string + expectedDataName string + importsBefore []string + exportArgs []string + rollbackTo string + }{ + { + description: "(216) having data difference between volume and snapshot, " + + "without source snapshot, offset + length == rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("3Mi")-1024), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(217) having data difference between volume and snapshot, " + + "without source snapshot, offset + length > rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(218) export for hole area, without source snapshot, offset + length == rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("2Mi"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1048576", + }, + { + description: "(219) export for hole area, without source snapshot, offset + length < rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1048576", + }, + { + description: "(220) having the same data between snapshots, offset + length == rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("3Mi")-1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "(221) having the same data between snapshots, offset + length > rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "(222) having the same hole area between snapshots, offset + length == rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("2Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1048576", + }, + { + description: "(223) having the same hole area between snapshots, offset + length > rbd volume size", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1048576", + }, + { + description: "(224) add data for the all hole area, offset + length == rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot1.bin", "/tmp/snapshot2-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("2Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[1], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo: "snapshot2-offset-1048576", + }, + { + description: "(225) add data for the all hole area, offset + length > rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot1.bin", "/tmp/snapshot2-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[1], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo: "snapshot2-offset-1048576", + }, + { + description: "(226) expand data area between snapshots, offset + length == rbd volume size", + expectedDataName: t.snapshots[3], + importsBefore: []string{"/tmp/snapshot2.bin", "/tmp/snapshot3-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("3Mi")-1024), + "--mid-snap-prefix", "snapshot3", + "--from-snap", t.snapshots[2], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[3]), + }, + rollbackTo: "snapshot3-offset-1024", + }, + { + description: "(227) expand data area between snapshots, offset + length > rbd volume size", + expectedDataName: t.snapshots[3], + importsBefore: []string{"/tmp/snapshot2.bin", "/tmp/snapshot3-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot3", + "--from-snap", t.snapshots[2], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[3]), + }, + rollbackTo: "snapshot3-offset-1024", + }, + { + description: "(228) discard data area between snapshots, offset + length == rbd volume size", + expectedDataName: t.snapshots[4], + importsBefore: []string{"/tmp/snapshot3.bin", "/tmp/snapshot4-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("2Mi"), + "--mid-snap-prefix", "snapshot4", + "--from-snap", t.snapshots[3], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[4]), + }, + rollbackTo: "snapshot4-offset-1048576", + }, + { + description: "(229) discard data area between snapshots, offset + length > rbd volume size", + expectedDataName: t.snapshots[4], + importsBefore: []string{"/tmp/snapshot3.bin", "/tmp/snapshot4-offset-1Mi.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot4", + "--from-snap", t.snapshots[3], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[4]), + }, + rollbackTo: "snapshot4-offset-1048576", + }, + } + + for _, tt := range tests { + By(tt.description) + + if len(tt.importsBefore) != 0 { + for _, file := range tt.importsBefore { + err = cluster.ImportDiff(file, t.poolName, t.dstImageName, + rollbackMap[file], t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + } + } + + // export from source snapshot to file + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).NotTo(HaveOccurred()) + + // rollback & import process + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo, t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CompareBlockWithFile(t.namespace, t.dstDeployName, tt.expectedDataName) + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + } + }) +} diff --git a/ceph/test/extend_test.go b/ceph/test/extend_test.go new file mode 100644 index 00000000..49c3055a --- /dev/null +++ b/ceph/test/extend_test.go @@ -0,0 +1,427 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type extendTest struct { + namespace string + poolName string + scName string + + srcDeployName string + srcPVCName string + srcImageName string // will be set in setupEnv after creating PVC + + dstDeployName string + dstPVCName string + dstImageName string // will be set in setupEnv after creating PVC + + snapshots []string +} + +func testExtend() { + test := &extendTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + + srcDeployName: util.GetUniqueName("pod-"), + srcPVCName: util.GetUniqueName("src-pvc-"), + dstDeployName: util.GetUniqueName("pod-"), + dstPVCName: util.GetUniqueName("dst-pvc-"), + + snapshots: []string{ + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + }, + } + + Describe("setup environment", test.setupEnv) + Describe("create snapshot", test.createSnapshot) + Describe("test main", test.test) + Describe("teardown environment", test.teardownEnv) +} + +//nolint:dupl +func (t *extendTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.srcPVCName, t.scName, "1Mi", cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.srcDeployName, t.srcPVCName, cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.srcPVCName) + Expect(err).NotTo(HaveOccurred()) + t.srcImageName = imageName + + err = cluster.CreatePVC(t.namespace, t.dstPVCName, t.scName, "1Mi", cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.dstDeployName, t.dstPVCName, cluster.VolumeModeBlock) + Expect(err).NotTo(HaveOccurred()) + imageName, err = cluster.GetImageNameByPVC(t.namespace, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + t.dstImageName = imageName + }) +} + +func (t *extendTest) createSnapshot() { + It("create snapshot", func() { + // snapshot0 has 0.5MiB data + err := cluster.WriteRandomBlock(t.namespace, t.srcDeployName, 0, Quantity2Int("512Ki")) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[0]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[0]) + Expect(err).NotTo(HaveOccurred()) + + // extend volume to 2.0MiB and make snapshot1 + err = cluster.ResizePVC(t.namespace, t.srcPVCName, "2Mi") + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[1]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[1]) + Expect(err).NotTo(HaveOccurred()) + + // snapshot2 has additional 1.0MiB data + err = cluster.WriteRandomBlock(t.namespace, t.srcDeployName, Quantity2Int("512Ki"), Quantity2Int("1Mi")) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[2]) + Expect(err).NotTo(HaveOccurred()) + err = cluster.GetBlockAsFile(t.namespace, t.srcDeployName, t.snapshots[2]) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *extendTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *extendTest) test() { + It("checks the consistency of export/import data when volume size (PV/PVC) is expanded", func() { + err := cluster.ExportDiff("/tmp/snapshot0.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Mi+1Ki.bin", + "--read-offset", "0", + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot2-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot2-offset-1Mi+1Ki.bin", + "--read-offset", "0", + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + ) + Expect(err).NotTo(HaveOccurred()) + + rollbackMap := map[string]string{ + "/tmp/snapshot0.bin": "", + "/tmp/snapshot1-offset-1Ki.bin": t.snapshots[0], + "/tmp/snapshot1-offset-1Mi+1Ki.bin": t.snapshots[0], + "/tmp/snapshot2-offset-1Ki.bin": t.snapshots[0], + "/tmp/snapshot2-offset-1Mi+1Ki.bin": t.snapshots[0], + } + + tests := []struct { + description string + expectedDataName string + importsBefore []string + exportArgs1 []string + rollbackTo1 string + exportArgs2 []string + rollbackTo2 string + }{ + { + description: "(230) expand volume but data is not written, " + + "the snapshot area contains data, offset + length < rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1024", + exportArgs2: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo2: "snapshot1-offset-1048576", + }, + { + description: "(231) expand volume but data is not written, " + + "the snapshot area contains data, offset + length == rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("2Mi")-1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1024", + }, + { + description: "(232) expand volume but data is not written, " + + "the snapshot area contains data, offset + length > rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1024", + }, + + { + description: "(233) expand volume but data is not written, " + + "the snapshot area does not contain data, offset + length < rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1049600", + exportArgs2: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+2048), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-2048), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo2: "snapshot1-offset-1050624", + }, + { + description: "(234) expand volume but data is not written, " + + "the snapshot area does not contain data, offset + length == rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1049600", + }, + { + description: "(235) expand volume but data is not written, " + + "the snapshot area does not contain data, offset + length > rbd volume size", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo1: "snapshot1-offset-1049600", + }, + + { + description: "(236) expand volume and data is written, " + + "the snapshot area contains data, offset + length < rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-1024), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1024", + exportArgs2: []string{ + "--read-offset", Quantity2Str("1Mi"), + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo2: "snapshot2-offset-1048576", + }, + { + description: "(237) expand volume and data is written, " + + "the snapshot area contains data, offset + length == rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("2Mi")-1024), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1024", + }, + { + description: "(238) expand volume and data is written, " + + "the snapshot area contains data, offset + length > rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1024", + }, + + { + description: "(239) expand volume and data is written, " + + "the snapshot area does not contain data before expand, offset + length < rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1049600", + exportArgs2: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+2048), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-2048), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo2: "snapshot2-offset-1050624", + }, + { + description: "(240) expand volume and data is written, " + + "the snapshot area does not contain data before expand, offset + length == rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", fmt.Sprintf("%d", Quantity2Int("1Mi")-1024), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1049600", + }, + { + description: "(241) expand volume and data is written, " + + "the snapshot area does not contain data before expand, offset + length > rbd volume size", + expectedDataName: t.snapshots[2], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot2-offset-1Mi+1Ki.bin"}, + exportArgs1: []string{ + "--read-offset", fmt.Sprintf("%d", Quantity2Int("1Mi")+1024), + "--read-length", Quantity2Str("5Mi"), + "--mid-snap-prefix", "snapshot2", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + rollbackTo1: "snapshot2-offset-1049600", + }, + } + + for _, tt := range tests { + By(tt.description) + + if len(tt.importsBefore) != 0 { + for _, file := range tt.importsBefore { + err = cluster.ImportDiff(file, t.poolName, t.dstImageName, + rollbackMap[file], t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + } + } + + // export from source snapshot to file + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs1...) + Expect(err).NotTo(HaveOccurred()) + + // rollback & import process + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo1, t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + // export and import until the tail of the volume if test target is in the middle position data + if len(tt.exportArgs2) != 0 { + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs2...) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo2, t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + } + + err = cluster.CompareBlockWithFile(t.namespace, t.dstDeployName, tt.expectedDataName) + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + } + }) +} diff --git a/ceph/test/options_test.go b/ceph/test/options_test.go new file mode 100644 index 00000000..c78fbfd9 --- /dev/null +++ b/ceph/test/options_test.go @@ -0,0 +1,153 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type optionsTest struct { + namespace string + poolName string + scName string + deployName string + pvcName string + snapName string + + // image name will be set in setupEnv after creating PVC + imageName string +} + +func testOptions() { + test := &optionsTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + deployName: util.GetUniqueName("pod-"), + pvcName: util.GetUniqueName("pvc-"), + snapName: util.GetUniqueName("snap-"), + } + + Describe("setup environment", test.setupEnv) + Describe("test main", test.test) + Describe("teardown environment", test.teardownEnv) +} + +func (t *optionsTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.pvcName, t.scName, "1Gi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.deployName, t.pvcName, cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.pvcName) + Expect(err).NotTo(HaveOccurred()) + t.imageName = imageName + + err = cluster.SnapCreate(t.poolName, imageName, t.snapName) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *optionsTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.imageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *optionsTest) test() { + It("test export-diff command will be fail", func() { + tests := []struct { + description string + exportArgs []string + }{ + { + description: "(179) specify --read-offset without --read-length", + exportArgs: []string{ + "--read-offset", "0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + description: "(180) specify --read-length without --read-offset", + exportArgs: []string{ + "--read-length", Quantity2Str("1Ki"), + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + + description: "(181) not specify snapshot name", + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "-p", t.poolName, + }, + }, + { + description: "(242) specify negative value for --read-offset", + exportArgs: []string{ + "--read-offset", "-1", + "--read-length", Quantity2Str("1Ki"), + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + description: "(243) specify not number value for --read-offset", + exportArgs: []string{ + "--read-offset", "a", + "--read-length", Quantity2Str("1Ki"), + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + description: "(244) specify negative value for --read-length", + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "-1", + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + description: "(245) specify not number value for --read-length", + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "a", + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + { + description: "(246) specify invalid mid-snap-prefix", + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "@invalid", + fmt.Sprintf("%s/%s@%s", t.poolName, t.imageName, t.snapName), + }, + }, + } + + for _, tt := range tests { + By(tt.description) + + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).To(HaveOccurred()) + } + }) +} diff --git a/ceph/test/regression_test.go b/ceph/test/regression_test.go new file mode 100644 index 00000000..d68083ca --- /dev/null +++ b/ceph/test/regression_test.go @@ -0,0 +1,315 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type regressionTest struct { + namespace string + poolName string + scName string + + srcDeployName string + srcPVCName string + srcImageName string // will be set in setupEnv after creating PVC + + dstDeployName string + dstPVCName string + dstImageName string // will be set in setupEnv after creating PVC + + snapshots []string +} + +func testRegression() { + test := ®ressionTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + + srcDeployName: util.GetUniqueName("pod-"), + srcPVCName: util.GetUniqueName("pvc-"), + + dstDeployName: util.GetUniqueName("pod-"), + dstPVCName: util.GetUniqueName("pvc-"), + + snapshots: []string{ + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + }, + } + + Describe("setup environment", test.setupEnv) + Describe("test export-diff for snapshot without from-snap option", test.testWithoutFromSnap) + Describe("test export-diff for snapshot with from-snap option", test.testWithFromSnap) + Describe("test export-diff for RBD image", test.testForRBDImage) + Describe("teardown environment", test.teardownEnv) +} + +func (t *regressionTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.srcPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.srcDeployName, t.srcPVCName, cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.srcPVCName) + Expect(err).NotTo(HaveOccurred()) + t.srcImageName = imageName + + // creating snapshots + // snapshots[0] and snapshots[1] have diff with the image + // snapshots[2] and snapshots[3] has no diff with the image + for i := 0; i < 3; i++ { + err := cluster.MakeRandomFile(t.snapshots[i], int(Quantity2Int("5Mi"))) + Expect(err).NotTo(HaveOccurred()) + err = cluster.PushFileToPod(t.snapshots[i], t.namespace, t.srcDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[i]) + Expect(err).NotTo(HaveOccurred()) + } + // crate snapshot[3] with the same data as snapshot[2] + // random file named snapshot[3] does not exist + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[3]) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.dstPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.dstDeployName, t.dstPVCName, cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err = cluster.GetImageNameByPVC(t.namespace, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + t.dstImageName = imageName + }) +} + +func (t *regressionTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *regressionTest) testWithoutFromSnap() { + It("export-diff for snapshot without from-snap option", func() { + tests := []struct { + description string + expectedDataName string + exportArgs []string + }{ + { + description: "(182) specify snapshot name with /@ format", + expectedDataName: t.snapshots[0], + exportArgs: []string{ + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "specify snapshot name with @ format", + expectedDataName: t.snapshots[0], + exportArgs: []string{ + "-p", t.poolName, + fmt.Sprintf("%s@%s", t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(183) specify snapshot name with --snap option", + expectedDataName: t.snapshots[0], + exportArgs: []string{ + "-p", t.poolName, + "--image", t.srcImageName, + "--snap", t.snapshots[0], + }, + }, + { + description: "(185) specify snapshot which don't have diff with RBD image", + expectedDataName: t.snapshots[2], + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[2]), + }, + }, + } + + for _, tt := range tests { + By(tt.description) + // export from source snapshot to file + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).NotTo(HaveOccurred()) + + // import to destination image + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + "", t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + // apply snapshot to the destination image + err = cluster.SnapRollback(t.poolName, t.dstImageName, tt.expectedDataName, t.namespace, t.dstDeployName) + Expect(err).NotTo(HaveOccurred()) + + // compare the data in the destination image with the expected data + err = cluster.CompareFilesInPod(tt.expectedDataName, t.namespace, t.dstDeployName, "/mnt/data") + // expect no difference + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemove(t.poolName, t.dstImageName, []string{tt.expectedDataName}) + Expect(err).NotTo(HaveOccurred()) + } + }) +} + +func (t *regressionTest) testWithFromSnap() { + It("export-diff for snapshot with from-snap option", func() { + // export snapshot[0] and import it to the destination image before running the tests + err := cluster.ExportDiff("/tmp/exported.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0])) + Expect(err).NotTo(HaveOccurred()) + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + "", t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + tests := []struct { + description string + expectedDataName string + exportArgs []string + rollbackTo string + }{ + { + description: "(186) specify snapshot name with /@ format", + expectedDataName: t.snapshots[1], + exportArgs: []string{ + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "specify snapshot name with @ format", + expectedDataName: t.snapshots[1], + exportArgs: []string{ + "--from-snap", t.snapshots[0], + "-p", t.poolName, + fmt.Sprintf("%s@%s", t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(187) specify snapshot name with --snap option", + expectedDataName: t.snapshots[1], + exportArgs: []string{ + "-p", t.poolName, + "--from-snap", t.snapshots[0], + "--image", t.srcImageName, + "--snap", t.snapshots[1], + }, + rollbackTo: t.snapshots[0], + }, + } + for _, tt := range tests { + By(tt.description) + // export from source snapshot to file + err = cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).NotTo(HaveOccurred()) + + // import to destination image + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo, t.namespace, t.dstDeployName, "") + Expect(err).NotTo(HaveOccurred()) + + // apply snapshot to the destination image + err = cluster.SnapRollback(t.poolName, t.dstImageName, tt.expectedDataName, t.namespace, t.dstDeployName) + Expect(err).NotTo(HaveOccurred()) + + // compare the data in the destination image with the expected data + err = cluster.CompareFilesInPod(tt.expectedDataName, t.namespace, t.dstDeployName, "/mnt/data") + // expect no difference + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemove(t.poolName, t.dstImageName, []string{tt.expectedDataName}) + Expect(err).NotTo(HaveOccurred()) + } + By("(189) specify snapshot which don't have diff with RBD image") + for snapIndex := 1; snapIndex <= 3; snapIndex++ { + err := cluster.ExportDiff("/tmp/exported.bin", + "--from-snap", t.snapshots[snapIndex-1], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[snapIndex])) + Expect(err).NotTo(HaveOccurred()) + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + t.snapshots[snapIndex-1], t.namespace, t.dstDeployName, "") + Expect(err).NotTo(HaveOccurred()) + } + // apply snapshot to the destination image + err = cluster.SnapRollback(t.poolName, t.dstImageName, t.snapshots[3], t.namespace, t.dstDeployName) + Expect(err).NotTo(HaveOccurred()) + // compare the data in the destination image with the expected data + err = cluster.CompareFilesInPod(t.snapshots[2], t.namespace, t.dstDeployName, "/mnt/data") + // expect no difference + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *regressionTest) testForRBDImage() { + It("(184) export-diff for RBD image without --from-snapshot", func() { + err := cluster.ExportDiff("/tmp/exported.bin", "-p", t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + "", t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CompareFilesInPod(t.snapshots[2], t.namespace, t.dstDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + }) + + It("(188) export-diff for RBD image with --from-snapshot", func() { + // export and import snapshots[1] to the destination image as preparation + err := cluster.ExportDiff("/tmp/exported.bin", + "-p", t.poolName, "--image", t.srcImageName, "--snap", t.snapshots[1]) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + "", t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + // test main: export-diff for RBD image with --from-snapshot + err = cluster.ExportDiff("/tmp/exported.bin", + "-p", t.poolName, "--image", t.srcImageName, "--from-snap", t.snapshots[1]) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + t.snapshots[1], t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CompareFilesInPod(t.snapshots[2], t.namespace, t.dstDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + }) +} diff --git a/ceph/test/rollback_test.go b/ceph/test/rollback_test.go new file mode 100644 index 00000000..46305ab3 --- /dev/null +++ b/ceph/test/rollback_test.go @@ -0,0 +1,242 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type rollbackTest struct { + namespace string + poolName string + scName string + + srcDeployName string + srcPVCName string + srcImageName string // will be set in setupEnv after creating PVC + + dstDeployName string + dstPVCName string + dstImageName string // will be set in setupEnv after creating PVC + + snapshots []string +} + +func testRollback() { + test := &rollbackTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + + srcDeployName: util.GetUniqueName("pod-"), + srcPVCName: util.GetUniqueName("src-pvc-"), + dstDeployName: util.GetUniqueName("pod-"), + dstPVCName: util.GetUniqueName("dst-pvc-"), + + snapshots: []string{ + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + }, + } + + Describe("setup environment", test.setupEnv) + Describe("test main", test.test) + Describe("teardown environment", test.teardownEnv) +} + +func (t *rollbackTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.srcPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.srcDeployName, t.srcPVCName, cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.srcPVCName) + Expect(err).NotTo(HaveOccurred()) + t.srcImageName = imageName + + err = cluster.CreatePVC(t.namespace, t.dstPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateDeployment(t.namespace, t.dstDeployName, t.dstPVCName, cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err = cluster.GetImageNameByPVC(t.namespace, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + t.dstImageName = imageName + + // creating snapshots + for i := 0; i < 2; i++ { + err := cluster.MakeRandomFile(t.snapshots[i], int(Quantity2Int("1Mi"))) + Expect(err).NotTo(HaveOccurred()) + err = cluster.PushFileToPod(t.snapshots[i], t.namespace, t.srcDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[i]) + Expect(err).NotTo(HaveOccurred()) + } + }) +} + +func (t *rollbackTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *rollbackTest) test() { + It("checks that rollback feature by writing random data to the target area before rollback", func() { + err := cluster.ExportDiff("/tmp/snapshot0-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot0.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + rollbackMap := map[string]string{ + "/tmp/snapshot0-offset-1Ki.bin": "", + "/tmp/snapshot1-offset-1Ki.bin": t.snapshots[0], + "/tmp/snapshot0.bin": "", + } + + tests := []struct { + description string + expectedDataName string + importsBefore []string + exportArgs []string + rollbackTo string + }{ + { + description: "(212) rbd volume and snapshot diff, whole area (1)", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--from-snap", t.snapshots[0], + "-p", t.poolName, + t.srcImageName, + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(213) rbd volume and snapshot diff, whole area (2)", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "0", + "--from-snap", t.snapshots[0], + "-p", t.poolName, + t.srcImageName, + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(214) export rbd volume without snapshot, whole area (1)", + expectedDataName: t.snapshots[1], + exportArgs: []string{ + "-p", t.poolName, + t.srcImageName, + }, + }, + { + description: "(215) export rbd volume without snapshot, whole area (2)", + expectedDataName: t.snapshots[1], + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "0", + "-p", t.poolName, + t.srcImageName, + }, + }, + { + description: "diff between snapshots, a part of volume", + expectedDataName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", "0", + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "rbd volume and snapshot diff, a part of volume", + expectedDataName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", "0", + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + } + + for _, tt := range tests { + By(tt.description) + + if len(tt.importsBefore) != 0 { + for _, file := range tt.importsBefore { + err = cluster.ImportDiff(file, t.poolName, t.dstImageName, + rollbackMap[file], t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + } + } + + // export from source snapshot to file + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).NotTo(HaveOccurred()) + + // overwrite data in pod before rollback + err = cluster.MakeRandomFile("overwrite-data", int(Quantity2Int("2Mi"))) + Expect(err).NotTo(HaveOccurred()) + err = cluster.PushFileToPod("overwrite-data", t.namespace, t.dstDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + + // rollback & import process + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo, t.namespace, t.dstDeployName, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CompareFilesInPod(tt.expectedDataName, t.namespace, t.dstDeployName, "/mnt/data") + Expect(err).NotTo(HaveOccurred()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + } + }) +} diff --git a/ceph/test/snapshot_name_test.go b/ceph/test/snapshot_name_test.go new file mode 100644 index 00000000..fa2d6d19 --- /dev/null +++ b/ceph/test/snapshot_name_test.go @@ -0,0 +1,428 @@ +package test + +import ( + "fmt" + + "github.com/cybozu-go/mantle/ceph/test/cluster" + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type snapshotNameTest struct { + namespace string + poolName string + scName string + + srcPVCName string + srcImageName string // will be set in setupEnv after creating PVC + + dstPVCName string + dstImageName string // will be set in setupEnv after creating PVC + + snapshots []string +} + +func testSnapshotName() { + test := &snapshotNameTest{ + namespace: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + scName: util.GetUniqueName("sc-"), + + srcPVCName: util.GetUniqueName("src-pvc-"), + dstPVCName: util.GetUniqueName("dst-pvc-"), + + snapshots: []string{ + util.GetUniqueName("snap-"), + util.GetUniqueName("snap-"), + }, + } + + Describe("setup environment", test.setupEnv) + Describe("test main 1", test.test1) + Describe("test main 2", test.test2) + Describe("teardown environment", test.teardownEnv) +} + +func (t *snapshotNameTest) setupEnv() { + It("create resources", func() { + err := cluster.CreatePool(t.poolName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CreateSC(t.scName, t.poolName) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreateNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.CreatePVC(t.namespace, t.srcPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err := cluster.GetImageNameByPVC(t.namespace, t.srcPVCName) + Expect(err).NotTo(HaveOccurred()) + t.srcImageName = imageName + + err = cluster.CreatePVC(t.namespace, t.dstPVCName, t.scName, "10Mi", cluster.VolumeModeFilesystem) + Expect(err).NotTo(HaveOccurred()) + imageName, err = cluster.GetImageNameByPVC(t.namespace, t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + t.dstImageName = imageName + + // creating snapshots + for i := 0; i < 2; i++ { + err = cluster.SnapCreate(t.poolName, t.srcImageName, t.snapshots[i]) + Expect(err).NotTo(HaveOccurred()) + } + }) +} + +func (t *snapshotNameTest) teardownEnv() { + It("delete resources", func() { + err := cluster.SnapRemoveAll(t.poolName, t.srcImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupNamespace(t.namespace) + Expect(err).NotTo(HaveOccurred()) + err = cluster.CleanupGlobal() + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (t *snapshotNameTest) test1() { + It("checks the snapshot names contained in the exported data", func() { + err := cluster.ExportDiff("/tmp/snapshot0-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot0.bin", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + ) + Expect(err).NotTo(HaveOccurred()) + + err = cluster.ExportDiff("/tmp/snapshot1-offset-1Ki.bin", + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + ) + Expect(err).NotTo(HaveOccurred()) + + rollbackMap := map[string]string{ + "/tmp/snapshot0-offset-1Ki.bin": "", + "/tmp/snapshot0.bin": "", + "/tmp/snapshot1-offset-1Ki.bin": t.snapshots[0], + } + + tests := []struct { + description string + expectedSnapshotName string + importsBefore []string + exportArgs []string + rollbackTo string + }{ + { + description: "(190) not specify offset or length, without from-snap", + expectedSnapshotName: t.snapshots[0], + exportArgs: []string{ + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(191) both offset and length are specified as 0, without from-snap", + expectedSnapshotName: t.snapshots[0], + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(192) offset == 0, offset + length < rbd volume size, without from-snap", + expectedSnapshotName: "mid-snap-offset-1024", + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(193) offset == 0, offset + length == rbd volume size, without from-snap", + expectedSnapshotName: t.snapshots[0], + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("10Mi"), + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(194) offset == 0, offset + length > rbd volume size, without from-snap", + expectedSnapshotName: t.snapshots[0], + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("11Mi"), + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(195) offset > 0, length is not specified, without from-snap", + expectedSnapshotName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", "0", + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(198) offset > 0, offset + length < rbd volume size, without from-snap", + expectedSnapshotName: "snapshot0-offset-2048", + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(199) offset > 0, offset + length == rbd volume size, without from-snap", + expectedSnapshotName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("10Mi")-1024), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(200) offset > 0, offset + length > rbd volume size, without from-snap", + expectedSnapshotName: t.snapshots[0], + importsBefore: []string{"/tmp/snapshot0-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("11Mi"), + "--mid-snap-prefix", "snapshot0", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + rollbackTo: "snapshot0-offset-1024", + }, + { + description: "(203) not specify offset or length, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(204) both offset and length are specified as 0, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--read-offset", "0", + "--read-length", "0", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(205) offset == 0, offset + length < rbd volume size, having from-snap", + expectedSnapshotName: "snapshot1-offset-1024", + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(206) offset == 0, offset + length == rbd volume size, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("10Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(207) offset == 0, offset + length > rbd volume size, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin"}, + exportArgs: []string{ + "--read-offset", "0", + "--read-length", Quantity2Str("11Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: t.snapshots[0], + }, + { + description: "(208) offset > 0, length is not specified, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", "0", + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "(209) offset > 0, offset + length < rbd volume size, having from-snap", + expectedSnapshotName: "snapshot1-offset-2048", + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("1Ki"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "(210) offset > 0, offset + length == rbd volume size, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", fmt.Sprintf("%d", Quantity2Int("10Mi")-1024), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + { + description: "(211) offset > 0, offset + length > rbd volume size, having from-snap", + expectedSnapshotName: t.snapshots[1], + importsBefore: []string{"/tmp/snapshot0.bin", "/tmp/snapshot1-offset-1Ki.bin"}, + exportArgs: []string{ + "--read-offset", Quantity2Str("1Ki"), + "--read-length", Quantity2Str("11Mi"), + "--mid-snap-prefix", "snapshot1", + "--from-snap", t.snapshots[0], + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[1]), + }, + rollbackTo: "snapshot1-offset-1024", + }, + } + + for _, tt := range tests { + By(tt.description) + + // export from source snapshot to file + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).NotTo(HaveOccurred()) + + if len(tt.importsBefore) != 0 { + // try import and should fail + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + "", t.namespace, "", t.dstPVCName) + Expect(err).To(HaveOccurred()) + + for _, file := range tt.importsBefore { + err = cluster.ImportDiff(file, t.poolName, t.dstImageName, + rollbackMap[file], t.namespace, "", t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + } + } + + // check snapshot before import + exists, err := cluster.SnapExists(t.poolName, t.dstImageName, tt.expectedSnapshotName) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + + // import to destination image + err = cluster.ImportDiff("/tmp/exported.bin", t.poolName, t.dstImageName, + tt.rollbackTo, t.namespace, "", t.dstPVCName) + Expect(err).NotTo(HaveOccurred()) + + // check snapshot after import + exists, err = cluster.SnapExists(t.poolName, t.dstImageName, tt.expectedSnapshotName) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + // cleanup + err = cluster.SnapRemoveAll(t.poolName, t.dstImageName) + Expect(err).NotTo(HaveOccurred()) + } + }) +} + +func (t *snapshotNameTest) test2() { + It("checks error if offset is greater than rbd volume size", func() { + tests := []struct { + description string + exportArgs []string + }{ + { + description: "(196) offset == rbd volume size, unset length", + exportArgs: []string{ + "--read-offset", Quantity2Str("10Mi"), + "--read-length", "0", + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(197) offset > rbd volume size, unset length", + exportArgs: []string{ + "--read-offset", Quantity2Str("11Mi"), + "--read-length", "0", + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(201) offset == rbd volume size, set length", + exportArgs: []string{ + "--read-offset", Quantity2Str("10Mi"), + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + { + description: "(202) offset > rbd volume size, set length", + exportArgs: []string{ + "--read-offset", Quantity2Str("11Mi"), + "--read-length", Quantity2Str("1Mi"), + "--mid-snap-prefix", "mid-snap", + fmt.Sprintf("%s/%s@%s", t.poolName, t.srcImageName, t.snapshots[0]), + }, + }, + } + + for _, tt := range tests { + By(tt.description) + err := cluster.ExportDiff("/tmp/exported.bin", tt.exportArgs...) + Expect(err).To(HaveOccurred()) + } + }) +} diff --git a/ceph/test/suite_test.go b/ceph/test/suite_test.go index 19e93ee6..a81bdf94 100644 --- a/ceph/test/suite_test.go +++ b/ceph/test/suite_test.go @@ -1,7 +1,39 @@ -package ceph +package test -import "testing" +import ( + "os" + "testing" + "time" -func TestCustomCeph(t *testing.T) { - t.Log("implement me!") + "github.com/cybozu-go/mantle/ceph/test/cluster" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + envSkipCephCmdTest = "SKIP_CEPH_CMD_TEST" +) + +func TestMTest(t *testing.T) { + if os.Getenv(envSkipCephCmdTest) == "1" { + t.Skipf("tests for custom rbd export-diff command are skipped by %s is set to 1", envSkipCephCmdTest) + } + defer cluster.RemoveWorkDir() + + RegisterFailHandler(Fail) + + SetDefaultEventuallyPollingInterval(time.Second) + SetDefaultEventuallyTimeout(3 * time.Minute) + EnforceDefaultTimeoutsWhenUsingContexts() + + RunSpecs(t, "tests for custom rbd export-diff command") } + +var _ = Describe("root of tests", func() { + Context("content", testContent) + Context("extend", testExtend) + Context("options", testOptions) + Context("regression", testRegression) + Context("rollback", testRollback) + Context("snapshot name", testSnapshotName) +}) diff --git a/ceph/test/util.go b/ceph/test/util.go new file mode 100644 index 00000000..634a9b40 --- /dev/null +++ b/ceph/test/util.go @@ -0,0 +1,16 @@ +package test + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func Quantity2Str(qStr string) string { + return fmt.Sprintf("%d", Quantity2Int(qStr)) +} + +func Quantity2Int(qStr string) uint64 { + q := resource.MustParse(qStr) + return uint64(q.Value()) +}