Add tests for custom rbd export-diff command

.gitignore

@@ -25,6 +25,7 @@ Dockerfile.cross
 *.swo
 *~
 
+ceph/ceph-custom.tar
 ceph/go.mod
 ceph/go.sum
 ceph/packages/

Makefile

@@ -89,7 +89,9 @@ mock: mockgen
 .PHONY: test
 test: manifests generate fmt vet envtest mock ## Run tests.
 # adding -p 1 -v to stream logs. see https://github.com/golang/go/issues/46959
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out -p 1 -v
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+	SKIP_CEPH_CMD_TEST=1 \
+	go test ./... -coverprofile cover.out -p 1 -v
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint:

ceph/Makefile

@@ -15,27 +15,27 @@ export MINIKUBE_HOME
 .PHONY: build
 build: ceph-custom.tar packages/touch
 
-ceph-custom.tar: build-docker
+ceph-custom.tar: Dockerfile $(PATCH_FILES)
+	docker build -f Dockerfile -t ceph-custom .
 	docker save ceph-custom -o $@
 
-packages/touch: build-docker
+packages/touch: ceph-custom.tar
 	rm -rf packages
 	docker create --name ceph-proc ceph-custom
 	docker cp ceph-proc:/packages .
 	docker rm ceph-proc
 	touch $@
 
-build-docker: Dockerfile $(PATCH_FILES)
-	docker build -f Dockerfile -t ceph-custom .
-
 .PHONY: test
 test: ceph-custom.tar
 	$(MAKE) -C ${E2E_DIR} launch-minikube MINIKUBE_PROFILE=minikube
 	$(MAKE) -C ${E2E_DIR} install-rook-ceph-operator
 	$(MAKE) -C ${E2E_DIR} install-rook-ceph-cluster1
 	$(MINIKUBE_CMD) image load ceph-custom.tar
 	$(KUBECTL_CMD) apply -k ./manifests/
-	go test -v -count=1 ./test/
+	env \
+	  KUBECTL=$(KUBECTL_CMD) \
+	  go test -v -count=1 -timeout 1h ./test/
 
 .PHONY: setup
 setup:

ceph/export-diff.patch

@@ -11,7 +11,7 @@
  // encryption arguments
  static const std::string ENCRYPTION_FORMAT("encryption-format");
 diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
-index ddcf0f2c30c..2ba0be2ebb1 100644
+index ddcf0f2c30c..1b72db2bde8 100644
 --- a/src/tools/rbd/action/Export.cc
 +++ b/src/tools/rbd/action/Export.cc
 @@ -123,16 +123,27 @@ private:
@@ -112,8 +112,8 @@ index ddcf0f2c30c..2ba0be2ebb1 100644
       "snapshot starting point")
 -    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object");
 +    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object")
-+    (at::READ_OFFSET.c_str(), po::value<uint64_t>(), "offset in bytes")
-+    (at::READ_LENGTH.c_str(), po::value<uint64_t>(), "length in bytes")
++    (at::READ_OFFSET.c_str(), po::value<int64_t>(), "offset in bytes")
++    (at::READ_LENGTH.c_str(), po::value<int64_t>(), "length in bytes")
 +    (at::MID_SNAP_PREFIX.c_str(), po::value<std::string>(),
 +     "the prefix of snapshot name in output diff when specifying offset and length");
    at::add_no_progress_option(options);
@@ -159,7 +159,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644
  int execute_diff(const po::variables_map &vm,
                   const std::vector<std::string> &ceph_global_init_args) {
    size_t arg_index = 0;
-@@ -290,6 +345,26 @@ int execute_diff(const po::variables_map &vm,
+@@ -290,6 +345,45 @@ int execute_diff(const po::variables_map &vm,
      from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
    }
 
@@ -170,23 +170,42 @@ index ddcf0f2c30c..2ba0be2ebb1 100644
 +
 +  uint64_t offset = 0;
 +  if (vm.count(at::READ_OFFSET)) {
-+    offset = vm[at::READ_OFFSET].as<uint64_t>();
++    // When passing a negative value as an argument for uint64_t typed arg using boost::program_options,
++    // it is casted without causing an error, so check logic was necessary.
++    int64_t s_offset = vm[at::READ_OFFSET].as<int64_t>();
++    if (s_offset < 0) {
++      std::cerr << "rbd: offset must be greater than or equal to 0" << std::endl;
++      return -EINVAL;
++    }
++    offset = static_cast<uint64_t>(s_offset);
 +  }
 +
 +  uint64_t length = 0;
 +  if (vm.count(at::READ_LENGTH)) {
-+    length = vm[at::READ_LENGTH].as<uint64_t>();
++    // When passing a negative value as an argument for uint64_t typed arg using boost::program_options,
++    // it is casted without causing an error, so check logic was necessary.
++    int64_t s_length = vm[at::READ_LENGTH].as<int64_t>();
++    if (s_length < 0) {
++      std::cerr << "rbd: length must be greater than or equal to 0" << std::endl;
++      return -EINVAL;
++    }
++    length = static_cast<uint64_t>(s_length);
 +  }
 +
 +  std::string mid_snap_prefix("mid-snap");
 +  if (vm.count(at::MID_SNAP_PREFIX)) {
 +    mid_snap_prefix = vm[at::MID_SNAP_PREFIX].as<std::string>();
++    r = utils::validate_snapshot_name(at::ARGUMENT_MODIFIER_SOURCE, mid_snap_prefix,
++                                      utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_SNAP);
++    if (r < 0) {
++      return r;
++    }
 +  }
 +
    librados::Rados rados;
    librados::IoCtx io_ctx;
    librbd::Image image;
-@@ -299,9 +374,28 @@ int execute_diff(const po::variables_map &vm,
+@@ -299,9 +393,28 @@ int execute_diff(const po::variables_map &vm,
      return r;
    }
 
@@ -215,7 +234,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644
                       vm[at::WHOLE_OBJECT].as<bool>(), path.c_str(),
                       vm[at::NO_PROGRESS].as<bool>());
    if (r < 0) {
-@@ -501,7 +595,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
+@@ -501,7 +614,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
    const char *last_snap = NULL;
    for (size_t i = 0; i < snaps.size(); ++i) {
      utils::snap_set(image, snaps[i].name.c_str());
@@ -225,7 +244,7 @@ index ddcf0f2c30c..2ba0be2ebb1 100644
      if (r < 0) {
        return r;
      }
-@@ -509,7 +604,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
+@@ -509,7 +623,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
      last_snap = snaps[i].name.c_str();
    }
    utils::snap_set(image, std::string(""));

ceph/test/cluster/block.go

@@ -0,0 +1,73 @@
+package cluster
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/cybozu-go/mantle/test/util"
+)
+
+func DiscardBlock(namespace, deployName string) error {
+	_, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--",
+		"blkdiscard", "/dev/rbd-device")
+	if err != nil {
+		return fmt.Errorf("failed to discard volume: %w", err)
+	}
+	return nil
+}
+
+func WriteRandomBlock(namespace, deployName string, offset, size uint64) error {
+	_, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--",
+		"dd", "if=/dev/urandom", "of=/dev/rbd-device", "bs=1K",
+		fmt.Sprintf("seek=%d", offset/1024), fmt.Sprintf("count=%d", size/1024), "oflag=direct,dsync")
+	if err != nil {
+		return fmt.Errorf("failed to write random block: %w", err)
+	}
+	return nil
+}
+
+func GetBlockAsFile(namespace, deployName, filename string) error {
+	const workFilename = "/tmp/work.bin"
+
+	_, err := Kubectl("exec", "-n", namespace, "deploy/"+deployName, "--",
+		"dd", "if=/dev/rbd-device", "of="+workFilename, "bs=1K")
+	if err != nil {
+		return fmt.Errorf("failed to get block as file: %w", err)
+	}
+
+	podName, err := GetPodNameByDeploy(namespace, deployName)
+	if err != nil {
+		return err
+	}
+	_, err = Kubectl("cp", namespace+"/"+podName+":"+workFilename, path.Join(workDir, filename))
+	if err != nil {
+		return fmt.Errorf("failed to copy file to pod: %w", err)
+	}
+
+	return RemoveFileByPod(namespace, deployName, workFilename)
+}
+
+func CompareBlockWithFile(namespace, deployName, filename string) error {
+	workFilename := util.GetUniqueName("compare-file-")
+	defer func() {
+		_ = os.Remove(path.Join(workDir, workFilename))
+	}()
+
+	if err := GetBlockAsFile(namespace, deployName, workFilename); err != nil {
+		return err
+	}
+
+	args := []string{path.Join(workDir, filename), path.Join(workDir, workFilename)}
+	log.Printf("📂 cmp %s", strings.Join(args, " "))
+	_, err := exec.Command("cmp", args...).CombinedOutput()
+	if err != nil {
+		showMD5Sum(path.Join(workDir, filename))
+		showMD5Sum(path.Join(workDir, workFilename))
+		return fmt.Errorf("the devices having differences: %w", err)
+	}
+	return nil
+}

ceph/test/cluster/filesystem.go

@@ -0,0 +1,105 @@
+package cluster
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/cybozu-go/mantle/test/util"
+)
+
+var workDir string
+
+func MakeRandomFile(filename string, size int) error {
+	args := []string{"if=/dev/urandom", "of=" + path.Join(workDir, filename), "bs=1K", fmt.Sprintf("count=%d", size/1024)}
+	log.Printf("📂 dd %s", strings.Join(args, " "))
+	command := exec.Command("dd", args...)
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+	defer showMD5Sum(path.Join(workDir, filename))
+	return command.Run()
+}
+
+func PushFileToPod(filename, namespace, deployName, dst string) error {
+	podName, err := GetPodNameByDeploy(namespace, deployName)
+	if err != nil {
+		return err
+	}
+	_, err = Kubectl("cp", path.Join(workDir, filename), namespace+"/"+podName+":"+dst)
+	if err != nil {
+		return fmt.Errorf("failed to copy file to pod: %w", err)
+	}
+
+	_, err = Kubectl("exec", "-n", namespace, podName, "--", "sync")
+	return err
+}
+
+func RemoveFileByPod(namespace, deployName, target string) error {
+	podName, err := GetPodNameByDeploy(namespace, deployName)
+	if err != nil {
+		return err
+	}
+	_, err = Kubectl("exec", "-n", namespace, podName, "--", "rm", "-f", target)
+	if err != nil {
+		return fmt.Errorf("failed to remove file in pod: %w", err)
+	}
+
+	_, err = Kubectl("exec", "-n", namespace, podName, "--", "sync")
+	return err
+}
+
+// CompareFilesInPod compares the file in the host(expected) with in the pod.
+func CompareFilesInPod(filename, namespace, deployName, target string) error {
+	workFilename := util.GetUniqueName("compare-file-")
+	defer func() {
+		_ = os.Remove(path.Join(workDir, workFilename))
+	}()
+
+	podName, err := GetPodNameByDeploy(namespace, deployName)
+	if err != nil {
+		return err
+	}
+	_, err = Kubectl("cp", namespace+"/"+podName+":"+target, path.Join(workDir, workFilename))
+	if err != nil {
+		return err
+	}
+
+	args := []string{path.Join(workDir, filename), path.Join(workDir, workFilename)}
+	log.Printf("📂 diff %s", strings.Join(args, " "))
+	_, err = exec.Command("diff", args...).CombinedOutput()
+	if err != nil {
+		showMD5Sum(path.Join(workDir, filename))
+		showMD5Sum(path.Join(workDir, workFilename))
+		return fmt.Errorf("the files having differences: %w", err)
+	}
+	return nil
+}
+
+func RemoveWorkDir() {
+	if err := os.RemoveAll(workDir); err != nil {
+		log.Fatalf("failed to remove workDir: %v", err)
+	}
+}
+
+func showMD5Sum(filename string) {
+	args := []string{filename}
+	log.Printf("📂 md5sum %s", strings.Join(args, " "))
+	command := exec.Command("md5sum", args...)
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+	err := command.Run()
+	if err != nil {
+		log.Fatalf("failed to run md5sum: %v", err)
+	}
+}
+
+func init() {
+	dir, err := os.MkdirTemp("", "test-mantle-ceph-")
+	if err != nil {
+		log.Fatalf("failed to create workDir: %v", err)
+	}
+	workDir = dir
+}