Skip to content

Commit

Permalink
add tests for custom rbd export-diff command
Browse files Browse the repository at this point in the history
Signed-off-by: Yuji Ito <[email protected]>
  • Loading branch information
llamerada-jp committed Jan 29, 2025
1 parent 87ac222 commit 93f1358
Show file tree
Hide file tree
Showing 10 changed files with 360 additions and 25 deletions.
13 changes: 7 additions & 6 deletions ceph/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,28 @@ export MINIKUBE_HOME
.PHONY: build
build: ceph-custom.tar packages/touch

ceph-custom.tar: build-docker
ceph-custom.tar: Dockerfile $(PATCH_FILES)
docker build -f Dockerfile -t ceph-custom .
docker save ceph-custom -o $@

packages/touch: build-docker
packages/touch: ceph-custom.tar
rm -rf packages
docker create --name ceph-proc ceph-custom
docker cp ceph-proc:/packages .
docker rm ceph-proc
touch $@

build-docker: Dockerfile $(PATCH_FILES)
docker build -f Dockerfile -t ceph-custom .

.PHONY: test
test: ceph-custom.tar
$(MAKE) -C ${E2E_DIR} launch-minikube MINIKUBE_PROFILE=minikube
$(MAKE) -C ${E2E_DIR} install-rook-ceph-operator
$(MAKE) -C ${E2E_DIR} install-rook-ceph-cluster1
$(MINIKUBE_CMD) image load ceph-custom.tar
$(KUBECTL_CMD) apply -k ./manifests/
go test -v -count=1 ./test/
env \
CEPH_TEST=1 \
KUBECTL=$(KUBECTL_CMD) \
go test -v -count=1 ./test/

.PHONY: setup
setup:
Expand Down
37 changes: 21 additions & 16 deletions ceph/export-diff.patch
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
// encryption arguments
static const std::string ENCRYPTION_FORMAT("encryption-format");
diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
index ddcf0f2c30cef..cc84a68193bdc 100644
index ddcf0f2c30cef..af43a94be13a5 100644
--- a/src/tools/rbd/action/Export.cc
+++ b/src/tools/rbd/action/Export.cc
@@ -123,16 +123,26 @@ class C_ExportDiff : public Context {
@@ -123,16 +123,27 @@ class C_ExportDiff : public Context {


int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
Expand All @@ -27,10 +27,11 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
int r;
librbd::image_info_t info;

+ std::cerr << "do_export_diff: fromsnapname=" << fromsnapname <<
+ " endsnapname=" << endsnapname << " out_fromsnapname=" << out_fromsnapname <<
+ " out_endsnapname=" << out_endsnapname << " offset=" << offset <<
+ " length=" << length << std::endl;
+ std::cerr << "do_export_diff(fromsnapname=" << (fromsnapname ? fromsnapname : "null") <<
+ " endsnapname=" << (endsnapname ? endsnapname : "null") <<
+ " out_fromsnapname=" << (out_fromsnapname ? out_fromsnapname : "null") <<
+ " out_endsnapname=" << (out_endsnapname ? out_endsnapname : "null") <<
+ " offset=" << offset << " length=" << length << ")" << std::endl;
+
r = image.stat(info, sizeof(info));
if (r < 0)
Expand All @@ -43,7 +44,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
{
// header
bufferlist bl;
@@ -143,10 +153,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
@@ -143,10 +154,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,

__u8 tag;
uint64_t len = 0;
Expand All @@ -56,7 +57,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
if (export_format == 2) {
len = from.length() + 4;
encode(len, bl);
@@ -154,10 +164,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
@@ -154,10 +165,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
encode(from, bl);
}

Expand All @@ -69,7 +70,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
if (export_format == 2) {
len = to.length() + 4;
encode(len, bl);
@@ -192,10 +202,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
@@ -192,10 +203,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
return r;
}
}
Expand All @@ -82,7 +83,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
&C_ExportDiff::export_diff_cb, (void *)&edc);
if (r < 0) {
goto out;
@@ -223,8 +233,9 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
@@ -223,8 +234,9 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
}

int do_export_diff(librbd::Image& image, const char *fromsnapname,
Expand All @@ -94,7 +95,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
{
int r;
int fd;
@@ -236,7 +247,9 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname,
@@ -236,7 +248,9 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname,
if (fd < 0)
return -errno;

Expand All @@ -105,7 +106,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644

if (fd != 1)
close(fd);
@@ -260,10 +273,52 @@ void get_arguments_diff(po::options_description *positional,
@@ -260,10 +274,52 @@ void get_arguments_diff(po::options_description *positional,
options->add_options()
(at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
"snapshot starting point")
Expand Down Expand Up @@ -159,7 +160,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
int execute_diff(const po::variables_map &vm,
const std::vector<std::string> &ceph_global_init_args) {
size_t arg_index = 0;
@@ -290,6 +345,26 @@ int execute_diff(const po::variables_map &vm,
@@ -290,6 +346,26 @@ int execute_diff(const po::variables_map &vm,
from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
}

Expand All @@ -186,14 +187,18 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
librados::Rados rados;
librados::IoCtx io_ctx;
librbd::Image image;
@@ -299,9 +374,24 @@ int execute_diff(const po::variables_map &vm,
@@ -299,9 +375,28 @@ int execute_diff(const po::variables_map &vm,
return r;
}

+ std::string out_from_snap_name = from_snap_name;
+ std::string out_end_snap_name = snap_name;
+
+ if (offset != 0 || length != 0) {
+ if (snap_name.empty()) {
+ std::cerr << "rbd: must specify snapshot when specifying offset and length" << std::endl;
+ return -EINVAL;
+ }
+ r = get_snapshot_name_for_offset_length(image, mid_snap_prefix,
+ &out_from_snap_name, &out_end_snap_name,
+ &offset, &length);
Expand All @@ -211,7 +216,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
vm[at::WHOLE_OBJECT].as<bool>(), path.c_str(),
vm[at::NO_PROGRESS].as<bool>());
if (r < 0) {
@@ -501,7 +591,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
@@ -501,7 +596,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
const char *last_snap = NULL;
for (size_t i = 0; i < snaps.size(); ++i) {
utils::snap_set(image, snaps[i].name.c_str());
Expand All @@ -221,7 +226,7 @@ index ddcf0f2c30cef..cc84a68193bdc 100644
if (r < 0) {
return r;
}
@@ -509,7 +600,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
@@ -509,7 +605,8 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
last_snap = snaps[i].name.c_str();
}
utils::snap_set(image, std::string(""));
Expand Down
2 changes: 2 additions & 0 deletions ceph/manifests/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- pool.yaml
- storageclass.yaml
- toolbox.yaml
namespace: rook-ceph
images:
Expand Down
10 changes: 10 additions & 0 deletions ceph/manifests/pool.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: test-pool
namespace: rook-ceph
spec:
failureDomain: osd
replicated:
size: 1
requireSafeReplicaSize: false
32 changes: 32 additions & 0 deletions ceph/manifests/storageclass.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: test-sc
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
clusterID: rook-ceph
pool: test-pool
imageFormat: "2"
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: test-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: test-sc
95 changes: 95 additions & 0 deletions ceph/test/cluster/kubectl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package cluster

import (
"bytes"
_ "embed"
"fmt"
"os"
"os/exec"
)

var (
//go:embed pvc-pod-template.yaml
templatePVCPod string
)

var kubectlCmd = os.Getenv("KUBECTL")

func Kubectl(args ...string) ([]byte, error) {
if len(kubectlCmd) == 0 {
return nil, fmt.Errorf("KUBECTL environment variable should be set")
}

var stdout bytes.Buffer
command := exec.Command(kubectlCmd, args...)
command.Stdout = &stdout

err := command.Run()
return stdout.Bytes(), err
}

func KubectlWithInput(stdin []byte, args ...string) ([]byte, error) {
if len(kubectlCmd) == 0 {
return nil, fmt.Errorf("KUBECTL environment variable should be set")
}

var stdout bytes.Buffer
command := exec.Command(kubectlCmd, args...)
command.Stdin = bytes.NewReader(stdin)
command.Stdout = &stdout

err := command.Run()
return stdout.Bytes(), err
}

func CreateNamespace(namespace string) error {
_, err := Kubectl("create", "namespace", namespace)
if err != nil {
return fmt.Errorf("failed to create namespace: %w", err)
}
return nil
}

func DeleteNamespace(namespace string) error {
_, err := Kubectl("delete", "namespace", namespace)
if err != nil {
return fmt.Errorf("failed to delete namespace: %w", err)
}
return nil
}

func CreatePodPVC(namespace, podName, pvcName string) error {
manifest := fmt.Sprintf(templatePVCPod, podName, namespace, pvcName, pvcName, namespace)
_, err := KubectlWithInput([]byte(manifest), "apply", "-f", "-")
if err != nil {
return fmt.Errorf("failed to create pod and pvc: %w", err)
}
return nil
}

func DeletePodPVC(namespace, podName, pvcName string) error {
_, err := Kubectl("delete", "pod", podName, "-n", namespace)
if err != nil {
return fmt.Errorf("failed to delete pod: %w", err)
}
_, err = Kubectl("delete", "pvc", pvcName, "-n", namespace)
if err != nil {
return fmt.Errorf("failed to delete pvc: %w", err)
}
return nil
}

func GetImageNameByPVC(namespace, pvcName string) (string, error) {
stdout, err := Kubectl("get", "-n", namespace, "pvc", pvcName, "-o", "jsonpath={.spec.volumeName}")
if err != nil {
return "", fmt.Errorf("failed to get volume name by PVC: %w", err)
}

volumeName := string(stdout)
stdout, err = Kubectl("get", "pv", volumeName, "-o", "jsonpath={.spec.csi.volumeAttributes.imageName}")
if err != nil {
return "", fmt.Errorf("failed to get image name by volume: %w", err)
}

return string(stdout), nil
}
30 changes: 30 additions & 0 deletions ceph/test/cluster/pvc-pod-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
apiVersion: v1
kind: Pod
metadata:
name: %s
namespace: %s
spec:
containers:
- name: ubuntu
image: ubuntu:20.04
command: ["/usr/bin/sleep", "infinity"]
volumeMounts:
- mountPath: /mnt
name: my-volume
volumes:
- name: my-volume
persistentVolumeClaim:
claimName: %s
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: %s
namespace: %s
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: test-sc
31 changes: 31 additions & 0 deletions ceph/test/cluster/rbd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package cluster

import "fmt"

func Rbd(args ...string) ([]byte, error) {
return Kubectl(append([]string{"exec", "-n", "rook-ceph", "deploy/rook-ceph-tools", "--", "rbd"}, args...)...)
}

func ExportDiff(args ...string) error {
stdout, err := Rbd(append([]string{"export-diff"}, args...)...)
if err != nil {
return fmt.Errorf("failed to run rbd export-diff command: %w, %s", err, string(stdout))
}
return nil
}

func SnapCreate(pool, image, snap string) error {
stdout, err := Rbd("snap", "create", pool+"/"+image+"@"+snap)
if err != nil {
return fmt.Errorf("failed to create snapshot: %w, %s", err, string(stdout))
}
return nil
}

func SnapRemove(pool, image, snap string) error {
stdout, err := Rbd("snap", "rm", pool+"/"+image+"@"+snap)
if err != nil {
return fmt.Errorf("failed to remove snapshot: %w, %s", err, string(stdout))
}
return nil
}
Loading

0 comments on commit 93f1358

Please sign in to comment.