ZTS: tests for dedup legacy/FDT tables

Very basic coverage to make sure things appear to work, have the right
format on disk, and pool upgrades and mixed table types work as
expected.

Reviewed-by: Alexander Motin <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Rob Norris <[email protected]>
Sponsored-by: Klara, Inc.
Sponsored-by: iXsystems, Inc.
Closes openzfs#15892

tests/runfiles/common.run

@@ -672,7 +672,9 @@ post =
 tags = ['functional', 'deadman']
 
 [tests/functional/dedup]
-tests = ['dedup_quota']
+tests = ['dedup_legacy_create', 'dedup_fdt_create', 'dedup_fdt_import',
+    'dedup_legacy_create', 'dedup_legacy_import', 'dedup_legacy_fdt_upgrade',
+    'dedup_legacy_fdt_mixed', 'dedup_quota']
 pre =
 post =
 tags = ['functional', 'dedup']

tests/zfs-tests/tests/Makefile.am

@@ -1424,6 +1424,12 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/deadman/deadman_zio.ksh \
 	functional/dedup/cleanup.ksh \
 	functional/dedup/setup.ksh \
+	functional/dedup/dedup_fdt_create.ksh \
+	functional/dedup/dedup_fdt_import.ksh \
+	functional/dedup/dedup_legacy_create.ksh \
+	functional/dedup/dedup_legacy_import.ksh \
+	functional/dedup/dedup_legacy_fdt_upgrade.ksh \
+	functional/dedup/dedup_legacy_fdt_mixed.ksh \
 	functional/dedup/dedup_quota.ksh \
 	functional/delegate/cleanup.ksh \
 	functional/delegate/setup.ksh \

tests/zfs-tests/tests/functional/dedup/dedup_fdt_create.ksh

@@ -0,0 +1,99 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2024 Klara, Inc.
+#
+
+# Simple test of dedup table operations (FDT)
+
+. $STF_SUITE/include/libtest.shlib
+
+log_assert "basic dedup (FDT) operations work"
+
+function cleanup
+{
+	destroy_pool $TESTPOOL
+}
+
+log_onexit cleanup
+
+# create a pool with fast dedup enabled. we disable block cloning to ensure
+# it doesn't get in the way of dedup, and we disable compression so our writes
+# create predictable results on disk
+# Use 'xattr=sa' to prevent selinux xattrs influencing our accounting
+log_must zpool create -f \
+    -o feature@fast_dedup=enabled \
+    -O dedup=on \
+    -o feature@block_cloning=disabled \
+    -O compression=off \
+    -O xattr=sa \
+    $TESTPOOL $DISKS
+
+# confirm the feature is enabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "enabled"
+
+# confirm there's no DDT keys in the MOS root
+log_mustnot eval "zdb -dddd $TESTPOOL 1 | grep -q DDT-sha256"
+
+# create a file. this is four full blocks, so will produce four entries in the
+# dedup table
+log_must dd if=/dev/urandom of=/$TESTPOOL/file1 bs=128k count=4
+log_must zpool sync
+
+# feature should now be active
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "active"
+
+# four entries in the unique table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-unique: 4 entries'"
+
+# single containing object in the MOS
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | wc -l) -eq 1
+obj=$(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | awk '{ print $NF }')
+
+# with only one ZAP inside
+log_must test $(zdb -dddd $TESTPOOL $obj | grep DDT-sha256-zap- | wc -l) -eq 1
+
+# copy the file
+log_must cp /$TESTPOOL/file1 /$TESTPOOL/file2
+log_must zpool sync
+
+# now four entries in the duplicate table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-duplicate: 4 entries'"
+
+# now two DDT ZAPs in the container object; DDT ZAPs aren't cleaned up until
+# the entire logical table is destroyed
+log_must test $(zdb -dddd $TESTPOOL $obj | grep DDT-sha256-zap- | wc -l) -eq 2
+
+# remove the files
+log_must rm -f /$TESTPOOL/file*
+log_must zpool sync
+
+# feature should move back to enabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "enabled"
+
+# all DDTs empty
+log_must eval "zdb -D $TESTPOOL | grep -q 'All DDTs are empty'"
+
+# logical table now destroyed; containing object destroyed
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | wc -l) -eq 0
+
+log_pass "basic dedup (FDT) operations work"

tests/zfs-tests/tests/functional/dedup/dedup_fdt_import.ksh

@@ -0,0 +1,112 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2024 Klara, Inc.
+#
+
+# Ensure dedup retains version after import (FDT)
+
+. $STF_SUITE/include/libtest.shlib
+
+log_assert "dedup (FDT) retains version after import"
+
+function cleanup
+{
+	destroy_pool $TESTPOOL
+}
+
+log_onexit cleanup
+
+# create a pool with fast dedup enabled. we disable block cloning to ensure
+# it doesn't get in the way of dedup, and we disable compression so our writes
+# create predictable results on disk
+# Use 'xattr=sa' to prevent selinux xattrs influencing our accounting
+log_must zpool create -f \
+    -o feature@fast_dedup=enabled \
+    -O dedup=on \
+    -o feature@block_cloning=disabled \
+    -O compression=off \
+    -O xattr=sa \
+    $TESTPOOL $DISKS
+
+# confirm the feature is enabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "enabled"
+
+# confirm there's no DDT keys in the MOS root
+log_mustnot eval "zdb -dddd $TESTPOOL 1 | grep -q DDT-sha256"
+
+# create a file. this is four full blocks, so will produce four entries in the
+# dedup table
+log_must dd if=/dev/urandom of=/$TESTPOOL/file1 bs=128k count=4
+log_must zpool sync
+
+# feature should now be active
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "active"
+
+# four entries in the unique table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-unique: 4 entries'"
+
+# single containing object in the MOS
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | wc -l) -eq 1
+obj=$(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | awk '{ print $NF }')
+
+# with only one ZAP inside
+log_must test $(zdb -dddd $TESTPOOL $obj | grep DDT-sha256-zap- | wc -l) -eq 1
+
+# export and import the pool
+zpool export $TESTPOOL
+zpool import $TESTPOOL
+
+# feature still active
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "active"
+
+# remove the file
+log_must rm -f /$TESTPOOL/file1
+log_must zpool sync
+
+# feature should revert to enabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "enabled"
+
+# all DDTs empty
+log_must eval "zdb -D $TESTPOOL | grep -q 'All DDTs are empty'"
+
+# logical table now destroyed; containing object destroyed
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | wc -l) -eq 0
+
+# create a new file
+log_must dd if=/dev/urandom of=/$TESTPOOL/file2 bs=128k count=4
+log_must zpool sync
+
+# feature should be active again
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "active"
+
+# four entries in the unique table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-unique: 4 entries'"
+
+# single containing object in the MOS
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | wc -l) -eq 1
+obj=$(zdb -dddd $TESTPOOL 1 | grep DDT-sha256 | awk '{ print $NF }')
+
+# with only one ZAP inside
+log_must test $(zdb -dddd $TESTPOOL $obj | grep DDT-sha256-zap- | wc -l) -eq 1
+
+log_pass "dedup (FDT) retains version after import"

tests/zfs-tests/tests/functional/dedup/dedup_legacy_create.ksh

@@ -0,0 +1,95 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2024 Klara, Inc.
+#
+
+# Simple test of dedup table operations (legacy)
+
+. $STF_SUITE/include/libtest.shlib
+
+log_assert "basic dedup (legacy) operations work"
+
+function cleanup
+{
+	destroy_pool $TESTPOOL
+}
+
+log_onexit cleanup
+
+# create a pool with legacy dedup enabled. we disable block cloning to ensure
+# it doesn't get in the way of dedup, and we disable compression so our writes
+# create predictable results on disk
+# Use 'xattr=sa' to prevent selinux xattrs influencing our accounting
+log_must zpool create -f \
+    -o feature@fast_dedup=disabled \
+    -O dedup=on \
+    -o feature@block_cloning=disabled \
+    -O compression=off \
+    -O xattr=sa \
+    $TESTPOOL $DISKS
+
+# confirm the feature is disabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "disabled"
+
+# confirm there's no DDT keys in the MOS root
+log_mustnot eval "zdb -dddd $TESTPOOL 1 | grep -q DDT-sha256"
+
+# create a file. this is four full blocks, so will produce four entries in the
+# dedup table
+log_must dd if=/dev/urandom of=/$TESTPOOL/file1 bs=128k count=4
+log_must zpool sync
+
+# feature should still be disabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "disabled"
+
+# should be four entries in the unique table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-unique: 4 entries'"
+
+# should be just one DDT ZAP in the MOS
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256-zap- | wc -l) -eq 1
+
+# copy the file
+log_must cp /$TESTPOOL/file1 /$TESTPOOL/file2
+log_must zpool sync
+
+# now four entries in the duplicate table
+log_must eval "zdb -D $TESTPOOL | grep -q 'DDT-sha256-zap-duplicate: 4 entries'"
+
+# now two DDT ZAPs in the MOS; DDT ZAPs aren't cleaned up until the entire
+# logical table is destroyed
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256-zap- | wc -l) -eq 2
+
+# remove the files
+log_must rm -f /$TESTPOOL/file*
+log_must zpool sync
+
+# feature should still be disabled
+log_must test $(get_pool_prop feature@fast_dedup $TESTPOOL) = "disabled"
+
+# all DDTs empty
+log_must eval "zdb -D $TESTPOOL | grep -q 'All DDTs are empty'"
+
+# logical table now destroyed; all DDT ZAPs removed
+log_must test $(zdb -dddd $TESTPOOL 1 | grep DDT-sha256-zap- | wc -l) -eq 0
+
+log_pass "basic dedup (legacy) operations work"