From e6137f715802e4bceb256bfdd9d3e8bfe3252e6f Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Wed, 8 Jan 2025 17:14:33 +1100 Subject: [PATCH] zinject: add "probe" device injection type Injecting a device probe failure is not possible by matching IO types, because probe IO goes to the label regions, which is explicitly excluded from injection. Even if it were possible, it would be awkward to do, because a probe is sequence of reads and writes. This commit adds a new IO "type" to match for injection, which looks for the ZIO_FLAG_PROBE flag instead. Any probe IO will be match the injection record and recieve the wanted error. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris --- cmd/zinject/zinject.c | 1 + include/sys/zfs_ioctl.h | 3 +- man/man8/zinject.8 | 23 +++---- module/zfs/zio_inject.c | 10 ++- tests/runfiles/common.run | 2 +- tests/zfs-tests/tests/Makefile.am | 1 + .../cli_root/zinject/zinject_probe.ksh | 63 +++++++++++++++++++ 7 files changed, 88 insertions(+), 15 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index d66b0d986f2d..4374e69a7f94 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -251,6 +251,7 @@ static const char *const iotypestrtable[ZINJECT_IOTYPES] = { [ZINJECT_IOTYPE_FLUSH] = "flush", [ZINJECT_IOTYPE_TRIM] = "trim", [ZINJECT_IOTYPE_ALL] = "all", + [ZINJECT_IOTYPE_PROBE] = "probe", }; static zinject_iotype_t diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index 9afe984e1749..a8c3ffc76455 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -471,7 +471,8 @@ typedef enum zinject_iotype { ZINJECT_IOTYPE_TRIM = ZIO_TYPE_TRIM, ZINJECT_IOTYPE_ALL = ZIO_TYPES, /* Room for future expansion for ZIO_TYPE_* */ - ZINJECT_IOTYPES = 16, + ZINJECT_IOTYPE_PROBE = 16, + ZINJECT_IOTYPES, } zinject_iotype_t; typedef struct zfs_share { diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index abccc4d086e0..53461681bb1d 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -19,11 +19,11 @@ .\" CDDL HEADER END .\" .\" Copyright 2013 Darik Horn . All rights reserved. -.\" Copyright (c) 2024, Klara Inc. +.\" Copyright (c) 2024, 2025, Klara, Inc. .\" .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS .\" -.Dd December 2, 2024 +.Dd January 14, 2025 .Dt ZINJECT 8 .Os . @@ -265,15 +265,16 @@ will be translated to the appropriate blkid range according to the object's properties. .It Fl s Ar seconds Run for this many seconds before reporting failure. -.It Fl T Ar failure -Set the failure type to one of -.Sy all , -.Sy flush , -.Sy claim , -.Sy free , -.Sy read , -or -.Sy write . +.It Fl T Ar type +Inject the error into I/O of this type. +.Bl -tag -compact -width "read, write, flush, claim, free" +.It Sy read , Sy write , Sy flush , Sy claim , Sy free +Fundamental I/O types +.It Sy all +All fundamental I/O types +.It Sy probe +Device probe I/O +.El .It Fl t Ar mos_type Set this to .Bl -tag -compact -width "spacemap" diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index 6848d46f73e7..f90044299cef 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -386,6 +386,10 @@ zio_match_iotype(zio_t *zio, uint32_t iotype) if (iotype >= ZINJECT_IOTYPES) return (B_FALSE); + /* Probe IOs only match IOTYPE_PROBE, regardless of their type. */ + if (zio->io_flags & ZIO_FLAG_PROBE) + return (iotype == ZINJECT_IOTYPE_PROBE); + /* Standard IO types, match against ZIO type. */ if (iotype < ZINJECT_IOTYPE_ALL) return (iotype == zio->io_type); @@ -405,9 +409,11 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) /* * We skip over faults in the labels unless it's during device open - * (i.e. zio == NULL) or a device flush (offset is meaningless) + * (i.e. zio == NULL) or a device flush (offset is meaningless). We let + * probe IOs through so we can match them to probe inject records. */ - if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) { + if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH && + !(zio->io_flags & ZIO_FLAG_PROBE)) { uint64_t offset = zio->io_offset; if (offset < VDEV_LABEL_START_SIZE || diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 9e186de37369..e2edfc9ebbb5 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -159,7 +159,7 @@ tests = ['json_sanity'] tags = ['functional', 'cli_root', 'json'] [tests/functional/cli_root/zinject] -tests = ['zinject_args', 'zinject_counts'] +tests = ['zinject_args', 'zinject_counts', 'zinject_probe'] pre = post = tags = ['functional', 'cli_root', 'zinject'] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 044a70d1998b..d0eb4c30db48 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/json/json_sanity.ksh \ functional/cli_root/zinject/zinject_args.ksh \ functional/cli_root/zinject/zinject_counts.ksh \ + functional/cli_root/zinject/zinject_probe.ksh \ functional/cli_root/zdb/zdb_002_pos.ksh \ functional/cli_root/zdb/zdb_003_pos.ksh \ functional/cli_root/zdb/zdb_004_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh new file mode 100755 index 000000000000..caa61e86971d --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh @@ -0,0 +1,63 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +log_assert "Check zinject can correctly inject a probe failure." + +DISK1=${DISKS%% *} + +function cleanup +{ + zinject -c all + default_cleanup_noexit +} + +log_onexit cleanup + +zpool create $TESTPOOL $DISK1 + +log_must zinject -d $DISK1 -e io -T probe $TESTPOOL +log_must zinject -d $DISK1 -e io -T write $TESTPOOL + +log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1 + +log_note "waiting for pool to suspend" +typeset -i tries=10 +until [[ $(cat /proc/spl/kstat/zfs/$TESTPOOL/state) == "SUSPENDED" ]] ; do + if ((tries-- == 0)); then + log_fail "pool didn't suspend" + fi + sleep 1 +done + +log_must zinject -c all +log_must zpool clear $TESTPOOL +log_must zpool destroy -f $TESTPOOL + +log_pass "zinject can correctly inject a probe failure."