From 39f89b35b8060d557a4f2617ca74b0c1b70c8deb Mon Sep 17 00:00:00 2001 From: Swagat Bora Date: Fri, 28 Feb 2025 06:41:07 +0000 Subject: [PATCH] Add support for remaining blkio settings Signed-off-by: Swagat Bora --- cmd/nerdctl/container/container_create.go | 29 +- cmd/nerdctl/container/container_run.go | 10 +- .../container_run_cgroup_linux_test.go | 122 +++++++ docs/command-reference.md | 2 +- pkg/api/types/container_types.go | 17 +- pkg/cmd/container/run_blkio_linux.go | 344 ++++++++++++++++++ pkg/cmd/container/run_cgroup_linux.go | 21 +- pkg/infoutil/infoutil.go | 40 +- 8 files changed, 557 insertions(+), 28 deletions(-) create mode 100644 pkg/cmd/container/run_blkio_linux.go diff --git a/cmd/nerdctl/container/container_create.go b/cmd/nerdctl/container/container_create.go index 5b6de8bd97a..85f5937901b 100644 --- a/cmd/nerdctl/container/container_create.go +++ b/cmd/nerdctl/container/container_create.go @@ -199,19 +199,42 @@ func processContainerCreateOptions(cmd *cobra.Command) (types.ContainerCreateOpt if err != nil { return opt, err } + opt.Cgroupns, err = cmd.Flags().GetString("cgroupns") + if err != nil { + return opt, err + } + opt.CgroupParent, err = cmd.Flags().GetString("cgroup-parent") + if err != nil { + return opt, err + } + opt.Device, err = cmd.Flags().GetStringSlice("device") + if err != nil { + return opt, err + } + // #endregion + + // #region for blkio flags opt.BlkioWeight, err = cmd.Flags().GetUint16("blkio-weight") if err != nil { return opt, err } - opt.Cgroupns, err = cmd.Flags().GetString("cgroupns") + opt.BlkioWeightDevice, err = cmd.Flags().GetStringArray("blkio-weight-device") if err != nil { return opt, err } - opt.CgroupParent, err = cmd.Flags().GetString("cgroup-parent") + opt.BlkioDeviceReadBps, err = cmd.Flags().GetStringArray("device-read-bps") if err != nil { return opt, err } - opt.Device, err = cmd.Flags().GetStringSlice("device") + opt.BlkioDeviceWriteBps, err = cmd.Flags().GetStringArray("device-write-bps") + if err != nil { + return opt, err + } + opt.BlkioDeviceReadIOps, err = cmd.Flags().GetStringArray("device-read-iops") + if err != nil { + return opt, err + } + opt.BlkioDeviceWriteIOps, err = cmd.Flags().GetStringArray("device-write-iops") if err != nil { return opt, err } diff --git a/cmd/nerdctl/container/container_run.go b/cmd/nerdctl/container/container_run.go index 361a3e0d0d9..7852a8c894c 100644 --- a/cmd/nerdctl/container/container_run.go +++ b/cmd/nerdctl/container/container_run.go @@ -155,7 +155,6 @@ func setCreateFlags(cmd *cobra.Command) { }) cmd.Flags().Int64("pids-limit", -1, "Tune container pids limit (set -1 for unlimited)") cmd.Flags().StringSlice("cgroup-conf", nil, "Configure cgroup v2 (key=value)") - cmd.Flags().Uint16("blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)") cmd.Flags().String("cgroupns", defaults.CgroupnsMode(), `Cgroup namespace to use, the default depends on the cgroup version ("host"|"private")`) cmd.Flags().String("cgroup-parent", "", "Optional parent cgroup for the container") cmd.RegisterFlagCompletionFunc("cgroupns", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -173,6 +172,15 @@ func setCreateFlags(cmd *cobra.Command) { cmd.Flags().String("rdt-class", "", "Name of the RDT class (or CLOS) to associate the container with") // #endregion + // #region blkio flags + cmd.Flags().Uint16("blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)") + cmd.Flags().StringArray("blkio-weight-device", []string{}, "Block IO weight (relative device weight) (default [])") + cmd.Flags().StringArray("device-read-bps", []string{}, "Limit read rate (bytes per second) from a device (default [])") + cmd.Flags().StringArray("device-read-iops", []string{}, "Limit read rate (IO per second) from a device (default [])") + cmd.Flags().StringArray("device-write-bps", []string{}, "Limit write rate (bytes per second) to a device (default [])") + cmd.Flags().StringArray("device-write-iops", []string{}, "Limit write rate (IO per second) to a device (default [])") + // #endregion + // user flags cmd.Flags().StringP("user", "u", "", "Username or UID (format: [:])") cmd.Flags().String("umask", "", "Set the umask inside the container. Defaults to 0022") diff --git a/cmd/nerdctl/container/container_run_cgroup_linux_test.go b/cmd/nerdctl/container/container_run_cgroup_linux_test.go index 287d372908e..2d67ad35654 100644 --- a/cmd/nerdctl/container/container_run_cgroup_linux_test.go +++ b/cmd/nerdctl/container/container_run_cgroup_linux_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/moby/sys/userns" + "github.com/opencontainers/runtime-spec/specs-go" "gotest.tools/v3/assert" "github.com/containerd/cgroups/v3" @@ -421,3 +422,124 @@ func TestRunBlkioWeightCgroupV2(t *testing.T) { base.Cmd("update", containerName, "--blkio-weight", "400").AssertOK() base.Cmd("exec", containerName, "cat", "io.bfq.weight").AssertOutExactly("default 400\n") } + +func TestRunBlkioSettingCgroupV2(t *testing.T) { + t.Parallel() + testutil.DockerIncompatible(t) + if cgroups.Mode() != cgroups.Unified { + t.Skip("test requires cgroup v2") + } + + // TODO: Fix logic to check if bfq is set as the scheduler for the block device + // + // if _, err := os.Stat("/sys/module/bfq"); err != nil { + // t.Skipf("test requires \"bfq\" module to be loaded: %v", err) + // } + + base := testutil.NewBase(t) + info := base.Info() + switch info.CgroupDriver { + case "none", "": + t.Skip("test requires cgroup driver") + } + + tests := []struct { + name string + args []string + validate func(t *testing.T, blockIO *specs.LinuxBlockIO) + }{ + { + name: "blkio-weight", + args: []string{"--blkio-weight", "150"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, uint16(150), *blockIO.Weight) + }, + }, + { + name: "blkio-weight-device", + args: []string{"--blkio-weight-device", "/dev/sda:100"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, 1, len(blockIO.WeightDevice)) + assert.Equal(t, uint16(100), *blockIO.WeightDevice[0].Weight) + }, + }, + { + name: "device-read-bps", + args: []string{"--device-read-bps", "/dev/sda:1048576"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, 1, len(blockIO.ThrottleReadBpsDevice)) + assert.Equal(t, uint64(1048576), blockIO.ThrottleReadBpsDevice[0].Rate) + }, + }, + { + name: "device-write-bps", + args: []string{"--device-write-bps", "/dev/sda:2097152"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, 1, len(blockIO.ThrottleWriteBpsDevice)) + assert.Equal(t, uint64(2097152), blockIO.ThrottleWriteBpsDevice[0].Rate) + }, + }, + { + name: "device-read-iops", + args: []string{"--device-read-iops", "/dev/sda:1000"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, 1, len(blockIO.ThrottleReadIOPSDevice)) + assert.Equal(t, uint64(1000), blockIO.ThrottleReadIOPSDevice[0].Rate) + }, + }, + { + name: "device-write-iops", + args: []string{"--device-write-iops", "/dev/sda:2000"}, + validate: func(t *testing.T, blockIO *specs.LinuxBlockIO) { + assert.Equal(t, 1, len(blockIO.ThrottleWriteIOPSDevice)) + assert.Equal(t, uint64(2000), blockIO.ThrottleWriteIOPSDevice[0].Rate) + }, + }, + } + + for _, tt := range tests { + tt := tt // capture range variable + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + containerName := testutil.Identifier(t) + + args := []string{"run", "-d", "--name", containerName} + args = append(args, tt.args...) + args = append(args, testutil.AlpineImage, "sleep", "infinity") + base.Cmd(args...).AssertOK() + t.Cleanup(func() { + base.Cmd("rm", "-f", containerName).Run() + }) + + // Connect to containerd + addr := base.ContainerdAddress() + client, err := containerd.New(addr, containerd.WithDefaultNamespace(testutil.Namespace)) + assert.NilError(t, err) + ctx := context.Background() + + // Get container ID + var cid string + walker := &containerwalker.ContainerWalker{ + Client: client, + OnFound: func(ctx context.Context, found containerwalker.Found) error { + if found.MatchCount > 1 { + return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req) + } + cid = found.Container.ID() + return nil + }, + } + err = walker.WalkAll(ctx, []string{containerName}, true) + assert.NilError(t, err) + + // Get container spec + container, err := client.LoadContainer(ctx, cid) + assert.NilError(t, err) + spec, err := container.Spec(ctx) + assert.NilError(t, err) + + // Run the validation function + tt.validate(t, spec.Linux.Resources.BlockIO) + }) + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index 41fcd969fd6..a4fa490d968 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -414,7 +414,7 @@ IPFS flags: - :nerd_face: `--ipfs-address`: Multiaddr of IPFS API (default uses `$IPFS_PATH` env variable if defined or local directory `~/.ipfs`) Unimplemented `docker run` flags: - `--blkio-weight-device`, `--cpu-rt-*`, `--device-*`, + `--cpu-rt-*`, `--device-cgroup-rule`, `--disable-content-trust`, `--expose`, `--health-*`, `--isolation`, `--no-healthcheck`, `--link*`, `--publish-all`, `--storage-opt`, `--userns`, `--volume-driver` diff --git a/pkg/api/types/container_types.go b/pkg/api/types/container_types.go index 3d0ae773ce1..5325915631d 100644 --- a/pkg/api/types/container_types.go +++ b/pkg/api/types/container_types.go @@ -140,8 +140,6 @@ type ContainerCreateOptions struct { PidsLimit int64 // CgroupConf specifies to configure cgroup v2 (key=value) CgroupConf []string - // BlkioWeight specifies the block IO (relative weight), between 10 and 1000, or 0 to disable (default 0) - BlkioWeight uint16 // Cgroupns specifies the cgroup namespace to use Cgroupns string // CgroupParent specifies the optional parent cgroup for the container @@ -150,6 +148,21 @@ type ContainerCreateOptions struct { Device []string // #endregion + // #region for blkio related flags + // BlkioWeight specifies the block IO (relative weight), between 10 and 1000, or 0 to disable (default 0) + BlkioWeight uint16 + // BlkioWeightDevice specifies the Block IO weight (relative device weight) + BlkioWeightDevice []string + // BlkioDeviceReadBps specifies the Block IO read rate limit(bytes per second) of a device + BlkioDeviceReadBps []string + // BlkioDeviceWriteBps specifies the Block IO write rate limit(bytes per second) of a device + BlkioDeviceWriteBps []string + // BlkioDeviceReadIOps specifies the Block IO read rate limit(IO per second) of a device + BlkioDeviceReadIOps []string + // BlkioDeviceWriteIOps specifies the Block IO read rate limit(IO per second) of a device + BlkioDeviceWriteIOps []string + // #endregion + // #region for intel RDT flags // RDTClass specifies the Intel Resource Director Technology (RDT) class RDTClass string diff --git a/pkg/cmd/container/run_blkio_linux.go b/pkg/cmd/container/run_blkio_linux.go new file mode 100644 index 00000000000..ce4cb31fbb2 --- /dev/null +++ b/pkg/cmd/container/run_blkio_linux.go @@ -0,0 +1,344 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package container + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/docker/go-units" + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/log" + + "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/infoutil" +) + +// WeightDevice is a structure that holds device:weight pair +type WeightDevice struct { + Path string + Weight uint16 +} + +func (w *WeightDevice) String() string { + return fmt.Sprintf("%s:%d", w.Path, w.Weight) +} + +// ThrottleDevice is a structure that holds device:rate_per_second pair +type ThrottleDevice struct { + Path string + Rate uint64 +} + +func (t *ThrottleDevice) String() string { + return fmt.Sprintf("%s:%d", t.Path, t.Rate) +} + +func toOCIWeightDevices(weightDevices []*WeightDevice) ([]specs.LinuxWeightDevice, error) { + var stat unix.Stat_t + blkioWeightDevices := make([]specs.LinuxWeightDevice, 0, len(weightDevices)) + + for _, weightDevice := range weightDevices { + if err := unix.Stat(weightDevice.Path, &stat); err != nil { + return nil, fmt.Errorf("failed to stat %s: %w", weightDevice.Path, err) + } + weight := weightDevice.Weight + d := specs.LinuxWeightDevice{Weight: &weight} + // The type is 32bit on mips. + d.Major = int64(unix.Major(uint64(stat.Rdev))) //nolint: unconvert + d.Minor = int64(unix.Minor(uint64(stat.Rdev))) //nolint: unconvert + blkioWeightDevices = append(blkioWeightDevices, d) + } + + return blkioWeightDevices, nil +} + +func toOCIThrottleDevices(devs []*ThrottleDevice) ([]specs.LinuxThrottleDevice, error) { + var stat unix.Stat_t + throttleDevices := make([]specs.LinuxThrottleDevice, 0, len(devs)) + + for _, d := range devs { + if err := unix.Stat(d.Path, &stat); err != nil { + return nil, fmt.Errorf("failed to stat %s: %w", d.Path, err) + } + d := specs.LinuxThrottleDevice{Rate: d.Rate} + // the type is 32bit on mips + d.Major = int64(unix.Major(uint64(stat.Rdev))) //nolint: unconvert + d.Minor = int64(unix.Minor(uint64(stat.Rdev))) //nolint: unconvert + throttleDevices = append(throttleDevices, d) + } + + return throttleDevices, nil +} + +func withBlkioWeight(blkioWeight uint16) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.Weight = &blkioWeight + return nil + } +} + +func withBlkioWeightDevice(weightDevices []specs.LinuxWeightDevice) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.WeightDevice = weightDevices + return nil + } +} + +func withBlkioReadBpsDevice(devices []specs.LinuxThrottleDevice) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.ThrottleReadBpsDevice = devices + return nil + } +} + +func withBlkioWriteBpsDevice(devices []specs.LinuxThrottleDevice) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.ThrottleWriteBpsDevice = devices + return nil + } +} + +func withBlkioReadIOPSDevice(devices []specs.LinuxThrottleDevice) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.ThrottleReadIOPSDevice = devices + return nil + } +} + +func withBlkioWriteIOPSDevice(devices []specs.LinuxThrottleDevice) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + if s.Linux.Resources.BlockIO == nil { + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{} + } + s.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice = devices + return nil + } +} + +func BlkioOCIOpts(options types.ContainerCreateOptions) ([]oci.SpecOpts, error) { + var opts []oci.SpecOpts + + // Handle BlkioWeight + if options.BlkioWeight != 0 { + if !infoutil.BlockIOWeight(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio weight missing, weight discarded") + } else { + if options.BlkioWeight < 10 || options.BlkioWeight > 1000 { + return nil, errors.New("range of blkio weight is from 10 to 1000") + } + opts = append(opts, withBlkioWeight(options.BlkioWeight)) + } + } + + // Handle BlkioWeightDevice + if len(options.BlkioWeightDevice) > 0 { + if !infoutil.BlockIOWeightDevice(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio weight device missing, weight device discarded") + } else { + weightDevices, err := validateWeightDevices(options.BlkioWeightDevice) + if err != nil { + return nil, fmt.Errorf("invalid weight device: %w", err) + } + linuxWeightDevices, err := toOCIWeightDevices(weightDevices) + if err != nil { + return nil, err + } + opts = append(opts, withBlkioWeightDevice(linuxWeightDevices)) + } + } + + // Handle BlockIOReadBpsDevice + if len(options.BlkioDeviceReadBps) > 0 { + log.L.Warn("BlkioDeviceReadBps setting found") + if !infoutil.BlockIOReadBpsDevice(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio read bps device missing, read bps device discarded") + } else { + log.L.Warn("kernel support found") + readBpsDevices, err := validateThrottleBpsDevices(options.BlkioDeviceReadBps) + if err != nil { + return nil, fmt.Errorf("invalid read bps device: %w", err) + } + throttleDevices, err := toOCIThrottleDevices(readBpsDevices) + if err != nil { + return nil, err + } + opts = append(opts, withBlkioReadBpsDevice(throttleDevices)) + } + } + + // Handle BlockIOWriteBpsDevice + if len(options.BlkioDeviceWriteBps) > 0 { + if !infoutil.BlockIOWriteBpsDevice(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio write bps device missing, write bps device discarded") + } else { + writeBpsDevices, err := validateThrottleBpsDevices(options.BlkioDeviceWriteBps) + if err != nil { + return nil, fmt.Errorf("invalid write bps device: %w", err) + } + throttleDevices, err := toOCIThrottleDevices(writeBpsDevices) + if err != nil { + return nil, err + } + opts = append(opts, withBlkioWriteBpsDevice(throttleDevices)) + } + } + + // Handle BlockIOReadIopsDevice + if len(options.BlkioDeviceReadIOps) > 0 { + if !infoutil.BlockIOReadIOpsDevice(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio read iops device missing, read iops device discarded") + } else { + readIopsDevices, err := validateThrottleIOpsDevices(options.BlkioDeviceReadIOps) + if err != nil { + return nil, fmt.Errorf("invalid read iops device: %w", err) + } + throttleDevices, err := toOCIThrottleDevices(readIopsDevices) + if err != nil { + return nil, err + } + opts = append(opts, withBlkioReadIOPSDevice(throttleDevices)) + } + } + + // Handle BlockIOWriteIopsDevice + if len(options.BlkioDeviceWriteIOps) > 0 { + if !infoutil.BlockIOWriteIOpsDevice(options.GOptions.CgroupManager) { + log.L.Warn("kernel support for cgroup blkio write iops device missing, write iops device discarded") + } else { + writeIopsDevices, err := validateThrottleIOpsDevices(options.BlkioDeviceWriteIOps) + if err != nil { + return nil, fmt.Errorf("invalid write iops device: %w", err) + } + throttleDevices, err := toOCIThrottleDevices(writeIopsDevices) + if err != nil { + return nil, err + } + opts = append(opts, withBlkioWriteIOPSDevice(throttleDevices)) + } + } + + return opts, nil +} + +// validateWeightDevices validates an array of device-weight strings +// +// from https://github.com/docker/cli/blob/master/opts/weightdevice.go#L15 +func validateWeightDevices(vals []string) ([]*WeightDevice, error) { + weightDevices := make([]*WeightDevice, 0, len(vals)) + for _, val := range vals { + k, v, ok := strings.Cut(val, ":") + if !ok || k == "" { + return nil, fmt.Errorf("bad format: %s", val) + } + if !strings.HasPrefix(k, "/dev/") { + return nil, fmt.Errorf("bad format for device path: %s", val) + } + weight, err := strconv.ParseUint(v, 10, 16) + if err != nil { + return nil, fmt.Errorf("invalid weight for device: %s", val) + } + if weight > 0 && (weight < 10 || weight > 1000) { + return nil, fmt.Errorf("invalid weight for device: %s", val) + } + + weightDevices = append(weightDevices, &WeightDevice{ + Path: k, + Weight: uint16(weight), + }) + } + return weightDevices, nil +} + +// validateThrottleBpsDevices validates an array of device-rate strings for bytes per second +// +// from https://github.com/docker/cli/blob/master/opts/throttledevice.go#L16 +func validateThrottleBpsDevices(vals []string) ([]*ThrottleDevice, error) { + throttleDevices := make([]*ThrottleDevice, 0, len(vals)) + for _, val := range vals { + k, v, ok := strings.Cut(val, ":") + if !ok || k == "" { + return nil, fmt.Errorf("bad format: %s", val) + } + + if !strings.HasPrefix(k, "/dev/") { + return nil, fmt.Errorf("bad format for device path: %s", val) + } + rate, err := units.RAMInBytes(v) + if err != nil { + return nil, fmt.Errorf("invalid rate for device: %s. The correct format is :[]. Number must be a positive integer. Unit is optional and can be kb, mb, or gb", val) + } + if rate < 0 { + return nil, fmt.Errorf("invalid rate for device: %s. The correct format is :[]. Number must be a positive integer. Unit is optional and can be kb, mb, or gb", val) + } + + throttleDevices = append(throttleDevices, &ThrottleDevice{ + Path: k, + Rate: uint64(rate), + }) + } + return throttleDevices, nil +} + +// validateThrottleIOpsDevices validates an array of device-rate strings for IO operations per second +// +// from https://github.com/docker/cli/blob/master/opts/throttledevice.go#L40 +func validateThrottleIOpsDevices(vals []string) ([]*ThrottleDevice, error) { + throttleDevices := make([]*ThrottleDevice, 0, len(vals)) + for _, val := range vals { + k, v, ok := strings.Cut(val, ":") + if !ok || k == "" { + return nil, fmt.Errorf("bad format: %s", val) + } + + if !strings.HasPrefix(k, "/dev/") { + return nil, fmt.Errorf("bad format for device path: %s", val) + } + rate, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid rate for device: %s. The correct format is :. Number must be a positive integer", val) + } + + throttleDevices = append(throttleDevices, &ThrottleDevice{ + Path: k, + Rate: rate, + }) + } + return throttleDevices, nil +} diff --git a/pkg/cmd/container/run_cgroup_linux.go b/pkg/cmd/container/run_cgroup_linux.go index af43b12c1fd..bdafe0caca3 100644 --- a/pkg/cmd/container/run_cgroup_linux.go +++ b/pkg/cmd/container/run_cgroup_linux.go @@ -179,14 +179,11 @@ func generateCgroupOpts(id string, options types.ContainerCreateOptions) ([]oci. } opts = append(opts, withUnified(unifieds)) - if options.BlkioWeight != 0 && !infoutil.BlockIOWeight(options.GOptions.CgroupManager) { - log.L.Warn("kernel support for cgroup blkio weight missing, weight discarded") - options.BlkioWeight = 0 - } - if options.BlkioWeight > 0 && options.BlkioWeight < 10 || options.BlkioWeight > 1000 { - return nil, errors.New("range of blkio weight is from 10 to 1000") + blkioOpts, err := BlkioOCIOpts(options) + if err != nil { + return nil, err } - opts = append(opts, withBlkioWeight(options.BlkioWeight)) + opts = append(opts, blkioOpts...) switch options.Cgroupns { case "private": @@ -308,16 +305,6 @@ func withUnified(unified map[string]string) oci.SpecOpts { } } -func withBlkioWeight(blkioWeight uint16) oci.SpecOpts { - return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - if blkioWeight == 0 { - return nil - } - s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{Weight: &blkioWeight} - return nil - } -} - func withCustomMemoryResources(memoryOptions customMemoryOptions) oci.SpecOpts { return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { if s.Linux != nil { diff --git a/pkg/infoutil/infoutil.go b/pkg/infoutil/infoutil.go index 2886c9fe049..e151a7b2d51 100644 --- a/pkg/infoutil/infoutil.go +++ b/pkg/infoutil/infoutil.go @@ -26,6 +26,7 @@ import ( "time" "github.com/Masterminds/semver/v3" + "github.com/docker/docker/pkg/sysinfo" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/introspection" @@ -244,13 +245,44 @@ func parseRuncVersion(runcVersionStdout []byte) (*dockercompat.ComponentVersion, }, nil } -// BlockIOWeight return whether Block IO weight is supported or not -func BlockIOWeight(cgroupManager string) bool { +// getMobySysInfo returns the moby system info for the given cgroup manager +func getMobySysInfo(cgroupManager string) *sysinfo.SysInfo { var info dockercompat.Info info.CgroupVersion = CgroupsVersion() info.CgroupDriver = cgroupManager - mobySysInfo := mobySysInfo(&info) + return mobySysInfo(&info) +} + +// BlockIOWeight returns whether Block IO weight is supported or not +func BlockIOWeight(cgroupManager string) bool { + // blkio weight is not available on cgroup v1 since kernel 5.0. + // On cgroup v2, blkio weight is implemented using io.weight + return getMobySysInfo(cgroupManager).BlkioWeight +} + +// BlockIOWeightDevice returns whether Block IO weight device is supported or not +func BlockIOWeightDevice(cgroupManager string) bool { // blkio weight is not available on cgroup v1 since kernel 5.0. // On cgroup v2, blkio weight is implemented using io.weight - return mobySysInfo.BlkioWeight + return getMobySysInfo(cgroupManager).BlkioWeightDevice +} + +// BlockIOReadBpsDevice returns whether Block IO read limit in bytes per second is supported or not +func BlockIOReadBpsDevice(cgroupManager string) bool { + return getMobySysInfo(cgroupManager).BlkioReadBpsDevice +} + +// BlockIOWriteBpsDevice returns whether Block IO write limit in bytes per second is supported or not +func BlockIOWriteBpsDevice(cgroupManager string) bool { + return getMobySysInfo(cgroupManager).BlkioWriteBpsDevice +} + +// BlockIOReadIOpsDevice returns whether Block IO read limit in IO per second is supported or not +func BlockIOReadIOpsDevice(cgroupManager string) bool { + return getMobySysInfo(cgroupManager).BlkioReadIOpsDevice +} + +// BlockIOWriteIOpsDevice returns whether Block IO write limit in IO per second is supported or not +func BlockIOWriteIOpsDevice(cgroupManager string) bool { + return getMobySysInfo(cgroupManager).BlkioWriteIOpsDevice }