multisocket plugin (coredns#6882)

* multisocket plugin improves performance in multiprocessor systems Signed-off-by: Viktor Rodionov <[email protected]> * - refactoring - update doc Signed-off-by: Viktor Rodionov <[email protected]> * remove port from reuseport plugin README Signed-off-by: Viktor Rodionov <[email protected]> * rename reuseport plugin to numsockets plugin Signed-off-by: Viktor Rodionov <[email protected]> * Add Recommendations to numsockets README Signed-off-by: Viktor Rodionov <[email protected]> * added numsockets test; made NUM_SOCKETS mandatory in doc Signed-off-by: Viktor Rodionov <[email protected]> * restart and whoami tests for numsockets plugin Signed-off-by: Viktor Rodionov <[email protected]> * default value for numsockets Signed-off-by: Viktor Rodionov <[email protected]> * caddy up Signed-off-by: Viktor Rodionov <[email protected]> * add numsockets to plugin.cfg Signed-off-by: Viktor Rodionov <[email protected]> * - rename numsockets plugin to multisocket - default as GOMAXPROCS - update README Signed-off-by: Viktor Rodionov <[email protected]> * resolve conflicts Signed-off-by: Viktor Rodionov <[email protected]> --------- Signed-off-by: Viktor Rodionov <[email protected]>
aks-lts · Nov 13, 2024 · 6c39f4b · 6c39f4b
1 parent 43fdf73
commit 6c39f4b
Show file tree

Hide file tree

Showing 11 changed files with 438 additions and 56 deletions.
diff --git a/core/dnsserver/config.go b/core/dnsserver/config.go
@@ -24,6 +24,10 @@ type Config struct {
 	// The port to listen on.
 	Port string
 
+	// The number of servers that will listen on one port.
+	// By default, one server will be running.
+	NumSockets int
+
 	// Root points to a base directory we find user defined "things".
 	// First consumer is the file plugin to looks for zone files in this place.
 	Root string

diff --git a/core/dnsserver/register.go b/core/dnsserver/register.go
@@ -134,69 +134,23 @@ func (h *dnsContext) InspectServerBlocks(sourceFile string, serverBlocks []caddy
 
 // MakeServers uses the newly-created siteConfigs to create and return a list of server instances.
 func (h *dnsContext) MakeServers() ([]caddy.Server, error) {
-	// Copy the Plugin, ListenHosts and Debug from first config in the block
-	// to all other config in the same block . Doing this results in zones
-	// sharing the same plugin instances and settings as other zones in
-	// the same block.
-	for _, c := range h.configs {
-		c.Plugin = c.firstConfigInBlock.Plugin
-		c.ListenHosts = c.firstConfigInBlock.ListenHosts
-		c.Debug = c.firstConfigInBlock.Debug
-		c.Stacktrace = c.firstConfigInBlock.Stacktrace
-
-		// Fork TLSConfig for each encrypted connection
-		c.TLSConfig = c.firstConfigInBlock.TLSConfig.Clone()
-		c.ReadTimeout = c.firstConfigInBlock.ReadTimeout
-		c.WriteTimeout = c.firstConfigInBlock.WriteTimeout
-		c.IdleTimeout = c.firstConfigInBlock.IdleTimeout
-		c.TsigSecret = c.firstConfigInBlock.TsigSecret
-	}
+	// Copy parameters from first config in the block to all other config in the same block
+	propagateConfigParams(h.configs)
 
 	// we must map (group) each config to a bind address
 	groups, err := groupConfigsByListenAddr(h.configs)
 	if err != nil {
 		return nil, err
 	}
+
 	// then we create a server for each group
 	var servers []caddy.Server
 	for addr, group := range groups {
-		// switch on addr
-		switch tr, _ := parse.Transport(addr); tr {
-		case transport.DNS:
-			s, err := NewServer(addr, group)
-			if err != nil {
-				return nil, err
-			}
-			servers = append(servers, s)
-
-		case transport.TLS:
-			s, err := NewServerTLS(addr, group)
-			if err != nil {
-				return nil, err
-			}
-			servers = append(servers, s)
-
-		case transport.QUIC:
-			s, err := NewServerQUIC(addr, group)
-			if err != nil {
-				return nil, err
-			}
-			servers = append(servers, s)
-
-		case transport.GRPC:
-			s, err := NewServergRPC(addr, group)
-			if err != nil {
-				return nil, err
-			}
-			servers = append(servers, s)
-
-		case transport.HTTPS:
-			s, err := NewServerHTTPS(addr, group)
-			if err != nil {
-				return nil, err
-			}
-			servers = append(servers, s)
+		serversForGroup, err := makeServersForGroup(addr, group)
+		if err != nil {
+			return nil, err
 		}
+		servers = append(servers, serversForGroup...)
 	}
 
 	// For each server config, check for View Filter plugins
@@ -299,6 +253,27 @@ func (h *dnsContext) validateZonesAndListeningAddresses() error {
 	return nil
 }
 
+// propagateConfigParams copies the necessary parameters from first config in the block
+// to all other config in the same block. Doing this results in zones
+// sharing the same plugin instances and settings as other zones in
+// the same block.
+func propagateConfigParams(configs []*Config) {
+	for _, c := range configs {
+		c.Plugin = c.firstConfigInBlock.Plugin
+		c.ListenHosts = c.firstConfigInBlock.ListenHosts
+		c.Debug = c.firstConfigInBlock.Debug
+		c.Stacktrace = c.firstConfigInBlock.Stacktrace
+		c.NumSockets = c.firstConfigInBlock.NumSockets
+
+		// Fork TLSConfig for each encrypted connection
+		c.TLSConfig = c.firstConfigInBlock.TLSConfig.Clone()
+		c.ReadTimeout = c.firstConfigInBlock.ReadTimeout
+		c.WriteTimeout = c.firstConfigInBlock.WriteTimeout
+		c.IdleTimeout = c.firstConfigInBlock.IdleTimeout
+		c.TsigSecret = c.firstConfigInBlock.TsigSecret
+	}
+}
+
 // groupConfigsByListenAddr groups site configs by their listen
 // (bind) address, so sites that use the same listener can be served
 // on the same server instance. The return value maps the listen
@@ -320,6 +295,63 @@ func groupConfigsByListenAddr(configs []*Config) (map[string][]*Config, error) {
 	return groups, nil
 }
 
+// makeServersForGroup creates servers for a specific transport and group.
+// It creates as many servers as specified in the NumSockets configuration.
+// If the NumSockets param is not specified, one server is created by default.
+func makeServersForGroup(addr string, group []*Config) ([]caddy.Server, error) {
+	// that is impossible, but better to check
+	if len(group) == 0 {
+		return nil, fmt.Errorf("no configs for group defined")
+	}
+	// create one server by default if no NumSockets specified
+	numSockets := 1
+	if group[0].NumSockets > 0 {
+		numSockets = group[0].NumSockets
+	}
+
+	var servers []caddy.Server
+	for range numSockets {
+		// switch on addr
+		switch tr, _ := parse.Transport(addr); tr {
+		case transport.DNS:
+			s, err := NewServer(addr, group)
+			if err != nil {
+				return nil, err
+			}
+			servers = append(servers, s)
+
+		case transport.TLS:
+			s, err := NewServerTLS(addr, group)
+			if err != nil {
+				return nil, err
+			}
+			servers = append(servers, s)
+
+		case transport.QUIC:
+			s, err := NewServerQUIC(addr, group)
+			if err != nil {
+				return nil, err
+			}
+			servers = append(servers, s)
+
+		case transport.GRPC:
+			s, err := NewServergRPC(addr, group)
+			if err != nil {
+				return nil, err
+			}
+			servers = append(servers, s)
+
+		case transport.HTTPS:
+			s, err := NewServerHTTPS(addr, group)
+			if err != nil {
+				return nil, err
+			}
+			servers = append(servers, s)
+		}
+	}
+	return servers, nil
+}
+
 // DefaultPort is the default port.
 const DefaultPort = transport.Port
 

diff --git a/core/dnsserver/zdirectives.go b/core/dnsserver/zdirectives.go
@@ -16,6 +16,7 @@ var Directives = []string{
 	"cancel",
 	"tls",
 	"timeouts",
+	"multisocket",
 	"reload",
 	"nsid",
 	"bufsize",

diff --git a/core/plugin/zplugin.go b/core/plugin/zplugin.go
@@ -39,6 +39,7 @@ import (
 	_ "github.com/coredns/coredns/plugin/metadata"
 	_ "github.com/coredns/coredns/plugin/metrics"
 	_ "github.com/coredns/coredns/plugin/minimal"
+	_ "github.com/coredns/coredns/plugin/multisocket"
 	_ "github.com/coredns/coredns/plugin/nsid"
 	_ "github.com/coredns/coredns/plugin/pprof"
 	_ "github.com/coredns/coredns/plugin/ready"

diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/aws/aws-sdk-go v1.55.5
 	github.com/aws/aws-sdk-go-v2/config v1.27.39
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.3
-	github.com/coredns/caddy v1.1.1
+	github.com/coredns/caddy v1.1.2-0.20241029205200-8de985351a98
 	github.com/dnstap/golang-dnstap v0.4.0
 	github.com/expr-lang/expr v1.16.9
 	github.com/farsightsec/golang-framestream v0.3.0

diff --git a/go.sum b/go.sum
@@ -92,8 +92,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
-github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
+github.com/coredns/caddy v1.1.2-0.20241029205200-8de985351a98 h1:c+Epklw9xk6BZ1OFBPWLA2PcL8QalKvl3if8CP9x8uw=
+github.com/coredns/caddy v1.1.2-0.20241029205200-8de985351a98/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=

diff --git a/plugin.cfg b/plugin.cfg
@@ -25,6 +25,7 @@ geoip:geoip
 cancel:cancel
 tls:tls
 timeouts:timeouts
+multisocket:multisocket
 reload:reload
 nsid:nsid
 bufsize:bufsize

diff --git a/plugin/multisocket/README.md b/plugin/multisocket/README.md
@@ -0,0 +1,68 @@
+# multisocket
+
+## Name
+
+*multisocket* - allows to start multiple servers that will listen on one port.
+
+## Description
+
+With *multisocket*, you can define the number of servers that will listen on the same port. The SO_REUSEPORT socket 
+option allows to open multiple listening sockets at the same address and port. In this case, kernel distributes incoming 
+connections between sockets.
+
+Enabling this option allows to start multiple servers, which increases the throughput of CoreDNS in environments with a 
+large number of CPU cores.
+
+## Syntax
+
+~~~
+multisocket [NUM_SOCKETS]
+~~~
+
+* **NUM_SOCKETS** - the number of servers that will listen on one port. Default value is equal to GOMAXPROCS.
+
+## Examples
+
+Start 5 TCP/UDP servers on the same port.
+
+~~~ corefile
+. {
+	multisocket 5
+	forward . /etc/resolv.conf
+}
+~~~
+
+Do not define `NUM_SOCKETS`, in this case it will take a value equal to GOMAXPROCS.
+
+~~~ corefile
+. {
+	multisocket
+	forward . /etc/resolv.conf
+}
+~~~
+
+## Recommendations
+
+The tests of the `multisocket` plugin, which were conducted for `NUM_SOCKETS` from 1 to 10, did not reveal any side 
+effects or performance degradation.
+
+This means that the `multisocket` plugin can be used with a default value that is equal to GOMAXPROCS.
+
+However, to achieve the best results, it is recommended to consider the specific environment and plugins used in 
+CoreDNS. To determine the optimal configuration, it is advisable to conduct performance tests with different 
+`NUM_SOCKETS`, measuring Queries Per Second (QPS) and system load.
+
+If conducting such tests is difficult, follow these recommendations:
+1. Determine the maximum CPU consumption of CoreDNS server without `multisocket` plugin. Estimate how much CPU CoreDNS
+   actually consumes in specific environment under maximum load.
+2. Align `NUM_SOCKETS` with the estimated CPU usage and CPU limits or system's available resources.
+   Examples:
+   - If CoreDNS consumes 4 CPUs and 8 CPUs are available, set `NUM_SOCKETS` to 2.
+   - If CoreDNS consumes 8 CPUs and 64 CPUs are available, set `NUM_SOCKETS` to 8.
+
+## Limitations
+
+The SO_REUSEPORT socket option is not available for some operating systems. It is available since Linux Kernel 3.9 and 
+not available for Windows at all.
+
+Using this plugin with a system that does not support SO_REUSEPORT will cause an `address already in use` error.
diff --git a/plugin/multisocket/multisocket.go b/plugin/multisocket/multisocket.go
@@ -0,0 +1,51 @@
+package multisocket
+
+import (
+	"fmt"
+	"runtime"
+	"strconv"
+
+	"github.com/coredns/caddy"
+	"github.com/coredns/coredns/core/dnsserver"
+	"github.com/coredns/coredns/plugin"
+)
+
+const pluginName = "multisocket"
+
+func init() { plugin.Register(pluginName, setup) }
+
+func setup(c *caddy.Controller) error {
+	err := parseNumSockets(c)
+	if err != nil {
+		return plugin.Error(pluginName, err)
+	}
+	return nil
+}
+
+func parseNumSockets(c *caddy.Controller) error {
+	config := dnsserver.GetConfig(c)
+	c.Next() // "multisocket"
+
+	args := c.RemainingArgs()
+
+	if len(args) > 1 || c.Next() {
+		return c.ArgErr()
+	}
+
+	if len(args) == 0 {
+		// Nothing specified; use default that is equal to GOMAXPROCS.
+		config.NumSockets = runtime.GOMAXPROCS(0)
+		return nil
+	}
+
+	numSockets, err := strconv.Atoi(args[0])
+	if err != nil {
+		return fmt.Errorf("invalid num sockets: %w", err)
+	}
+	if numSockets < 1 {
+		return fmt.Errorf("num sockets can not be zero or negative: %d", numSockets)
+	}
+	config.NumSockets = numSockets
+
+	return nil
+}