Add API and CLI commands to promote/demote nodes in the Raft cluster …

…(#996) (#1072)

* add commands to promote and demote raft peers

Building on top of the new non-voter feature, this allows controlling
the voting status of individual nodes using the CLI.



* document new commands and api endpoints



* remove DR Token options

This is not supported by OpenBao and probably never will be.



* add fallback if autopilot is disabled

Use the raw Raft backend functions to promote or demote a node.



---------

Signed-off-by: Jan Martens <[email protected]>

command/commands.go

@@ -404,6 +404,16 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) map[string]cli.Co
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"operator raft promote": func() (cli.Command, error) {
+			return &OperatorRaftPromoteCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
+		"operator raft demote": func() (cli.Command, error) {
+			return &OperatorRaftDemoteCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"operator raft snapshot": func() (cli.Command, error) {
 			return &OperatorRaftSnapshotCommand{
 				BaseCommand: getBaseCommand(),

command/operator_raft_demote.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2025 OpenBao a Series of LF Projects, LLC
+// SPDX-License-Identifier: MPL-2.0
+
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/cli"
+	"github.com/posener/complete"
+)
+
+var (
+	_ cli.Command             = (*OperatorRaftDemoteCommand)(nil)
+	_ cli.CommandAutocomplete = (*OperatorRaftDemoteCommand)(nil)
+)
+
+type OperatorRaftDemoteCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorRaftDemoteCommand) Synopsis() string {
+	return "Demotes a voter to a permanent non-voter"
+}
+
+func (c *OperatorRaftDemoteCommand) Help() string {
+	helpText := `
+Usage: bao operator raft demote <server_id>
+
+  Demotes voter to a permanent non-voter.
+
+	  $ bao operator raft demote node1
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftDemoteCommand) Flags() *FlagSets {
+	return c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+}
+
+func (c *OperatorRaftDemoteCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftDemoteCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftDemoteCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	serverID := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		serverID = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(serverID) == 0 {
+		c.UI.Error("Server id is required")
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	_, err = client.Logical().Write("sys/storage/raft/demote", map[string]interface{}{
+		"server_id": serverID,
+	})
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error promoting server: %s", err))
+		return 2
+	}
+
+	c.UI.Output("Server demoted successfully!")
+
+	return 0
+}

command/operator_raft_promote.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2025 OpenBao a Series of LF Projects, LLC
+// SPDX-License-Identifier: MPL-2.0
+
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/cli"
+	"github.com/posener/complete"
+)
+
+var (
+	_ cli.Command             = (*OperatorRaftPromoteCommand)(nil)
+	_ cli.CommandAutocomplete = (*OperatorRaftPromoteCommand)(nil)
+)
+
+type OperatorRaftPromoteCommand struct {
+	*BaseCommand
+}
+
+func (c *OperatorRaftPromoteCommand) Synopsis() string {
+	return "Promotes a permanent non-voter to a voter"
+}
+
+func (c *OperatorRaftPromoteCommand) Help() string {
+	helpText := `
+Usage: bao operator raft promote <server_id>
+
+  Promotes a permanent non-voter to a voter.
+
+	  $ bao operator raft promote node1
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorRaftPromoteCommand) Flags() *FlagSets {
+	return c.flagSet(FlagSetHTTP | FlagSetOutputFormat)
+}
+
+func (c *OperatorRaftPromoteCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictAnything
+}
+
+func (c *OperatorRaftPromoteCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *OperatorRaftPromoteCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	serverID := ""
+
+	args = f.Args()
+	switch len(args) {
+	case 1:
+		serverID = strings.TrimSpace(args[0])
+	default:
+		c.UI.Error(fmt.Sprintf("Incorrect arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	if len(serverID) == 0 {
+		c.UI.Error("Server id is required")
+		return 1
+	}
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	_, err = client.Logical().Write("sys/storage/raft/promote", map[string]interface{}{
+		"server_id": serverID,
+	})
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error promoting server: %s", err))
+		return 2
+	}
+
+	c.UI.Output("Server promoted successfully!")
+
+	return 0
+}

physical/raft/raft.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -1232,6 +1233,115 @@ func (b *RaftBackend) RemovePeer(ctx context.Context, peerID string) error {
 	return b.autopilot.RemoveServer(raft.ServerID(peerID))
 }
 
+// PromotePeer promotes a permanent non-voter to voter
+func (b *RaftBackend) PromotePeer(ctx context.Context, peerID string) error {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	if b.disableAutopilot {
+		if b.raft == nil {
+			return errors.New("raft storage is not initialized")
+		}
+		peers, err := b.Peers(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to get Raft peers: %s", err)
+		}
+
+		found := false
+		addr := ""
+		for _, peer := range peers {
+			if peer.ID == peerID {
+				addr = peer.Address
+				found = true
+				break
+			}
+		}
+		if !found {
+			return fmt.Errorf("server %s not found in raft configuration", peerID)
+		}
+
+		future := b.raft.AddVoter(raft.ServerID(peerID), raft.ServerAddress(addr), 0, 0)
+		if err := future.Error(); err != nil {
+			return fmt.Errorf("failed to promote non-voter to voter: %s", err)
+		}
+		return nil
+	}
+
+	if b.autopilot == nil {
+		return errors.New("raft storage autopilot is not initialized")
+	}
+
+	if !b.delegate.IsNonVoter(raft.ServerID(peerID)) {
+		return errors.New("server is not a non-voter")
+	}
+
+	b.logger.Trace("promoting non-voter to voter", "id", peerID)
+	return b.delegate.RemoveNonVoter(raft.ServerID(peerID))
+}
+
+// DemotePeer demotes a voter to a permanent non-voter
+func (b *RaftBackend) DemotePeer(ctx context.Context, peerID string) error {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	if b.disableAutopilot {
+		if b.raft == nil {
+			return errors.New("raft storage is not initialized")
+		}
+
+		// refuse to demote current leader to not trigger a leader election
+		// when the leader is demoted. This is not necessary if autopilot is enabled,
+		// as it will handle this case for us and only demote the leader after a
+		// leader election
+		if strings.EqualFold(peerID, b.localID) {
+			return errors.New("refusing to demote current leader")
+		}
+
+		peers, err := b.Peers(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to get Raft peers: %s", err)
+		}
+
+		found := false
+		for _, peer := range peers {
+			if peer.ID == peerID {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return fmt.Errorf("server %s not found in raft configuration", peerID)
+		}
+
+		future := b.raft.DemoteVoter(raft.ServerID(peerID), 0, 0)
+		if err := future.Error(); err != nil {
+			return fmt.Errorf("failed to demote voter to non-voter: %s", err)
+		}
+		return nil
+	}
+
+	if b.autopilot == nil {
+		return errors.New("raft storage autopilot is not initialized")
+	}
+
+	b.logger.Trace("demoting voter to non-voter", "id", peerID)
+
+	if b.delegate.IsNonVoter(raft.ServerID(peerID)) {
+		return errors.New("server is already a non-voter")
+	}
+
+	return b.delegate.AddNonVoter(raft.ServerID(peerID))
+}
+
 // GetConfigurationOffline is used to read the stale, last known raft
 // configuration to this node. It accesses the last state written into the
 // FSM. When a server is online use GetConfiguration instead.

physical/raft/raft_autopilot_promoter.go

@@ -43,9 +43,14 @@ func (_ *CustomPromoter) CalculatePromotionsAndDemotions(c *autopilot.Config, s
 	minStableDuration := s.ServerStabilizationTime(c)
 	nonVoters := c.Ext.(map[raft.ServerID]bool)
 	for id, server := range s.Servers {
-		// Ignore non-voters
 		if _, ok := nonVoters[id]; ok {
-			continue
+			// If the server is marked as a non-voter, demote it
+			if server.State == autopilot.RaftVoter {
+				changes.Demotions = append(changes.Demotions, id)
+			} else {
+				// If the server is already a non-voter, skip it
+				continue
+			}
 		}
 		// If the server is healthy and stable, promote it to a voter
 		if server.State == autopilot.RaftNonVoter && server.Health.IsStable(now, minStableDuration) {