Skip to content

Commit

Permalink
Don't use join tokens to bootstrap embedded kubelet
Browse files Browse the repository at this point in the history 
When a controller bootstraps its embedded kubelet, it doesn't have to
use a join token at all. Instead, it can just bootstrap the kubelet
configuration using its own admin kubeconfig.

Add a new KubeconfigGetter argument to the worker start method.
If running from a controller, this will simply point to the admin
kubeconfig. When running as a standalone worker, this will actually be
backed by the join token, if any.

Signed-off-by: Tom Wieczorek <[email protected]>
  • Loading branch information
@twz123
twz123 committed Jan 30, 2025
1 parent 730b82d commit bc75178
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 83 deletions.
44 changes: 8 additions & 36 deletions cmd/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"syscall"
"time"

"github.com/avast/retry-go"
workercmd "github.com/k0sproject/k0s/cmd/worker"
"github.com/k0sproject/k0s/internal/pkg/dir"
"github.com/k0sproject/k0s/internal/pkg/file"
Expand Down Expand Up @@ -61,10 +60,13 @@ import (
"github.com/k0sproject/k0s/pkg/performance"
"github.com/k0sproject/k0s/pkg/telemetry"
"github.com/k0sproject/k0s/pkg/token"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/rest"

"github.com/avast/retry-go"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

type command config.CLIOptions
Expand Down Expand Up @@ -622,7 +624,7 @@ func (c *command) start(ctx context.Context) error {

if c.EnableWorker {
perfTimer.Checkpoint("starting-worker")
if err := c.startWorker(ctx, c.WorkerProfile, nodeConfig); err != nil {
if err := c.startWorker(ctx, c.WorkerProfile); err != nil {
logrus.WithError(err).Error("Failed to start controller worker")
} else {
perfTimer.Checkpoint("started-worker")
Expand All @@ -641,41 +643,11 @@ func (c *command) start(ctx context.Context) error {
return nil
}

func (c *command) startWorker(ctx context.Context, profile string, nodeConfig *v1beta1.ClusterConfig) error {
var bootstrapConfig string
if !file.Exists(c.K0sVars.KubeletAuthConfigPath) {
// wait for controller to start up
err := retry.Do(func() error {
if !file.Exists(c.K0sVars.AdminKubeConfigPath) {
return fmt.Errorf("file does not exist: %s", c.K0sVars.AdminKubeConfigPath)
}
return nil
}, retry.Context(ctx))
if err != nil {
return err
}

err = retry.Do(func() error {
// five minutes here are coming from maximum theoretical duration of kubelet bootstrap process
// we use retry.Do with 10 attempts, back-off delay and delay duration 500 ms which gives us
// 225 seconds here
tokenAge := time.Second * 225
cfg, err := token.CreateKubeletBootstrapToken(ctx, nodeConfig.Spec.API, c.K0sVars, token.RoleWorker, tokenAge)
if err != nil {
return err
}
bootstrapConfig = cfg
return nil
}, retry.Context(ctx))
if err != nil {
return err
}
}
func (c *command) startWorker(ctx context.Context, profile string) error {
// Cast and make a copy of the controller command so it can use the same
// opts to start the worker. Needs to be a copy so the original token and
// possibly other args won't get messed up.
wc := workercmd.Command(*(*config.CLIOptions)(c))
wc.TokenArg = bootstrapConfig
wc.WorkerProfile = profile
wc.Labels = append(wc.Labels, fields.OneTermEqualSelector(constant.K0SNodeRoleLabel, "control-plane").String())
wc.DisableIPTables = true
Expand All @@ -684,7 +656,7 @@ func (c *command) startWorker(ctx context.Context, profile string, nodeConfig *v
taint := fields.OneTermEqualSelector(key, ":NoSchedule")
wc.Taints = append(wc.Taints, taint.String())
}
return wc.Start(ctx)
return wc.Start(ctx, kubernetes.KubeconfigFromFile(c.K0sVars.AdminKubeConfigPath))
}

// If we've got an etcd data directory in place for embedded etcd, or a ca for
Expand Down
78 changes: 73 additions & 5 deletions cmd/worker/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ import (
"github.com/k0sproject/k0s/pkg/component/worker/nllb"
"github.com/k0sproject/k0s/pkg/config"
"github.com/k0sproject/k0s/pkg/kubernetes"
"github.com/k0sproject/k0s/pkg/token"

"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"

"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -75,8 +79,9 @@ func NewWorkerCmd() *cobra.Command {
c.TokenArg = args[0]
}

if c.TokenArg != "" && c.TokenFile != "" {
return errors.New("you can only pass one token argument either as a CLI argument 'k0s worker [token]' or as a flag 'k0s worker --token-file [path]'")
getBootstrapKubeconfig, err := kubeconfigGetterFromJoinToken(c.TokenFile, c.TokenArg)
if err != nil {
return err
}

if err := (&sysinfo.K0sSysinfoSpec{
Expand All @@ -98,7 +103,7 @@ func NewWorkerCmd() *cobra.Command {
logrus.Infof("The file %s is no longer used and can safely be deleted", legacyCAFile)
}

return c.Start(ctx)
return c.Start(ctx, getBootstrapKubeconfig)
},
}

Expand All @@ -110,9 +115,72 @@ func NewWorkerCmd() *cobra.Command {
return cmd
}

func kubeconfigGetterFromJoinToken(tokenFile, tokenArg string) (clientcmd.KubeconfigGetter, error) {
if tokenArg != "" {
if tokenFile != "" {
return nil, errors.New("you can only pass one token argument either as a CLI argument 'k0s worker [token]' or as a flag 'k0s worker --token-file [path]'")
}

kubeconfig, err := loadKubeconfigFromJoinToken(tokenArg)
if err != nil {
return nil, err
}

return func() (*clientcmdapi.Config, error) {
return kubeconfig, nil
}, nil
}

if tokenFile == "" {
return nil, nil
}

return func() (*clientcmdapi.Config, error) {
return loadKubeconfigFromTokenFile(tokenFile)
}, nil
}

func loadKubeconfigFromJoinToken(tokenData string) (*clientcmdapi.Config, error) {
decoded, err := token.DecodeJoinToken(tokenData)
if err != nil {
return nil, fmt.Errorf("failed to decode join token: %w", err)
}

kubeconfig, err := clientcmd.Load(decoded)
if err != nil {
return nil, fmt.Errorf("failed to load kubeconfig from join token: %w", err)
}

if tokenType := token.GetTokenType(kubeconfig); tokenType != "kubelet-bootstrap" {
return nil, fmt.Errorf("wrong token type %s, expected type: kubelet-bootstrap", tokenType)
}

return kubeconfig, nil
}

func loadKubeconfigFromTokenFile(path string) (*clientcmdapi.Config, error) {
var problem string
tokenBytes, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
problem = "not found"
} else if err != nil {
return nil, fmt.Errorf("failed to read token file: %w", err)
} else if len(tokenBytes) == 0 {
problem = "is empty"
}
if problem != "" {
return nil, fmt.Errorf("token file %q %s"+
`: obtain a new token via "k0s token create ..." and store it in the file`+
` or reinstall this node via "k0s install --force ..." or "k0sctl apply --force ..."`,
path, problem)
}

return loadKubeconfigFromJoinToken(string(tokenBytes))
}

// Start starts the worker components based on the given [config.CLIOptions].
func (c *Command) Start(ctx context.Context) error {
if err := worker.BootstrapKubeletClientConfig(ctx, c.K0sVars, &c.WorkerOptions); err != nil {
func (c *Command) Start(ctx context.Context, getBootstrapKubeconfig clientcmd.KubeconfigGetter) error {
if err := worker.BootstrapKubeletClientConfig(ctx, c.K0sVars, &c.WorkerOptions, getBootstrapKubeconfig); err != nil {
return fmt.Errorf("failed to bootstrap kubelet client configuration: %w", err)
}

Expand Down
62 changes: 20 additions & 42 deletions pkg/component/worker/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@ import (
"github.com/k0sproject/k0s/internal/pkg/flags"
"github.com/k0sproject/k0s/pkg/config"
"github.com/k0sproject/k0s/pkg/node"
"github.com/k0sproject/k0s/pkg/token"

apitypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"

"github.com/avast/retry-go"
"github.com/sirupsen/logrus"
)

func BootstrapKubeletClientConfig(ctx context.Context, k0sVars *config.CfgVars, workerOpts *config.WorkerOptions) error {
func BootstrapKubeletClientConfig(ctx context.Context, k0sVars *config.CfgVars, workerOpts *config.WorkerOptions, getBootstrapKubeconfig clientcmd.KubeconfigGetter) error {
// The node name used during bootstrapping needs to match the node name
// selected by kubelet. Otherwise, kubelet will have problems interacting
// with a Node object that doesn't match the name in the certificates.
Expand Down Expand Up @@ -77,50 +77,19 @@ func BootstrapKubeletClientConfig(ctx context.Context, k0sVars *config.CfgVars,
case file.Exists(bootstrapKubeconfigPath):
// Nothing to do here.

// 3: A join token has been given.
// Bootstrap the kubelet kubeconfig via the embedded bootstrap config.
case workerOpts.TokenArg != "" || workerOpts.TokenFile != "":
var tokenData string
if workerOpts.TokenArg != "" {
tokenData = workerOpts.TokenArg
} else {
var problem string
data, err := os.ReadFile(workerOpts.TokenFile)
if errors.Is(err, os.ErrNotExist) {
problem = "not found"
} else if err != nil {
return fmt.Errorf("failed to read token file: %w", err)
} else if len(data) == 0 {
problem = "is empty"
}
if problem != "" {
return fmt.Errorf("token file %q %s"+
`: obtain a new token via "k0s token create ..." and store it in the file`+
` or reinstall this node via "k0s install --force ..." or "k0sctl apply --force ..."`,
workerOpts.TokenFile, problem)
}

tokenData = string(data)
}

// Join token given, so use that.
kubeconfig, err := token.DecodeJoinToken(tokenData)
// 3: A bootstrap kubeconfig can be created (usually via a join token).
// Bootstrap the kubelet kubeconfig via a temporary bootstrap config file.
case getBootstrapKubeconfig != nil:
bootstrapKubeconfig, err := getBootstrapKubeconfig()
if err != nil {
return fmt.Errorf("failed to decode join token: %w", err)
}

// Load the bootstrap kubeconfig to validate it.
if bootstrapKubeconfig, err := clientcmd.Load(kubeconfig); err != nil {
return fmt.Errorf("failed to parse bootstrap kubeconfig from join token: %w", err)
} else if actual := token.GetTokenType(bootstrapKubeconfig); actual != token.WorkerTokenAuthName {
return fmt.Errorf("wrong token type %s, expected type: %s", actual, token.WorkerTokenAuthName)
return fmt.Errorf("failed to get bootstrap kubeconfig: %w", err)
}

// Write the kubelet bootstrap kubeconfig to a temporary file, as the
// kubelet bootstrap API only accepts files.
bootstrapKubeconfigPath, err = writeKubeletBootstrapKubeconfig(kubeconfig)
bootstrapKubeconfigPath, err = writeKubeletBootstrapKubeconfig(*bootstrapKubeconfig)
if err != nil {
return fmt.Errorf("failed to write bootstrap kubeconfig file: %w", err)
return fmt.Errorf("failed to write bootstrap kubeconfig: %w", err)
}

// Ensure that the temporary kubelet bootstrap kubeconfig file will be
Expand Down Expand Up @@ -164,7 +133,16 @@ func BootstrapKubeletClientConfig(ctx context.Context, k0sVars *config.CfgVars,
return nil
}

func writeKubeletBootstrapKubeconfig(kubeconfig []byte) (string, error) {
func writeKubeletBootstrapKubeconfig(kubeconfig clientcmdapi.Config) (string, error) {
if err := clientcmdapi.MinifyConfig(&kubeconfig); err != nil {
return "", fmt.Errorf("failed to minify bootstrap kubeconfig: %w", err)
}

bytes, err := clientcmd.Write(kubeconfig)
if err != nil {
return "", err
}

dir := os.Getenv("XDG_RUNTIME_DIR")
if dir == "" && runtime.GOOS != "windows" {
dir = "/run"
Expand All @@ -175,7 +153,7 @@ func writeKubeletBootstrapKubeconfig(kubeconfig []byte) (string, error) {
return "", err
}

_, writeErr := bootstrapFile.Write(kubeconfig)
_, writeErr := bootstrapFile.Write(bytes)
closeErr := bootstrapFile.Close()

if writeErr != nil || closeErr != nil {
Expand Down

0 comments on commit bc75178

Please sign in to comment.