Skip to content

Commit

Permalink
Merge pull request #42 from banzaicloud/taints
Browse files Browse the repository at this point in the history
Added support to set taints at Node registration
  • Loading branch information
Ecsy authored May 3, 2019
2 parents 8cb7d5d + 48de9a3 commit b299cad
Show file tree
Hide file tree
Showing 15 changed files with 186 additions and 0 deletions.
3 changes: 3 additions & 0 deletions cmd/pke/app/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ const (

// FlagDisableDefaultStorageClass adds default storage class.
FlagDisableDefaultStorageClass = "disable-default-storage-class"

// FlagTaints specifies the taints the Node should be registered with.
FlagTaints = "taints"
)

var (
Expand Down
7 changes: 7 additions & 0 deletions cmd/pke/app/phases/kubeadm/controlplane/controlplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ type ControlPlane struct {
azureExcludeMasterFromStandardLB bool
cidr string
disableDefaultStorageClass bool
taints []string
}

func NewCommand(out io.Writer) *cobra.Command {
Expand Down Expand Up @@ -181,6 +182,8 @@ func (c *ControlPlane) RegisterFlags(flags *pflag.FlagSet) {
flags.String(constants.FlagInfrastructureCIDR, "192.168.64.0/20", "network CIDR for the actual machine")
// Storage class
flags.Bool(constants.FlagDisableDefaultStorageClass, false, "Do not deploy a default storage class")
// Taints
flags.StringSlice(constants.FlagTaints, []string{"node-role.kubernetes.io/master:NoSchedule"}, "Specifies the taints the Node should be registered with")

c.addHAControlPlaneFlags(flags)
}
Expand Down Expand Up @@ -534,6 +537,10 @@ func (c *ControlPlane) masterBootstrapParameters(cmd *cobra.Command) (err error)
return
}
c.disableDefaultStorageClass, err = cmd.Flags().GetBool(constants.FlagDisableDefaultStorageClass)
if err != nil {
return
}
c.taints, err = cmd.Flags().GetStringSlice(constants.FlagTaints)

return
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func TestWriteKubeadmConfig(t *testing.T) {
controllerManagerSigningCA: "/etc/kubernetes/pki/cm-signing-ca.crt",
apiServerCertSANs: []string{"almafa", "vadkorte"},
withPluginPSP: true,
taints: []string{"node-role.kubernetes.io/master:NoSchedule"},
}

err := c.WriteKubeadmConfig(os.Stdout, filename)
Expand Down
16 changes: 16 additions & 0 deletions cmd/pke/app/phases/kubeadm/controlplane/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/Masterminds/semver"
"github.com/banzaicloud/pke/cmd/pke/app/phases/kubeadm"
"github.com/banzaicloud/pke/cmd/pke/app/util/kubernetes"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -88,6 +89,11 @@ func (c ControlPlane) WriteKubeadmConfig(out io.Writer, filename string) error {
}
defer func() { _ = w.Close() }()

taints, err := kubernetes.ParseTaints(c.taints)
if err != nil {
return err
}

type data struct {
APIServerAdvertiseAddress string
APIServerBindPort string
Expand All @@ -107,6 +113,7 @@ func (c ControlPlane) WriteKubeadmConfig(out io.Writer, filename string) error {
ImageRepository string
EncryptionProviderPrefix string
WithPluginPSP bool
Taints []kubernetes.Taint
}

d := data{
Expand All @@ -128,6 +135,7 @@ func (c ControlPlane) WriteKubeadmConfig(out io.Writer, filename string) error {
ImageRepository: c.imageRepository,
EncryptionProviderPrefix: encryptionProviderPrefix,
WithPluginPSP: c.withPluginPSP,
Taints: taints,
}

return tmpl.Execute(w, d)
Expand All @@ -144,6 +152,10 @@ localAPIEndpoint:
bindPort: {{ .APIServerBindPort }}{{end}}
nodeRegistration:
criSocket: "unix:///run/containerd/containerd.sock"
taints:{{if not .Taints}} []{{end}}{{range .Taints}}
- key: "{{.Key}}"
value: "{{.Value}}"
effect: "{{.Effect}}"{{end}}
kubeletExtraArgs:
{{if .Nodepool }}
node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
Expand Down Expand Up @@ -253,6 +265,10 @@ kind: InitConfiguration
bindPort: {{ .APIServerBindPort }}{{end}}
nodeRegistration:
criSocket: "unix:///run/containerd/containerd.sock"
taints:{{if not .Taints}} []{{end}}{{range .Taints}}
- key: "{{.Key}}"
value: "{{.Value}}"
effect: "{{.Effect}}"{{end}}
kubeletExtraArgs:
{{if .Nodepool }}
node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
Expand Down
16 changes: 16 additions & 0 deletions cmd/pke/app/phases/kubeadm/node/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/Masterminds/semver"
"github.com/banzaicloud/pke/cmd/pke/app/phases/kubeadm"
"github.com/banzaicloud/pke/cmd/pke/app/util/kubernetes"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -83,6 +84,11 @@ func (n Node) writeKubeadmConfig(out io.Writer, filename string) error {
}
defer func() { _ = w.Close() }()

taints, err := kubernetes.ParseTaints(n.taints)
if err != nil {
return err
}

type data struct {
APIServerAdvertiseAddress string
APIServerBindPort string
Expand All @@ -91,6 +97,7 @@ func (n Node) writeKubeadmConfig(out io.Writer, filename string) error {
CACertHash string
CloudProvider string
Nodepool string
Taints []kubernetes.Taint
}

d := data{
Expand All @@ -101,6 +108,7 @@ func (n Node) writeKubeadmConfig(out io.Writer, filename string) error {
CACertHash: n.caCertHash,
CloudProvider: n.cloudProvider,
Nodepool: n.nodepool,
Taints: taints,
}

return tmpl.Execute(w, d)
Expand All @@ -117,6 +125,10 @@ controlPlane:
bindPort: {{ .APIServerBindPort }}{{end}}
nodeRegistration:
criSocket: "unix:///run/containerd/containerd.sock"
taints:{{if not .Taints}} []{{end}}{{range .Taints}}
- key: "{{.Key}}"
value: "{{.Value}}"
effect: "{{.Effect}}"{{end}}
kubeletExtraArgs:
{{if .Nodepool }}
node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
Expand Down Expand Up @@ -156,6 +168,10 @@ apiEndpoint:
bindPort: {{ .APIServerBindPort }}{{end}}
nodeRegistration:
criSocket: "unix:///run/containerd/containerd.sock"
taints:{{if not .Taints}} []{{end}}{{range .Taints}}
- key: "{{.Key}}"
value: "{{.Value}}"
effect: "{{.Effect}}"{{end}}
kubeletExtraArgs:
{{if .Nodepool }}
node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
Expand Down
7 changes: 7 additions & 0 deletions cmd/pke/app/phases/kubeadm/node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type Node struct {
azureVMType string
azureLoadBalancerSku string
azureRouteTableName string
taints []string
}

func NewCommand(out io.Writer) *cobra.Command {
Expand Down Expand Up @@ -109,6 +110,8 @@ func (n *Node) RegisterFlags(flags *pflag.FlagSet) {
flags.String(constants.FlagAzureVMType, "standard", "The type of azure nodes. Candidate values are: vmss and standard")
flags.String(constants.FlagAzureLoadBalancerSku, "basic", "Sku of Load Balancer and Public IP. Candidate values are: basic and standard")
flags.String(constants.FlagAzureRouteTableName, "kubernetes-routes", "The name of the route table attached to the subnet that the cluster is deployed in")
// Taints
flags.StringSlice(constants.FlagTaints, nil, "Specifies the taints the Node should be registered with")
}

func (n *Node) Validate(cmd *cobra.Command) error {
Expand Down Expand Up @@ -228,6 +231,10 @@ func (n *Node) workerBootstrapParameters(cmd *cobra.Command) (err error) {
return
}
n.azureRouteTableName, err = cmd.Flags().GetString(constants.FlagAzureRouteTableName)
if err != nil {
return
}
n.taints, err = cmd.Flags().GetStringSlice(constants.FlagTaints)

return
}
Expand Down
1 change: 1 addition & 0 deletions cmd/pke/app/phases/kubeadm/node/node_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func TestWriteKubeadmConfig(t *testing.T) {
caCertHash: "sha256:xxx",
cloudProvider: constants.CloudProviderAmazon,
nodepool: "pool2",
taints: []string{"node-role.kubernetes.io/master:NoSchedule-"},
}

err := n.writeKubeadmConfig(os.Stdout, filename)
Expand Down
80 changes: 80 additions & 0 deletions cmd/pke/app/util/kubernetes/taint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright © 2019 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
"strings"

"github.com/pkg/errors"
)

type Taint struct {
Key string
Value string
Effect string
}

func isTaint(s string) bool {
return strings.Contains(s, "=") || strings.Contains(s, ":") || strings.HasSuffix(s, "-")
}

func ParseTaints(taints []string) ([]Taint, error) {
t := make([]Taint, 0)
for _, taint := range taints {
// Remove leading and trailing spaces
taint = strings.Trim(taint, " ")
// Skip empty
if taint == "" {
continue
}
// Validate taint
if !isTaint(taint) {
return nil, errors.New("invalid taint spec: " + taint)
}

var (
key string
value string
effect string
)

// key=value
eq := strings.Index(taint, "=")
if eq > -1 {
key = taint[:eq]
}

// effect
colon := strings.Index(taint, ":")
if colon > -1 {
effect = taint[colon+1:]
if eq > -1 {
value = taint[eq+1 : colon]
} else {
key = taint[eq+1 : colon]
}
} else {
effect = taint
}

t = append(t, Taint{
Key: key,
Value: value,
Effect: effect,
})
}

return t, nil
}
49 changes: 49 additions & 0 deletions cmd/pke/app/util/kubernetes/taint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright © 2019 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestParseTaints(t *testing.T) {
testCases := []struct {
taint string
expected *Taint
err bool
}{
{"foo=bar:NoSchedule", &Taint{Key: "foo", Value: "bar", Effect: "NoSchedule"}, false},
{"foo:PreferNoSchedule", &Taint{Key: "foo", Value: "", Effect: "PreferNoSchedule"}, false},
{"dedicated:NoSchedule-", &Taint{Key: "dedicated", Effect: "NoSchedule-"}, false},
{"dedicated-", &Taint{Effect: "dedicated-"}, false},
{"", nil, false},
{"xxx", nil, true},
}
for _, tc := range testCases {
taint, err := ParseTaints([]string{tc.taint})
if tc.err {
require.Error(t, err)
continue
}
require.NoError(t, err)
if tc.expected == nil {
require.Empty(t, taint)
} else {
require.Equal(t, tc.expected, &taint[0])
}
}
}
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_master.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ pke install master [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with (default [node-role.kubernetes.io/master:NoSchedule])
--with-plugin-psp Enable PodSecurityPolicy admission plugin
```

Expand Down
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_master_kubernetes-controlplane.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ pke install master kubernetes-controlplane [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with (default [node-role.kubernetes.io/master:NoSchedule])
--with-plugin-psp Enable PodSecurityPolicy admission plugin
```

Expand Down
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_single.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ pke install single [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with (default [node-role.kubernetes.io/master:NoSchedule])
--with-plugin-psp Enable PodSecurityPolicy admission plugin
```

Expand Down
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_single_kubernetes-controlplane.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ pke install single kubernetes-controlplane [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with (default [node-role.kubernetes.io/master:NoSchedule])
--with-plugin-psp Enable PodSecurityPolicy admission plugin
```

Expand Down
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_worker.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ pke install worker [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with
```

### SEE ALSO
Expand Down
1 change: 1 addition & 0 deletions cmd/pke/docs/pke_install_worker_kubernetes-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ pke install worker kubernetes-node [flags]
--pipeline-org-id int32 Organization ID to use with Pipeline API
-t, --pipeline-token string Token for accessing Pipeline API
-u, --pipeline-url string Pipeline API server url
--taints strings Specifies the taints the Node should be registered with
```

### SEE ALSO
Expand Down

0 comments on commit b299cad

Please sign in to comment.