Skip to content

Commit

Permalink
Merge branch 'main' into refactor-node-mectirs-collector
Browse files Browse the repository at this point in the history
Signed-off-by: 黄金 <[email protected]>

# Conflicts:
#	pkg/koordlet/metricsadvisor/framework/plugin.go
  • Loading branch information
LambdaHJ committed May 25, 2023
2 parents ceddba7 + f81bbd8 commit cb12c09
Show file tree
Hide file tree
Showing 106 changed files with 7,058 additions and 3,378 deletions.
1 change: 1 addition & 0 deletions .licenseignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pkg/descheduler/controllers/migration/controllerfinder
pkg/scheduler/frameworkext/temporary_snapshot.go
pkg/scheduler/frameworkext/temporary_snapshot_test.go
pkg/scheduler/plugins/coscheduling
pkg/scheduler/plugins/reservation/error_channel.go
pkg/koordlet/util/kubelet
pkg/util/cpuset/cpuset.go
pkg/util/cpuset/cpuset_test.go
Expand Down
5 changes: 5 additions & 0 deletions apis/config/v1alpha1/cluster_colocation_profile_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
)

// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
Expand All @@ -38,6 +39,10 @@ type ClusterColocationProfileSpec struct {
// +optional
Selector *metav1.LabelSelector `json:"selector,omitempty"`

// Probability indicates profile will make effect with a probability.
// +optional
Probability *intstr.IntOrString `json:"probability,omitempty"`

// QoSClass describes the type of Koordinator QoS that the Pod is running.
// The value will be injected into Pod as label koordinator.sh/qosClass.
// Options are LSE/LSR/LS/BE/SYSTEM.
Expand Down
6 changes: 6 additions & 0 deletions apis/config/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 23 additions & 0 deletions apis/scheduling/v1alpha1/reservation_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,31 @@ type ReservationSpec struct {
// +kubebuilder:default=true
// +optional
AllocateOnce *bool `json:"allocateOnce,omitempty"`
// AllocatePolicy represents the allocation policy of reserved resources that Reservation expects.
// +kubebuilder:validation:Enum=Aligned;Restricted
// +optional
AllocatePolicy ReservationAllocatePolicy `json:"allocatePolicy,omitempty"`
}

type ReservationAllocatePolicy string

const (
// ReservationAllocatePolicyDefault means that there is no restriction on the policy of reserved resources,
// and allocated from the Reservation first, and if it is insufficient, it is allocated from the node.
ReservationAllocatePolicyDefault ReservationAllocatePolicy = ""
// ReservationAllocatePolicyAligned indicates that the Pod allocates resources from the Reservation first.
// If the remaining resources of the Reservation are insufficient, it can be allocated from the node,
// but it is required to strictly follow the resource specifications of the Pod.
// This can be used to avoid the problem that a Pod uses multiple Reservations at the same time.
ReservationAllocatePolicyAligned ReservationAllocatePolicy = "Aligned"
// ReservationAllocatePolicyRestricted means that the resources
// requested by the Pod overlap with the resources reserved by the Reservation,
// then these intersection resources can only be allocated from the Reservation,
// but resources declared in Pods but not reserved in Reservations can be allocated from Nodes.
// ReservationAllocatePolicyRestricted includes the semantics of ReservationAllocatePolicyAligned.
ReservationAllocatePolicyRestricted ReservationAllocatePolicy = "Restricted"
)

// ReservationTemplateSpec describes the data a Reservation should have when created from a template
type ReservationTemplateSpec struct {
// Standard object's metadata.
Expand Down
25 changes: 1 addition & 24 deletions cmd/koord-manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,13 @@ import (
"time"

"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/klog/v2"
"k8s.io/klog/v2/klogr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1"

configv1alpha1 "github.com/koordinator-sh/koordinator/apis/config/v1alpha1"
schedulingv1alpha1 "github.com/koordinator-sh/koordinator/apis/scheduling/v1alpha1"
slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
"github.com/koordinator-sh/koordinator/cmd/koord-manager/extensions"
"github.com/koordinator-sh/koordinator/cmd/koord-manager/options"
extclient "github.com/koordinator-sh/koordinator/pkg/client"
Expand All @@ -53,28 +46,12 @@ import (
)

var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")

restConfigQPS = flag.Int("rest-config-qps", 30, "QPS of rest config.")
restConfigBurst = flag.Int("rest-config-burst", 50, "Burst of rest config.")
)

func init() {
_ = clientgoscheme.AddToScheme(scheme)
_ = configv1alpha1.AddToScheme(clientgoscheme.Scheme)
_ = slov1alpha1.AddToScheme(clientgoscheme.Scheme)
_ = schedulingv1alpha1.AddToScheme(clientgoscheme.Scheme)

_ = configv1alpha1.AddToScheme(scheme)
_ = slov1alpha1.AddToScheme(scheme)
_ = schedulingv1alpha1.AddToScheme(scheme)
_ = v1alpha1.AddToScheme(scheme)

scheme.AddUnversionedTypes(metav1.SchemeGroupVersion, &metav1.UpdateOptions{}, &metav1.DeleteOptions{}, &metav1.CreateOptions{})
// +kubebuilder:scaffold:scheme
}

func main() {
var metricsAddr, pprofAddr string
var healthProbeAddr string
Expand Down Expand Up @@ -132,7 +109,7 @@ func main() {
}
}
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Scheme: scheme,
Scheme: options.Scheme,
MetricsBindAddress: metricsAddr,
HealthProbeBindAddress: healthProbeAddr,
LeaderElection: enableLeaderElection,
Expand Down
2 changes: 1 addition & 1 deletion cmd/koord-manager/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (o *Options) InitFlags(fs *flag.FlagSet) {

func (o *Options) ApplyTo(m manager.Manager) error {
for controllerName, addFn := range o.ControllerAddFuncs {
if isControllerEnabled(controllerName, o.Controllers) {
if !isControllerEnabled(controllerName, o.Controllers) {
klog.Warningf("controller %q is disabled", controllerName)
continue
}
Expand Down
130 changes: 130 additions & 0 deletions cmd/koord-manager/options/options_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
"fmt"
"sort"
"testing"

"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"
)

type fakeManager struct {
manager.Manager
ControllerAdded []string
}

type fakeController struct {
manager.Runnable
ControllerName string
}

func (c *fakeController) Add(mgr ctrl.Manager) error {
m, ok := mgr.(*fakeManager)
if !ok {
return fmt.Errorf("fakeController only supports fakeManager")
}
m.ControllerAdded = append(m.ControllerAdded, c.ControllerName)
return nil
}

func TestOptions(t *testing.T) {
t.Run("test", func(t *testing.T) {
opt := NewOptions()
assert.NotNil(t, opt)

args := []string{
"",
"--controllers=noderesource,nodemetric",
}
opt.InitFlags(nil)
pflag.NewFlagSet(args[0], pflag.ExitOnError)
err := pflag.CommandLine.Parse(args[1:])
assert.NoError(t, err)
assert.Equal(t, []string{"noderesource", "nodemetric"}, opt.Controllers)
})
}

func TestOptionsApplyTo(t *testing.T) {
controllerA := &fakeController{
ControllerName: "a",
}
controllerB := &fakeController{
ControllerName: "b",
}
type args struct {
controllersToAdd []string
controllers map[string]func(manager.Manager) error
}
tests := []struct {
name string
args args
want []string
}{
{
name: "add each controllers",
args: args{
controllersToAdd: []string{"a", "b"},
controllers: map[string]func(manager.Manager) error{
"a": controllerA.Add,
"b": controllerB.Add,
},
},
want: []string{"a", "b"},
},
{
name: "add all controllers",
args: args{
controllersToAdd: []string{"*"},
controllers: map[string]func(manager.Manager) error{
"a": controllerA.Add,
"b": controllerB.Add,
},
},
want: []string{"a", "b"},
},
{
name: "add some controllers",
args: args{
controllersToAdd: []string{"b"},
controllers: map[string]func(manager.Manager) error{
"a": controllerA.Add,
"b": controllerB.Add,
},
},
want: []string{"b"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
opt := &Options{
Controllers: tt.args.controllersToAdd,
ControllerAddFuncs: tt.args.controllers,
}

mgr := &fakeManager{}
err := opt.ApplyTo(mgr)
assert.NoError(t, err)
sort.Strings(mgr.ControllerAdded)
assert.Equal(t, tt.want, mgr.ControllerAdded)
})
}
}
45 changes: 45 additions & 0 deletions cmd/koord-manager/options/scheme.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
Copyright 2022 The Koordinator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1"

configv1alpha1 "github.com/koordinator-sh/koordinator/apis/config/v1alpha1"
schedulingv1alpha1 "github.com/koordinator-sh/koordinator/apis/scheduling/v1alpha1"
slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
)

var Scheme = runtime.NewScheme()

func init() {
_ = clientgoscheme.AddToScheme(Scheme)
_ = configv1alpha1.AddToScheme(clientgoscheme.Scheme)
_ = slov1alpha1.AddToScheme(clientgoscheme.Scheme)
_ = schedulingv1alpha1.AddToScheme(clientgoscheme.Scheme)

_ = configv1alpha1.AddToScheme(Scheme)
_ = slov1alpha1.AddToScheme(Scheme)
_ = schedulingv1alpha1.AddToScheme(Scheme)
_ = v1alpha1.AddToScheme(Scheme)

Scheme.AddUnversionedTypes(metav1.SchemeGroupVersion, &metav1.UpdateOptions{}, &metav1.DeleteOptions{}, &metav1.CreateOptions{})
// +kubebuilder:scaffold:scheme
}
18 changes: 9 additions & 9 deletions cmd/koord-scheduler/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ import (

schedulerserverconfig "github.com/koordinator-sh/koordinator/cmd/koord-scheduler/app/config"
"github.com/koordinator-sh/koordinator/cmd/koord-scheduler/app/options"
"github.com/koordinator-sh/koordinator/pkg/scheduler/eventhandlers"
"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext"
"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext/defaultprofile"
"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext/eventhandlers"
"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext/services"
utilroutes "github.com/koordinator-sh/koordinator/pkg/util/routes"
)
Expand Down Expand Up @@ -336,6 +337,8 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
// Get the completed config
cc := c.Complete()

defaultprofile.AppendDefaultPlugins(cc.ComponentConfig.Profiles)

// NOTE(joseph): K8s scheduling framework does not provide extension point for initialization.
// Currently, only by copying the initialization code and implementing custom initialization.
frameworkExtenderFactory, err := frameworkext.NewFrameworkExtenderFactory(
Expand Down Expand Up @@ -384,9 +387,6 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
return nil, nil, nil, err
}

// TODO(joseph): Some extensions can also be made in the future,
// such as replacing some interfaces in Scheduler to implement custom logic

// extend framework to hook run plugin functions
for k := range sched.Profiles {
extender := frameworkExtenderFactory.GetExtender(k)
Expand All @@ -395,11 +395,11 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
}
}

schedulerInternalHandler := &eventhandlers.SchedulerInternalHandlerImpl{
Scheduler: sched,
}
eventhandlers.AddScheduleEventHandler(sched, schedulerInternalHandler, frameworkExtenderFactory.KoordinatorSharedInformerFactory())
eventhandlers.AddReservationErrorHandler(sched, schedulerInternalHandler, frameworkExtenderFactory.KoordinatorClientSet(), frameworkExtenderFactory.KoordinatorSharedInformerFactory())
frameworkExtenderFactory.InitScheduler(sched)
schedAdapter := frameworkExtenderFactory.Scheduler()

eventhandlers.AddScheduleEventHandler(sched, schedAdapter, frameworkExtenderFactory.KoordinatorSharedInformerFactory())
eventhandlers.AddReservationErrorHandler(sched, schedAdapter, frameworkExtenderFactory.KoordinatorClientSet(), frameworkExtenderFactory.KoordinatorSharedInformerFactory())

return &cc, sched, frameworkExtenderFactory, nil
}
2 changes: 2 additions & 0 deletions cmd/koord-scheduler/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/batchresource"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/compatibledefaultpreemption"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/coscheduling"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/defaultprebind"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/deviceshare"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/elasticquota"
"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/loadaware"
Expand All @@ -49,6 +50,7 @@ var koordinatorPlugins = map[string]frameworkruntime.PluginFactory{
deviceshare.Name: deviceshare.New,
elasticquota.Name: elasticquota.New,
compatibledefaultpreemption.Name: compatibledefaultpreemption.New,
defaultprebind.Name: defaultprebind.New,
}

func flatten(plugins map[string]frameworkruntime.PluginFactory) []app.Option {
Expand Down
Loading

0 comments on commit cb12c09

Please sign in to comment.