Skip to content

Commit

Permalink
Merge pull request #684 from ozline/dev
Browse files Browse the repository at this point in the history
refactor resource-recommend controller
  • Loading branch information
luomingmeng authored Oct 28, 2024
2 parents 0c234bf + 3b8ea47 commit 4fe5c45
Show file tree
Hide file tree
Showing 27 changed files with 1,051 additions and 1,070 deletions.
19 changes: 15 additions & 4 deletions cmd/katalyst-controller/app/controller/resourcerecommender.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,30 @@ const (

func StartResourceRecommenderController(
ctx context.Context,
_ *katalyst.GenericContext,
controlCtx *katalyst.GenericContext,
conf *config.Configuration,
_ interface{},
_ string,
) (bool, error) {
resourceRecommenderController, err := controller.NewResourceRecommenderController(ctx,
oomRecorderController, err := controller.NewPodOOMRecorderController(ctx, controlCtx,
conf.GenericConfiguration,
conf.GenericControllerConfiguration,
conf.ControllersConfiguration.ResourceRecommenderConfig)
if err != nil {
klog.Errorf("failed to new ResourceRecommender controller")
klog.Errorf("failed to new PodOOMRecorder controller")
return false, err
}
recController, err := controller.NewResourceRecommendController(ctx, controlCtx,
conf.GenericConfiguration,
conf.GenericControllerConfiguration,
conf.ControllersConfiguration.ResourceRecommenderConfig,
oomRecorderController.Recorder)
if err != nil {
klog.Errorf("failed to new ResourceRecommend Controller")
return false, err
}
go oomRecorderController.Run()
go recController.Run()

go resourceRecommenderController.Run()
return true, nil
}
12 changes: 12 additions & 0 deletions cmd/katalyst-controller/app/options/resourcerecommender.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import (
"github.com/kubewharf/katalyst-core/pkg/util/datasource/prometheus"
)

const (
defaultRecSyncWorkers = 1
defaultResourceRecommendReSyncPeriod = 24 * time.Hour
)

type ResourceRecommenderOptions struct {
OOMRecordMaxNumber int `desc:"max number for oom record"`

Expand All @@ -39,6 +44,9 @@ type ResourceRecommenderOptions struct {
// LogVerbosityLevel to specify log verbosity level. (The default level is 4)
// Set it to something larger than 4 if more detailed logs are needed.
LogVerbosityLevel string

RecSyncWorkers int
RecSyncPeriod time.Duration
}

// NewResourceRecommenderOptions creates a new Options with a default config.
Expand Down Expand Up @@ -79,6 +87,8 @@ func (o *ResourceRecommenderOptions) AddFlags(fss *cliflag.NamedFlagSets) {
fs.StringVar(&o.DataSourcePromConfig.BaseFilter, "resourcerecommend-prometheus-promql-base-filter", "", ""+
"Get basic filters in promql for historical usage data. This filter is added to all promql statements. "+
"Supports filters format of promql, e.g: group=\\\"Katalyst\\\",cluster=\\\"cfeaf782fasdfe\\\"")
fs.IntVar(&o.RecSyncWorkers, "res-sync-workers", defaultRecSyncWorkers, "num of goroutine to sync recs")
fs.DurationVar(&o.RecSyncPeriod, "resource-recommend-resync-period", defaultResourceRecommendReSyncPeriod, "period for recommend controller to sync resource recommend")
}

func (o *ResourceRecommenderOptions) ApplyTo(c *controller.ResourceRecommenderConfig) error {
Expand All @@ -88,6 +98,8 @@ func (o *ResourceRecommenderOptions) ApplyTo(c *controller.ResourceRecommenderCo
c.DataSource = o.DataSource
c.DataSourcePromConfig = o.DataSourcePromConfig
c.LogVerbosityLevel = o.LogVerbosityLevel
c.RecSyncWorkers = o.RecSyncWorkers
c.RecSyncPeriod = o.RecSyncPeriod
return nil
}

Expand Down
7 changes: 6 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.18
require (
bou.ke/monkey v1.0.2
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
github.com/bytedance/mockey v1.2.11
github.com/cespare/xxhash v1.1.0
github.com/cilium/ebpf v0.7.0
github.com/containerd/cgroups v1.0.1
Expand All @@ -29,6 +30,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.39.0
github.com/slok/kubewebhook v0.11.0
github.com/smartystreets/goconvey v1.6.4
github.com/spf13/cobra v1.6.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.1
Expand Down Expand Up @@ -93,6 +95,7 @@ require (
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
Expand All @@ -101,6 +104,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
Expand All @@ -117,6 +121,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/smartystreets/assertions v1.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
Expand All @@ -129,14 +134,14 @@ require (
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
gomodules.xyz/jsonpatch/v3 v3.0.1 // indirect
gomodules.xyz/orderedmap v0.1.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
11 changes: 8 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bytedance/mockey v1.2.11 h1:xIR17ILTtyeIh0iNTCbtslRnB7/N2o16wQvmtlbMivA=
github.com/bytedance/mockey v1.2.11/go.mod h1:bNrUnI1u7+pAc0TYDgPATM+wF2yzHxmNH+iDXg4AOCU=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down Expand Up @@ -250,7 +252,6 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8=
github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
Expand Down Expand Up @@ -454,6 +455,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
Expand Down Expand Up @@ -518,7 +520,6 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
Expand All @@ -542,6 +543,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
Expand Down Expand Up @@ -823,7 +825,9 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
github.com/slok/kubewebhook v0.11.0 h1:mRUOHXpMNxROTcqGVq06BQX2r13cT1Kjw0ylcJhTg0g=
github.com/slok/kubewebhook v0.11.0/go.mod h1:HWkaQH3ZbQpLeP3ylW/NPhOaYByxCIRU36HPmUEoqyo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
Expand Down Expand Up @@ -1003,6 +1007,8 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff h1:XmKBi9R6duxOB3lfc72wyrwiOY7X2Jl1wuI+RFOyMDE=
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down Expand Up @@ -1367,7 +1373,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
gomodules.xyz/jsonpatch/v3 v3.0.1 h1:Te7hKxV52TKCbNYq3t84tzKav3xhThdvSsSp/W89IyI=
gomodules.xyz/jsonpatch/v3 v3.0.1/go.mod h1:CBhndykehEwTOlEfnsfJwvkFQbSN8YZFr9M+cIHAJto=
gomodules.xyz/orderedmap v0.1.0 h1:fM/+TGh/O1KkqGR5xjTKg6bU8OKBkg7p0Y+x/J9m8Os=
Expand Down
114 changes: 114 additions & 0 deletions pkg/client/control/resourcerecommend.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package control

import (
"context"
"encoding/json"
"fmt"

v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/jsonmergepatch"

apis "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1"
clientset "github.com/kubewharf/katalyst-api/pkg/client/clientset/versioned"
)

// ResourceRecommendUpdater is used to update ResourceRecommend CR
type ResourceRecommendUpdater interface {
UpdateResourceRecommend(ctx context.Context, rec *apis.ResourceRecommend,
opts v1.UpdateOptions) (*apis.ResourceRecommend, error)

PatchResourceRecommend(ctx context.Context, oldRec *apis.ResourceRecommend,
newRec *apis.ResourceRecommend) error

CreateResourceRecommend(ctx context.Context, rec *apis.ResourceRecommend,
opts v1.CreateOptions) (*apis.ResourceRecommend, error)
}
type DummyResourceRecommendUpdater struct{}

func (d *DummyResourceRecommendUpdater) UpdateResourceRecommend(_ context.Context, _ *apis.ResourceRecommend,
_ v1.UpdateOptions,
) (*apis.ResourceRecommend, error) {
return nil, nil
}

func (d *DummyResourceRecommendUpdater) PatchResourceRecommend(_ context.Context, _ *apis.ResourceRecommend,
_ *apis.ResourceRecommend,
) error {
return nil
}

func (d *DummyResourceRecommendUpdater) CreateResourceRecommend(_ context.Context, _ *apis.ResourceRecommend,
_ v1.CreateOptions,
) (*apis.ResourceRecommend, error) {
return nil, nil
}

type RealResourceRecommendUpdater struct {
client clientset.Interface
}

func NewRealResourceRecommendUpdater(client clientset.Interface) *RealResourceRecommendUpdater {
return &RealResourceRecommendUpdater{
client: client,
}
}

func (r *RealResourceRecommendUpdater) UpdateResourceRecommend(ctx context.Context, rec *apis.ResourceRecommend,
opts v1.UpdateOptions,
) (*apis.ResourceRecommend, error) {
if rec == nil {
return nil, fmt.Errorf("can't update a nil ResourceRecommend")
}

return r.client.RecommendationV1alpha1().ResourceRecommends(rec.Namespace).Update(ctx, rec, opts)
}

func (r *RealResourceRecommendUpdater) PatchResourceRecommend(ctx context.Context, oldRec *apis.ResourceRecommend,
newRec *apis.ResourceRecommend,
) error {
if oldRec == nil || newRec == nil {
return fmt.Errorf("can't patch a nil ResourceRecommend")
}

oldData, err := json.Marshal(oldRec)
if err != nil {
return err
}
newData, err := json.Marshal(newRec)
if err != nil {
return err
}

patchBytes, err := jsonmergepatch.CreateThreeWayJSONMergePatch(oldData, newData, oldData)
if err != nil {
return fmt.Errorf("failed to create merge patch for ResourceRecommend %q/%q: %v", oldRec.Namespace, oldRec.Name, err)
}

_, err = r.client.RecommendationV1alpha1().ResourceRecommends(oldRec.Namespace).Patch(ctx, oldRec.Name, types.MergePatchType, patchBytes, v1.PatchOptions{}, "status")
return err
}

func (r *RealResourceRecommendUpdater) CreateResourceRecommend(ctx context.Context, rec *apis.ResourceRecommend, opts v1.CreateOptions) (*apis.ResourceRecommend, error) {
if rec == nil {
return nil, fmt.Errorf("can't update a nil ResourceRecommend")
}

return r.client.RecommendationV1alpha1().ResourceRecommends(rec.Namespace).Create(ctx, rec, opts)
}
Loading

0 comments on commit 4fe5c45

Please sign in to comment.