diff --git a/cmd/apiserver/app/options/options.go b/cmd/apiserver/app/options/options.go index d3f0b6ac6..e77c2712f 100644 --- a/cmd/apiserver/app/options/options.go +++ b/cmd/apiserver/app/options/options.go @@ -21,6 +21,7 @@ import ( "github.com/clusterpedia-io/clusterpedia/pkg/apiserver" generatedopenapi "github.com/clusterpedia-io/clusterpedia/pkg/generated/openapi" + "github.com/clusterpedia-io/clusterpedia/pkg/kubeapiserver" "github.com/clusterpedia-io/clusterpedia/pkg/storage" storageoptions "github.com/clusterpedia-io/clusterpedia/pkg/storage/options" ) @@ -40,7 +41,8 @@ type ClusterPediaServerOptions struct { Traces *genericoptions.TracingOptions Metrics *metrics.Options - Storage *storageoptions.StorageOptions + Storage *storageoptions.StorageOptions + ResourceServer *kubeapiserver.Options } func NewServerOptions() *ClusterPediaServerOptions { @@ -70,7 +72,8 @@ func NewServerOptions() *ClusterPediaServerOptions { Traces: genericoptions.NewTracingOptions(), Metrics: metrics.NewOptions(), - Storage: storageoptions.NewStorageOptions(), + Storage: storageoptions.NewStorageOptions(), + ResourceServer: kubeapiserver.NewOptions(), } } @@ -94,6 +97,11 @@ func (o *ClusterPediaServerOptions) Config() (*apiserver.Config, error) { return nil, err } + resourceServerConfig, err := o.ResourceServer.Config() + if err != nil { + return nil, err + } + if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { return nil, fmt.Errorf("error create self-signed certificates: %v", err) } @@ -121,6 +129,7 @@ func (o *ClusterPediaServerOptions) Config() (*apiserver.Config, error) { return &apiserver.Config{ GenericConfig: genericConfig, StorageFactory: storage, + ExtraConfig: resourceServerConfig, }, nil } @@ -178,6 +187,7 @@ func (o *ClusterPediaServerOptions) Flags() cliflag.NamedFlagSets { o.Metrics.AddFlags(fss.FlagSet("metrics")) o.Storage.AddFlags(fss.FlagSet("storage")) + o.ResourceServer.AddFlags(fss.FlagSet("resource server")) return fss } diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 874917ee3..9067a99e7 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -64,6 +64,7 @@ type Config struct { GenericConfig *genericapiserver.RecommendedConfig StorageFactory storage.StorageFactory + ExtraConfig *kubeapiserver.ExtraConfig } type ClusterPediaServer struct { @@ -75,6 +76,7 @@ type completedConfig struct { ClientConfig *clientrest.Config StorageFactory storage.StorageFactory + ExtraConfig *kubeapiserver.ExtraConfig } // CompletedConfig embeds a private pointer that cannot be instantiated outside of this package. @@ -90,6 +92,7 @@ func (cfg *Config) Complete() CompletedConfig { cfg.GenericConfig.Complete(), cfg.GenericConfig.ClientConfig, cfg.StorageFactory, + cfg.ExtraConfig, } return CompletedConfig{&c} } @@ -121,11 +124,10 @@ func (config completedConfig) New() (*ClusterPediaServer, error) { resourceServerConfig.GenericConfig.ExternalAddress = config.GenericConfig.ExternalAddress resourceServerConfig.GenericConfig.LoopbackClientConfig = config.GenericConfig.LoopbackClientConfig resourceServerConfig.GenericConfig.TracerProvider = config.GenericConfig.TracerProvider - resourceServerConfig.ExtraConfig = kubeapiserver.ExtraConfig{ - InformerFactory: clusterpediaInformerFactory, - StorageFactory: config.StorageFactory, - InitialAPIGroupResources: initialAPIGroupResources, - } + resourceServerConfig.InformerFactory = clusterpediaInformerFactory + resourceServerConfig.StorageFactory = config.StorageFactory + resourceServerConfig.InitialAPIGroupResources = initialAPIGroupResources + resourceServerConfig.ExtraConfig = config.ExtraConfig kubeResourceAPIServer, methods, err := resourceServerConfig.Complete().New(genericapiserver.NewEmptyDelegate()) if err != nil { return nil, err diff --git a/pkg/kubeapiserver/apiserver.go b/pkg/kubeapiserver/apiserver.go index 68d4741d4..70e74150e 100644 --- a/pkg/kubeapiserver/apiserver.go +++ b/pkg/kubeapiserver/apiserver.go @@ -66,15 +66,17 @@ func NewDefaultConfig() *Config { } type ExtraConfig struct { - StorageFactory storage.StorageFactory - InformerFactory informers.SharedInformerFactory - InitialAPIGroupResources []*restmapper.APIGroupResources + AllowedProxySubresources map[schema.GroupResource]sets.Set[string] } type Config struct { GenericConfig *genericapiserver.RecommendedConfig - ExtraConfig ExtraConfig + StorageFactory storage.StorageFactory + InformerFactory informers.SharedInformerFactory + InitialAPIGroupResources []*restmapper.APIGroupResources + + ExtraConfig *ExtraConfig } func (c *Config) Complete() CompletedConfig { @@ -83,8 +85,11 @@ func (c *Config) Complete() CompletedConfig { } completed := &completedConfig{ - GenericConfig: c.GenericConfig.Complete(), - ExtraConfig: &c.ExtraConfig, + GenericConfig: c.GenericConfig.Complete(), + StorageFactory: c.StorageFactory, + InformerFactory: c.InformerFactory, + InitialAPIGroupResources: c.InitialAPIGroupResources, + ExtraConfig: c.ExtraConfig, } c.GenericConfig.RequestInfoResolver = wrapRequestInfoResolverForNamespace{ @@ -96,7 +101,10 @@ func (c *Config) Complete() CompletedConfig { type completedConfig struct { GenericConfig genericapiserver.CompletedConfig - ExtraConfig *ExtraConfig + StorageFactory storage.StorageFactory + InformerFactory informers.SharedInformerFactory + InitialAPIGroupResources []*restmapper.APIGroupResources + ExtraConfig *ExtraConfig } type CompletedConfig struct { @@ -106,10 +114,10 @@ type CompletedConfig struct { var sortedMethods = []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*genericapiserver.GenericAPIServer, []string, error) { - if c.ExtraConfig.StorageFactory == nil { + if c.StorageFactory == nil { return nil, nil, errors.New("kubeapiserver.New() called with config.StorageFactory == nil") } - if c.ExtraConfig.InformerFactory == nil { + if c.InformerFactory == nil { return nil, nil, errors.New("kubeapiserver.New() called with config.InformerFactory == nil") } @@ -123,7 +131,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) delegate = http.NotFoundHandler() } - restManager := NewRESTManager(c.GenericConfig.Serializer, runtime.ContentTypeJSON, c.ExtraConfig.StorageFactory, c.ExtraConfig.InitialAPIGroupResources) + restManager := NewRESTManager(c.GenericConfig.Serializer, runtime.ContentTypeJSON, c.StorageFactory, c.InitialAPIGroupResources) discoveryManager := discovery.NewDiscoveryManager(c.GenericConfig.Serializer, restManager, delegate) // handle root discovery request @@ -136,15 +144,20 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) delegate: delegate, rest: restManager, discovery: discoveryManager, - clusterLister: c.ExtraConfig.InformerFactory.Cluster().V1alpha2().PediaClusters().Lister(), + clusterLister: c.InformerFactory.Cluster().V1alpha2().PediaClusters().Lister(), } genericserver.Handler.NonGoRestfulMux.HandlePrefix("/api/", resourceHandler) genericserver.Handler.NonGoRestfulMux.HandlePrefix("/apis/", resourceHandler) - controller := NewClusterResourceController(restManager, discoveryManager, c.ExtraConfig.InformerFactory.Cluster().V1alpha2().PediaClusters()) + controller := NewClusterResourceController(restManager, discoveryManager, c.InformerFactory.Cluster().V1alpha2().PediaClusters()) methodSet := sets.New("GET") for _, rest := range proxyrest.GetSubresourceRESTs(controller) { + allows := c.ExtraConfig.AllowedProxySubresources[rest.ParentGroupResource()] + if allows == nil || !allows.Has(rest.Subresource()) { + continue + } + if err := restManager.preRegisterSubresource(subresource{ gr: rest.ParentGroupResource(), kind: rest.ParentKind(), diff --git a/pkg/kubeapiserver/options.go b/pkg/kubeapiserver/options.go new file mode 100644 index 000000000..acea674c2 --- /dev/null +++ b/pkg/kubeapiserver/options.go @@ -0,0 +1,79 @@ +package kubeapiserver + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" +) + +type Options struct { + AllowedProxySubresources []string +} + +func NewOptions() *Options { + return &Options{} +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + var resources []string + for r, srs := range supportedProxyCoreSubresources { + for _, sr := range srs { + resources = append(resources, r+"/"+sr) + } + } + + // To explicitly specify subresources, enabling all subresources of a parent resource + // using a pattern like `/*` is currently not supported. + // + // If you have a better solution, please submit an issue! + fs.StringSliceVar(&o.AllowedProxySubresources, "allowed-proxy-subresources", o.AllowedProxySubresources, ""+ + "List of subresources that support proxying requests to the specified cluster, formatted as '[resource/subresource],[subresource],...'. "+ + fmt.Sprintf("Supported proxy subresources include %q", strings.Join(resources, ",")), + ) +} + +var supportedProxyCoreSubresources = map[string][]string{ + "pods": {"proxy", "log", "exec", "attach", "portfowrd"}, + "nodes": {"proxy"}, + "services": {"proxy"}, +} + +func (o *Options) Config() (*ExtraConfig, error) { + subresources := make(map[schema.GroupResource]sets.Set[string]) + + for _, subresource := range o.AllowedProxySubresources { + var resource string + switch slice := strings.Split(strings.TrimSpace(subresource), "/"); len(slice) { + case 1: + subresource = slice[0] + case 2: + resource, subresource = slice[0], slice[1] + default: + return nil, fmt.Errorf("--allowed-proxy-subresources: invalid format %q", subresource) + } + + var matched bool + for r, srs := range supportedProxyCoreSubresources { + for _, sr := range srs { + if (resource == "" || resource == r) && subresource == sr { + gr := schema.GroupResource{Group: "", Resource: r} + set := subresources[gr] + if set == nil { + set = sets.New[string]() + subresources[gr] = set + } + set.Insert(sr) + matched = true + break + } + } + } + if !matched { + return nil, fmt.Errorf("--allowed-proxy-subresources: unsupported subresources or invalid format %q", subresource) + } + } + return &ExtraConfig{AllowedProxySubresources: subresources}, nil +}