From b64f5c70de9801f488d5e49aeea82eb2fa056fa0 Mon Sep 17 00:00:00 2001 From: moh-osman3 Date: Wed, 6 Mar 2024 03:33:47 -0500 Subject: [PATCH 1/4] fork configgrpc and add memorylimiterextension config option --- collector/config/configgrpc/configgrpc.go | 546 +++++++ .../config/configgrpc/configgrpc_test.go | 1338 +++++++++++++++++ collector/config/configgrpc/testdata/ca.crt | 20 + .../config/configgrpc/testdata/client.crt | 20 + .../config/configgrpc/testdata/client.key | 27 + .../config/configgrpc/testdata/server.crt | 20 + .../config/configgrpc/testdata/server.key | 27 + collector/config/configgrpc/wrappedstream.go | 34 + .../config/configgrpc/wrappedstream_test.go | 43 + 9 files changed, 2075 insertions(+) create mode 100644 collector/config/configgrpc/configgrpc.go create mode 100644 collector/config/configgrpc/configgrpc_test.go create mode 100644 collector/config/configgrpc/testdata/ca.crt create mode 100644 collector/config/configgrpc/testdata/client.crt create mode 100644 collector/config/configgrpc/testdata/client.key create mode 100644 collector/config/configgrpc/testdata/server.crt create mode 100644 collector/config/configgrpc/testdata/server.key create mode 100644 collector/config/configgrpc/wrappedstream.go create mode 100644 collector/config/configgrpc/wrappedstream_test.go diff --git a/collector/config/configgrpc/configgrpc.go b/collector/config/configgrpc/configgrpc.go new file mode 100644 index 00000000..10f78f1d --- /dev/null +++ b/collector/config/configgrpc/configgrpc.go @@ -0,0 +1,546 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package configgrpc // import "go.opentelemetry.io/collector/config/configgrpc" + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/mostynb/go-grpc-compression/nonclobbering/snappy" + "github.com/mostynb/go-grpc-compression/nonclobbering/zstd" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/client" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configauth" + "go.opentelemetry.io/collector/config/configcompression" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/extension/auth" +) + +var ( + errMetadataNotFound = errors.New("no request metadata found") + errMemoryLimitReached = status.Error(codes.ResourceExhausted, "Memory limit has been reached, too many requests.") +) + +// KeepaliveClientConfig exposes the keepalive.ClientParameters to be used by the exporter. +// Refer to the original data-structure for the meaning of each parameter: +// https://godoc.org/google.golang.org/grpc/keepalive#ClientParameters +type KeepaliveClientConfig struct { + Time time.Duration `mapstructure:"time"` + Timeout time.Duration `mapstructure:"timeout"` + PermitWithoutStream bool `mapstructure:"permit_without_stream"` +} + +// ClientConfig defines common settings for a gRPC client configuration. +type ClientConfig struct { + // The target to which the exporter is going to send traces or metrics, + // using the gRPC protocol. The valid syntax is described at + // https://github.com/grpc/grpc/blob/master/doc/naming.md. + Endpoint string `mapstructure:"endpoint"` + + // The compression key for supported compression types within collector. + Compression configcompression.Type `mapstructure:"compression"` + + // TLSSetting struct exposes TLS client configuration. + TLSSetting configtls.TLSClientSetting `mapstructure:"tls"` + + // The keepalive parameters for gRPC client. See grpc.WithKeepaliveParams. + // (https://godoc.org/google.golang.org/grpc#WithKeepaliveParams). + Keepalive *KeepaliveClientConfig `mapstructure:"keepalive"` + + // ReadBufferSize for gRPC client. See grpc.WithReadBufferSize. + // (https://godoc.org/google.golang.org/grpc#WithReadBufferSize). + ReadBufferSize int `mapstructure:"read_buffer_size"` + + // WriteBufferSize for gRPC gRPC. See grpc.WithWriteBufferSize. + // (https://godoc.org/google.golang.org/grpc#WithWriteBufferSize). + WriteBufferSize int `mapstructure:"write_buffer_size"` + + // WaitForReady parameter configures client to wait for ready state before sending data. + // (https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md) + WaitForReady bool `mapstructure:"wait_for_ready"` + + // The headers associated with gRPC requests. + Headers map[string]configopaque.String `mapstructure:"headers"` + + // Sets the balancer in grpclb_policy to discover the servers. Default is pick_first. + // https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md + BalancerName string `mapstructure:"balancer_name"` + + // WithAuthority parameter configures client to rewrite ":authority" header + // (godoc.org/google.golang.org/grpc#WithAuthority) + Authority string `mapstructure:"authority"` + + // Auth configuration for outgoing RPCs. + Auth *configauth.Authentication `mapstructure:"auth"` + + // MemoryLimiter is memory limiter this receiver will use to restrict incoming requests + MemoryLimiter *component.ID `mapstructure:"memory_limiter"` +} + +// KeepaliveServerConfig is the configuration for keepalive. +type KeepaliveServerConfig struct { + ServerParameters *KeepaliveServerParameters `mapstructure:"server_parameters"` + EnforcementPolicy *KeepaliveEnforcementPolicy `mapstructure:"enforcement_policy"` +} + +// KeepaliveServerParameters allow configuration of the keepalive.ServerParameters. +// The same default values as keepalive.ServerParameters are applicable and get applied by the server. +// See https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters for details. +type KeepaliveServerParameters struct { + MaxConnectionIdle time.Duration `mapstructure:"max_connection_idle"` + MaxConnectionAge time.Duration `mapstructure:"max_connection_age"` + MaxConnectionAgeGrace time.Duration `mapstructure:"max_connection_age_grace"` + Time time.Duration `mapstructure:"time"` + Timeout time.Duration `mapstructure:"timeout"` +} + +// KeepaliveEnforcementPolicy allow configuration of the keepalive.EnforcementPolicy. +// The same default values as keepalive.EnforcementPolicy are applicable and get applied by the server. +// See https://godoc.org/google.golang.org/grpc/keepalive#EnforcementPolicy for details. +type KeepaliveEnforcementPolicy struct { + MinTime time.Duration `mapstructure:"min_time"` + PermitWithoutStream bool `mapstructure:"permit_without_stream"` +} + +// ServerConfig defines common settings for a gRPC server configuration. +type ServerConfig struct { + // Server net.Addr config. For transport only "tcp" and "unix" are valid options. + NetAddr confignet.NetAddr `mapstructure:",squash"` + + // Configures the protocol to use TLS. + // The default value is nil, which will cause the protocol to not use TLS. + TLSSetting *configtls.TLSServerSetting `mapstructure:"tls"` + + // MaxRecvMsgSizeMiB sets the maximum size (in MiB) of messages accepted by the server. + MaxRecvMsgSizeMiB uint64 `mapstructure:"max_recv_msg_size_mib"` + + // MaxConcurrentStreams sets the limit on the number of concurrent streams to each ServerTransport. + // It has effect only for streaming RPCs. + MaxConcurrentStreams uint32 `mapstructure:"max_concurrent_streams"` + + // ReadBufferSize for gRPC server. See grpc.ReadBufferSize. + // (https://godoc.org/google.golang.org/grpc#ReadBufferSize). + ReadBufferSize int `mapstructure:"read_buffer_size"` + + // WriteBufferSize for gRPC server. See grpc.WriteBufferSize. + // (https://godoc.org/google.golang.org/grpc#WriteBufferSize). + WriteBufferSize int `mapstructure:"write_buffer_size"` + + // Keepalive anchor for all the settings related to keepalive. + Keepalive *KeepaliveServerConfig `mapstructure:"keepalive"` + + // Auth for this receiver + Auth *configauth.Authentication `mapstructure:"auth"` + + // Include propagates the incoming connection's metadata to downstream consumers. + // Experimental: *NOTE* this option is subject to change or removal in the future. + IncludeMetadata bool `mapstructure:"include_metadata"` + + MemoryLimiter *component.ID `mapstructure:"memory_limiter"` +} + +type memoryLimiterExtension = interface{ MustRefuse() bool } + +// SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.ClientConfig.Endpoint. +func (gcs *ClientConfig) SanitizedEndpoint() string { + switch { + case gcs.isSchemeHTTP(): + return strings.TrimPrefix(gcs.Endpoint, "http://") + case gcs.isSchemeHTTPS(): + return strings.TrimPrefix(gcs.Endpoint, "https://") + default: + return gcs.Endpoint + } +} + +func (gcs *ClientConfig) isSchemeHTTP() bool { + return strings.HasPrefix(gcs.Endpoint, "http://") +} + +func (gcs *ClientConfig) isSchemeHTTPS() bool { + return strings.HasPrefix(gcs.Endpoint, "https://") +} + +// ToClientConn creates a client connection to the given target. By default, it's +// a non-blocking dial (the function won't wait for connections to be +// established, and connecting happens in the background). To make it a blocking +// dial, use grpc.WithBlock() dial option. +func (gcs *ClientConfig) ToClientConn(ctx context.Context, host component.Host, settings component.TelemetrySettings, extraOpts ...grpc.DialOption) (*grpc.ClientConn, error) { + opts, err := gcs.toDialOptions(host, settings) + if err != nil { + return nil, err + } + opts = append(opts, extraOpts...) + return grpc.DialContext(ctx, gcs.SanitizedEndpoint(), opts...) +} + +func (gcs *ClientConfig) toDialOptions(host component.Host, settings component.TelemetrySettings) ([]grpc.DialOption, error) { + var opts []grpc.DialOption + if gcs.Compression.IsCompressed() { + cp, err := getGRPCCompressionName(gcs.Compression) + if err != nil { + return nil, err + } + opts = append(opts, grpc.WithDefaultCallOptions(grpc.UseCompressor(cp))) + } + + tlsCfg, err := gcs.TLSSetting.LoadTLSConfig() + if err != nil { + return nil, err + } + cred := insecure.NewCredentials() + if tlsCfg != nil { + cred = credentials.NewTLS(tlsCfg) + } else if gcs.isSchemeHTTPS() { + cred = credentials.NewTLS(&tls.Config{}) + } + opts = append(opts, grpc.WithTransportCredentials(cred)) + + if gcs.ReadBufferSize > 0 { + opts = append(opts, grpc.WithReadBufferSize(gcs.ReadBufferSize)) + } + + if gcs.WriteBufferSize > 0 { + opts = append(opts, grpc.WithWriteBufferSize(gcs.WriteBufferSize)) + } + + if gcs.Keepalive != nil { + keepAliveOption := grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: gcs.Keepalive.Time, + Timeout: gcs.Keepalive.Timeout, + PermitWithoutStream: gcs.Keepalive.PermitWithoutStream, + }) + opts = append(opts, keepAliveOption) + } + + if gcs.Auth != nil { + if host.GetExtensions() == nil { + return nil, errors.New("no extensions configuration available") + } + + grpcAuthenticator, cerr := gcs.Auth.GetClientAuthenticator(host.GetExtensions()) + if cerr != nil { + return nil, cerr + } + + perRPCCredentials, perr := grpcAuthenticator.PerRPCCredentials() + if perr != nil { + return nil, err + } + opts = append(opts, grpc.WithPerRPCCredentials(perRPCCredentials)) + } + + if gcs.BalancerName != "" { + valid := validateBalancerName(gcs.BalancerName) + if !valid { + return nil, fmt.Errorf("invalid balancer_name: %s", gcs.BalancerName) + } + opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingPolicy":"%s"}`, gcs.BalancerName))) + } + + if gcs.Authority != "" { + opts = append(opts, grpc.WithAuthority(gcs.Authority)) + } + + otelOpts := []otelgrpc.Option{ + otelgrpc.WithTracerProvider(settings.TracerProvider), + otelgrpc.WithMeterProvider(settings.MeterProvider), + otelgrpc.WithPropagators(otel.GetTextMapPropagator()), + } + + // Enable OpenTelemetry observability plugin. + opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelOpts...))) + + return opts, nil +} + +func validateBalancerName(balancerName string) bool { + return balancer.Get(balancerName) != nil +} + +// ToServer returns a grpc.Server for the configuration +// Deprecated: [0.96.0] Use ToServerContext instead. +func (gss *ServerConfig) ToServer(host component.Host, settings component.TelemetrySettings, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { + return gss.ToServerContext(context.Background(), host, settings, extraOpts...) +} + +// ToServerContext returns a grpc.Server for the configuration +func (gss *ServerConfig) ToServerContext(_ context.Context, host component.Host, settings component.TelemetrySettings, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { + opts, err := gss.toServerOption(host, settings) + if err != nil { + return nil, err + } + opts = append(opts, extraOpts...) + return grpc.NewServer(opts...), nil +} + +func shouldWarn(endpoint string) bool { + if endpoint == ":" { + // : (aka 0.0.0.0:0) + return true + } + + if strings.HasPrefix(endpoint, ":") { + // : (aka 0.0.0.0:) + _, err := strconv.ParseInt(endpoint[1:], 10, 64) + // If it's not a number, it's probably invalid, don't warn. + return err == nil + } + + // : + host, _, err := net.SplitHostPort(endpoint) + if err != nil { // Probably invalid, don't warn. + return false + } + + ip := net.ParseIP(host) + return ip != nil && ip.IsUnspecified() +} + +// WarnOnUnspecifiedHost emits a warning if an endpoint has an unspecified host. +func WarnOnUnspecifiedHost(logger *zap.Logger, endpoint string) { + if shouldWarn(endpoint) { + logger.Warn( + "Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks. Enable the feature gate to change the default and remove this warning.", + zap.String( + "documentation", + "https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks", + ), + zap.String("feature gate ID", "component.UseLocalHostAsDefaultHost"), + ) + } +} + +func (gss *ServerConfig) toServerOption(host component.Host, settings component.TelemetrySettings) ([]grpc.ServerOption, error) { + switch gss.NetAddr.Transport { + case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": + WarnOnUnspecifiedHost(settings.Logger, gss.NetAddr.Endpoint) + } + + var opts []grpc.ServerOption + + if gss.TLSSetting != nil { + tlsCfg, err := gss.TLSSetting.LoadTLSConfig() + if err != nil { + return nil, err + } + opts = append(opts, grpc.Creds(credentials.NewTLS(tlsCfg))) + } + + if gss.MaxRecvMsgSizeMiB > 0 { + opts = append(opts, grpc.MaxRecvMsgSize(int(gss.MaxRecvMsgSizeMiB*1024*1024))) + } + + if gss.MaxConcurrentStreams > 0 { + opts = append(opts, grpc.MaxConcurrentStreams(gss.MaxConcurrentStreams)) + } + + if gss.ReadBufferSize > 0 { + opts = append(opts, grpc.ReadBufferSize(gss.ReadBufferSize)) + } + + if gss.WriteBufferSize > 0 { + opts = append(opts, grpc.WriteBufferSize(gss.WriteBufferSize)) + } + + // The default values referenced in the GRPC docs are set within the server, so this code doesn't need + // to apply them over zero/nil values before passing these as grpc.ServerOptions. + // The following shows the server code for applying default grpc.ServerOptions. + // https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L184-L200 + if gss.Keepalive != nil { + if gss.Keepalive.ServerParameters != nil { + svrParams := gss.Keepalive.ServerParameters + opts = append(opts, grpc.KeepaliveParams(keepalive.ServerParameters{ + MaxConnectionIdle: svrParams.MaxConnectionIdle, + MaxConnectionAge: svrParams.MaxConnectionAge, + MaxConnectionAgeGrace: svrParams.MaxConnectionAgeGrace, + Time: svrParams.Time, + Timeout: svrParams.Timeout, + })) + } + // The default values referenced in the GRPC are set within the server, so this code doesn't need + // to apply them over zero/nil values before passing these as grpc.ServerOptions. + // The following shows the server code for applying default grpc.ServerOptions. + // https://github.com/grpc/grpc-go/blob/120728e1f775e40a2a764341939b78d666b08260/internal/transport/http2_server.go#L202-L205 + if gss.Keepalive.EnforcementPolicy != nil { + enfPol := gss.Keepalive.EnforcementPolicy + opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: enfPol.MinTime, + PermitWithoutStream: enfPol.PermitWithoutStream, + })) + } + } + + var uInterceptors []grpc.UnaryServerInterceptor + var sInterceptors []grpc.StreamServerInterceptor + + if gss.Auth != nil { + authenticator, err := gss.Auth.GetServerAuthenticator(host.GetExtensions()) + if err != nil { + return nil, err + } + + uInterceptors = append(uInterceptors, func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + return authUnaryServerInterceptor(ctx, req, info, handler, authenticator) + }) + sInterceptors = append(sInterceptors, func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return authStreamServerInterceptor(srv, ss, info, handler, authenticator) + }) + } + + if gss.MemoryLimiter != nil { + memoryLimiter, err := getMemoryLimiterExtension(gss.MemoryLimiter, host.GetExtensions()) + if err != nil { + return nil, err + } + + uInterceptors = append(uInterceptors, func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + return memoryLimiterUnaryServerInterceptor(ctx, req, info, handler, memoryLimiter) + }) + sInterceptors = append(sInterceptors, func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return memoryLimiterStreamServerInterceptor(srv, ss, info, handler, memoryLimiter) + }) + } + + otelOpts := []otelgrpc.Option{ + otelgrpc.WithTracerProvider(settings.TracerProvider), + otelgrpc.WithMeterProvider(settings.MeterProvider), + otelgrpc.WithPropagators(otel.GetTextMapPropagator()), + } + + // Enable OpenTelemetry observability plugin. + + uInterceptors = append(uInterceptors, enhanceWithClientInformation(gss.IncludeMetadata)) + sInterceptors = append(sInterceptors, enhanceStreamWithClientInformation(gss.IncludeMetadata)) + + opts = append(opts, grpc.StatsHandler(otelgrpc.NewServerHandler(otelOpts...)), grpc.ChainUnaryInterceptor(uInterceptors...), grpc.ChainStreamInterceptor(sInterceptors...)) + + return opts, nil +} + +func memoryLimiterUnaryServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ml memoryLimiterExtension) (any, error) { + if ml.MustRefuse() { + return nil, errMemoryLimitReached + } + + return handler(ctx, req) +} + +func memoryLimiterStreamServerInterceptor(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler, ml memoryLimiterExtension) error { + ctx := stream.Context() + if ml.MustRefuse() { + return errMemoryLimitReached + } + + return handler(srv, wrapServerStream(ctx, stream)) +} + +func getMemoryLimiterExtension(extID *component.ID, extensions map[component.ID]component.Component) (memoryLimiterExtension, error) { + if ext, found := extensions[*extID]; found { + if server, ok := ext.(memoryLimiterExtension); ok { + return server, nil + } + return nil, fmt.Errorf("requested MemoryLimiter, %s, is not a memoryLimiterExtension", extID) + } + + return nil, fmt.Errorf("failed to resolve memoryLimiterExtension %q: %s", extID, "memory limiter extension not found") +} + +// getGRPCCompressionName returns compression name registered in grpc. +func getGRPCCompressionName(compressionType configcompression.Type) (string, error) { + switch compressionType { + case configcompression.TypeGzip: + return gzip.Name, nil + case configcompression.TypeSnappy: + return snappy.Name, nil + case configcompression.TypeZstd: + return zstd.Name, nil + default: + return "", fmt.Errorf("unsupported compression type %q", compressionType) + } +} + +// enhanceWithClientInformation intercepts the incoming RPC, replacing the incoming context with one that includes +// a client.Info, potentially with the peer's address. +func enhanceWithClientInformation(includeMetadata bool) func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(contextWithClient(ctx, includeMetadata), req) + } +} + +func enhanceStreamWithClientInformation(includeMetadata bool) func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return handler(srv, wrapServerStream(contextWithClient(ss.Context(), includeMetadata), ss)) + } +} + +// contextWithClient attempts to add the peer address to the client.Info from the context. When no +// client.Info exists in the context, one is created. +func contextWithClient(ctx context.Context, includeMetadata bool) context.Context { + cl := client.FromContext(ctx) + if p, ok := peer.FromContext(ctx); ok { + cl.Addr = p.Addr + } + if includeMetadata { + if md, ok := metadata.FromIncomingContext(ctx); ok { + copiedMD := md.Copy() + if len(md[client.MetadataHostName]) == 0 && len(md[":authority"]) > 0 { + copiedMD[client.MetadataHostName] = md[":authority"] + } + cl.Metadata = client.NewMetadata(copiedMD) + } + } + return client.NewContext(ctx, cl) +} + +func authUnaryServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, server auth.Server) (any, error) { + headers, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMetadataNotFound + } + + ctx, err := server.Authenticate(ctx, headers) + if err != nil { + return nil, err + } + + return handler(ctx, req) +} + +func authStreamServerInterceptor(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler, server auth.Server) error { + ctx := stream.Context() + headers, ok := metadata.FromIncomingContext(ctx) + if !ok { + return errMetadataNotFound + } + + ctx, err := server.Authenticate(ctx, headers) + if err != nil { + return err + } + + return handler(srv, wrapServerStream(ctx, stream)) +} \ No newline at end of file diff --git a/collector/config/configgrpc/configgrpc_test.go b/collector/config/configgrpc/configgrpc_test.go new file mode 100644 index 00000000..d6ebdd17 --- /dev/null +++ b/collector/config/configgrpc/configgrpc_test.go @@ -0,0 +1,1338 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package configgrpc + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + + "go.opentelemetry.io/collector/client" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configauth" + "go.opentelemetry.io/collector/config/configcompression" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/extension/auth" + "go.opentelemetry.io/collector/extension/auth/authtest" + "go.opentelemetry.io/collector/extension/extensiontest" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" +) + +// testBalancerBuilder facilitates testing validateBalancerName(). +type testBalancerBuilder struct{} + +func (testBalancerBuilder) Build(balancer.ClientConn, balancer.BuildOptions) balancer.Balancer { + return nil +} + +func (testBalancerBuilder) Name() string { + return "configgrpc_balancer_test" +} + +func init() { + balancer.Register(testBalancerBuilder{}) +} + +var ( + componentID = component.MustNewID("component") + testAuthID = component.MustNewID("testauth") + mockID = component.MustNewID("mock") + doesntExistID = component.MustNewID("doesntexist") +) + +func TestDefaultGrpcClientSettings(t *testing.T) { + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + gcs := &ClientConfig{ + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + } + opts, err := gcs.toDialOptions(componenttest.NewNopHost(), tt.TelemetrySettings()) + assert.NoError(t, err) + assert.Len(t, opts, 2) +} + +func TestAllGrpcClientSettings(t *testing.T) { + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + tests := []struct { + settings ClientConfig + name string + host component.Host + }{ + { + name: "test all with gzip compression", + settings: ClientConfig{ + Headers: map[string]configopaque.String{ + "test": "test", + }, + Endpoint: "localhost:1234", + Compression: configcompression.TypeGzip, + TLSSetting: configtls.TLSClientSetting{ + Insecure: false, + }, + Keepalive: &KeepaliveClientConfig{ + Time: time.Second, + Timeout: time.Second, + PermitWithoutStream: true, + }, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + WaitForReady: true, + BalancerName: "round_robin", + Authority: "pseudo-authority", + Auth: &configauth.Authentication{AuthenticatorID: testAuthID}, + }, + host: &mockHost{ + ext: map[component.ID]component.Component{ + testAuthID: &authtest.MockClient{}, + }, + }, + }, + { + name: "test all with snappy compression", + settings: ClientConfig{ + Headers: map[string]configopaque.String{ + "test": "test", + }, + Endpoint: "localhost:1234", + Compression: configcompression.TypeSnappy, + TLSSetting: configtls.TLSClientSetting{ + Insecure: false, + }, + Keepalive: &KeepaliveClientConfig{ + Time: time.Second, + Timeout: time.Second, + PermitWithoutStream: true, + }, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + WaitForReady: true, + BalancerName: "round_robin", + Authority: "pseudo-authority", + Auth: &configauth.Authentication{AuthenticatorID: testAuthID}, + }, + host: &mockHost{ + ext: map[component.ID]component.Component{ + testAuthID: &authtest.MockClient{}, + }, + }, + }, + { + name: "test all with zstd compression", + settings: ClientConfig{ + Headers: map[string]configopaque.String{ + "test": "test", + }, + Endpoint: "localhost:1234", + Compression: configcompression.TypeZstd, + TLSSetting: configtls.TLSClientSetting{ + Insecure: false, + }, + Keepalive: &KeepaliveClientConfig{ + Time: time.Second, + Timeout: time.Second, + PermitWithoutStream: true, + }, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + WaitForReady: true, + BalancerName: "configgrpc_balancer_test", + Authority: "pseudo-authority", + Auth: &configauth.Authentication{AuthenticatorID: testAuthID}, + }, + host: &mockHost{ + ext: map[component.ID]component.Component{ + testAuthID: &authtest.MockClient{}, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + opts, err := test.settings.toDialOptions(test.host, tt.TelemetrySettings()) + assert.NoError(t, err) + assert.Len(t, opts, 9) + }) + } +} + +func TestDefaultGrpcServerSettings(t *testing.T) { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "0.0.0.0:1234", + }, + } + opts, err := gss.toServerOption(componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + assert.Len(t, opts, 3) +} + +func TestAllGrpcServerSettingsExceptAuth(t *testing.T) { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "localhost:1234", + Transport: "tcp", + }, + TLSSetting: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{}, + ClientCAFile: "", + }, + MaxRecvMsgSizeMiB: 1, + MaxConcurrentStreams: 1024, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + Keepalive: &KeepaliveServerConfig{ + ServerParameters: &KeepaliveServerParameters{ + MaxConnectionIdle: time.Second, + MaxConnectionAge: time.Second, + MaxConnectionAgeGrace: time.Second, + Time: time.Second, + Timeout: time.Second, + }, + EnforcementPolicy: &KeepaliveEnforcementPolicy{ + MinTime: time.Second, + PermitWithoutStream: true, + }, + }, + } + opts, err := gss.toServerOption(componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + assert.Len(t, opts, 10) +} + +func TestGrpcServerAuthSettings(t *testing.T) { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "0.0.0.0:1234", + }, + } + gss.Auth = &configauth.Authentication{ + AuthenticatorID: mockID, + } + host := &mockHost{ + ext: map[component.ID]component.Component{ + mockID: auth.NewServer(), + }, + } + srv, err := gss.ToServerContext(context.Background(), host, componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + assert.NotNil(t, srv) +} + +func TestGrpcServerAuthSettings_Deprecated(t *testing.T) { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "0.0.0.0:1234", + }, + } + gss.Auth = &configauth.Authentication{ + AuthenticatorID: mockID, + } + host := &mockHost{ + ext: map[component.ID]component.Component{ + mockID: auth.NewServer(), + }, + } + srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + assert.NotNil(t, srv) +} + +type mockMemoryLimiterExtension struct { + iD component.ID + mustRefuse bool + component.StartFunc + component.ShutdownFunc +} + +func (mml *mockMemoryLimiterExtension) MustRefuse() bool { + return mml.mustRefuse +} + +func TestGetMemoryLimiterExtension(t *testing.T) { + badID := component.NewID("badmemlimiter") + notMLExtensionErr := fmt.Errorf("requested MemoryLimiter, %s, is not a memoryLimiterExtension", badID) + + missingID := component.NewID("missingmemlimiter") + missingMLExtensionErr := fmt.Errorf("failed to resolve memoryLimiterExtension %q: %s", missingID, "memory limiter extension not found") + + validID := component.NewID("memorylimiter") + + tests := []struct { + name string + componentID component.ID + // getExtErr refers to whether the requested memory limiter extension was successfully found. + getExtErr error + }{ + { + name: "memory extension found", + componentID: validID, + getExtErr: nil, + }, + { + name: "not a memory limiter extension", + componentID: badID, + getExtErr: notMLExtensionErr, + }, + { + name: "memory limiter extension not found", + componentID: missingID, + getExtErr: missingMLExtensionErr, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + comID := test.componentID + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "localhost:0", + Transport: "tcp", + }, + MemoryLimiter: &comID, + } + + nopExt, err := extensiontest.NewNopFactory().CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), ServerConfig{}) + assert.NoError(t, err) + ml := &mockMemoryLimiterExtension{iD: comID} + extList := map[component.ID]component.Component{ + component.NewID("memorylimiter"): ml, + badID: nopExt, + } + + host := &mockHost{ + ext: extList, + } + + // ToServer calls getMemoryLimiterExtension(). + srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + + // desired extension was not found. + if test.getExtErr != nil { + assert.Equal(t, test.getExtErr.Error(), err.Error()) + assert.Nil(t, srv) + return + } + + assert.NoError(t, err) + }) + } + +} + +func TestGrpcUnaryServerMemoryLimiterSettings(t *testing.T) { + tests := []struct { + name string + refused bool + // interceptErr refers to whether memorylimiterextension allowed the request. + interceptErr error + }{ + { + name: "unary memory limiter extension accept", + refused: false, + interceptErr: nil, + }, + { + name: "unary memory limiter extension refuse", + refused: true, + interceptErr: errMemoryLimitReached, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + comID := component.NewID("memorylimiter") + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "localhost:0", + Transport: "tcp", + }, + MemoryLimiter: &comID, + } + + ml := &mockMemoryLimiterExtension{iD: comID, mustRefuse: test.refused} + extList := map[component.ID]component.Component{ + comID: ml, + } + + host := &mockHost{ + ext: extList, + } + + srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + + // found extension so finish setting up server and client to test interceptor. + assert.NoError(t, err) + mock := &grpcTraceServer{} + + ptraceotlp.RegisterGRPCServer(srv, mock) + + defer srv.Stop() + + l, err := gss.NetAddr.Listen(context.Background()) + require.NoError(t, err) + + go func() { + _ = srv.Serve(l) + }() + + // setup client + gcs := &ClientConfig{ + Endpoint: l.Addr().String(), + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + } + + tt, err := componenttest.SetupTelemetry(comID) + require.NoError(t, err) + defer func() { + require.NoError(t, tt.Shutdown(context.Background())) + }() + + grpcClientConn, errClient := gcs.ToClientConn(context.Background(), componenttest.NewNopHost(), tt.TelemetrySettings()) + require.NoError(t, errClient) + defer func() { assert.NoError(t, grpcClientConn.Close()) }() + + cl := ptraceotlp.NewGRPCClient(grpcClientConn) + ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelFunc() + + resp, errResp := cl.Export(ctx, ptraceotlp.NewExportRequest()) + assert.ErrorIs(t, test.interceptErr, errResp) + + if test.interceptErr != nil { + assert.Equal(t, resp, ptraceotlp.ExportResponse{}) + } else { + assert.NotNil(t, resp) + assert.NotEqual(t, resp, ptraceotlp.ExportResponse{}) + } + }) + } +} + +func TestGrpcStreamServerMemoryLimiterSettings(t *testing.T) { + tests := []struct { + name string + refused bool + // interceptErr refers to whether memorylimiterextension allowed the request. + interceptErr error + }{ + { + name: "stream memory limiter extension accept", + refused: false, + interceptErr: nil, + }, + { + name: "stream memory limiter extension refuse", + refused: true, + interceptErr: errMemoryLimitReached, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + comID := component.NewID("memorylimiter") + + ml := &mockMemoryLimiterExtension{iD: comID, mustRefuse: test.refused} + handlerCalled := false + handler := func(_ any, _ grpc.ServerStream) error { + handlerCalled = true + return nil + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) + streamServer := &mockServerStream{ + ctx: ctx, + } + + // test + err := memoryLimiterStreamServerInterceptor(nil, streamServer, &grpc.StreamServerInfo{}, handler, ml) + + // verify + assert.ErrorIs(t, test.interceptErr, err) + if test.refused { + assert.False(t, handlerCalled) + } else { + assert.True(t, handlerCalled) + } + }) + } + +} + +func TestGRPCClientSettingsError(t *testing.T) { + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + tests := []struct { + settings ClientConfig + err string + host component.Host + }{ + { + err: "^failed to load TLS config: failed to load CA CertPool File: failed to load cert /doesnt/exist:", + settings: ClientConfig{ + Headers: nil, + Endpoint: "", + Compression: "", + TLSSetting: configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: "/doesnt/exist", + }, + Insecure: false, + ServerName: "", + }, + Keepalive: nil, + }, + }, + { + err: "^failed to load TLS config: failed to load TLS cert and key: for auth via TLS, provide both certificate and key, or neither", + settings: ClientConfig{ + Headers: nil, + Endpoint: "", + Compression: "", + TLSSetting: configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CertFile: "/doesnt/exist", + }, + Insecure: false, + ServerName: "", + }, + Keepalive: nil, + }, + }, + { + err: "invalid balancer_name: test", + settings: ClientConfig{ + Headers: map[string]configopaque.String{ + "test": "test", + }, + Endpoint: "localhost:1234", + Compression: "gzip", + TLSSetting: configtls.TLSClientSetting{ + Insecure: false, + }, + Keepalive: &KeepaliveClientConfig{ + Time: time.Second, + Timeout: time.Second, + PermitWithoutStream: true, + }, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + WaitForReady: true, + BalancerName: "test", + }, + }, + { + err: "failed to resolve authenticator \"doesntexist\": authenticator not found", + settings: ClientConfig{ + Endpoint: "localhost:1234", + Auth: &configauth.Authentication{AuthenticatorID: doesntExistID}, + }, + host: &mockHost{ext: map[component.ID]component.Component{}}, + }, + { + err: "no extensions configuration available", + settings: ClientConfig{ + Endpoint: "localhost:1234", + Auth: &configauth.Authentication{AuthenticatorID: doesntExistID}, + }, + host: &mockHost{}, + }, + { + err: "unsupported compression type \"zlib\"", + settings: ClientConfig{ + Endpoint: "localhost:1234", + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + Compression: "zlib", + }, + host: &mockHost{}, + }, + { + err: "unsupported compression type \"deflate\"", + settings: ClientConfig{ + Endpoint: "localhost:1234", + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + Compression: "deflate", + }, + host: &mockHost{}, + }, + { + err: "unsupported compression type \"bad\"", + settings: ClientConfig{ + Endpoint: "localhost:1234", + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + Compression: "bad", + }, + host: &mockHost{}, + }, + } + for _, test := range tests { + t.Run(test.err, func(t *testing.T) { + _, err := test.settings.ToClientConn(context.Background(), test.host, tt.TelemetrySettings()) + assert.Error(t, err) + assert.Regexp(t, test.err, err) + }) + } +} + +func TestUseSecure(t *testing.T) { + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + gcs := &ClientConfig{ + Headers: nil, + Endpoint: "", + Compression: "", + TLSSetting: configtls.TLSClientSetting{}, + Keepalive: nil, + } + dialOpts, err := gcs.toDialOptions(componenttest.NewNopHost(), tt.TelemetrySettings()) + assert.NoError(t, err) + assert.Len(t, dialOpts, 2) +} + +func TestGRPCServerWarning(t *testing.T) { + tests := []struct { + name string + settings ServerConfig + len int + }{ + { + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "0.0.0.0:1234", + Transport: "tcp", + }, + }, + len: 1, + }, + { + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "127.0.0.1:1234", + Transport: "tcp", + }, + }, + len: 0, + }, + { + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "0.0.0.0:1234", + Transport: "unix", + }, + }, + len: 0, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + set := componenttest.NewNopTelemetrySettings() + logger, observed := observer.New(zap.DebugLevel) + set.Logger = zap.New(logger) + + opts, err := test.settings.toServerOption(componenttest.NewNopHost(), set) + require.NoError(t, err) + require.NotNil(t, opts) + _ = grpc.NewServer(opts...) + + require.Len(t, observed.FilterLevelExact(zap.WarnLevel).All(), test.len) + }) + } + +} + +func TestGRPCServerSettingsError(t *testing.T) { + tests := []struct { + settings ServerConfig + err string + }{ + { + err: "^failed to load TLS config: failed to load CA CertPool File: failed to load cert /doesnt/exist:", + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "127.0.0.1:1234", + Transport: "tcp", + }, + TLSSetting: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: "/doesnt/exist", + }, + }, + }, + }, + { + err: "^failed to load TLS config: failed to load TLS cert and key: for auth via TLS, provide both certificate and key, or neither", + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "127.0.0.1:1234", + Transport: "tcp", + }, + TLSSetting: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CertFile: "/doesnt/exist", + }, + }, + }, + }, + { + err: "^failed to load client CA CertPool: failed to load CA /doesnt/exist:", + settings: ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "127.0.0.1:1234", + Transport: "tcp", + }, + TLSSetting: &configtls.TLSServerSetting{ + ClientCAFile: "/doesnt/exist", + }, + }, + }, + } + for _, test := range tests { + t.Run(test.err, func(t *testing.T) { + _, err := test.settings.ToServerContext(context.Background(), componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + assert.Regexp(t, test.err, err) + }) + } +} + +func TestGRPCServerSettings_ToListener_Error(t *testing.T) { + settings := ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "127.0.0.1:1234567", + Transport: "tcp", + }, + } + _, err := settings.NetAddr.Listen(context.Background()) + assert.Error(t, err) +} + +func TestHttpReception(t *testing.T) { + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + tests := []struct { + name string + tlsServerCreds *configtls.TLSServerSetting + tlsClientCreds *configtls.TLSClientSetting + hasError bool + }{ + { + name: "noTLS", + tlsServerCreds: nil, + tlsClientCreds: &configtls.TLSClientSetting{ + Insecure: true, + }, + }, + { + name: "TLS", + tlsServerCreds: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "server.crt"), + KeyFile: filepath.Join("testdata", "server.key"), + }, + }, + tlsClientCreds: &configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + }, + ServerName: "localhost", + }, + }, + { + name: "NoServerCertificates", + tlsServerCreds: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + }, + }, + tlsClientCreds: &configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + }, + ServerName: "localhost", + }, + hasError: true, + }, + { + name: "mTLS", + tlsServerCreds: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "server.crt"), + KeyFile: filepath.Join("testdata", "server.key"), + }, + ClientCAFile: filepath.Join("testdata", "ca.crt"), + }, + tlsClientCreds: &configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "client.crt"), + KeyFile: filepath.Join("testdata", "client.key"), + }, + ServerName: "localhost", + }, + }, + { + name: "NoClientCertificate", + tlsServerCreds: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "server.crt"), + KeyFile: filepath.Join("testdata", "server.key"), + }, + ClientCAFile: filepath.Join("testdata", "ca.crt"), + }, + tlsClientCreds: &configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + }, + ServerName: "localhost", + }, + hasError: true, + }, + { + name: "WrongClientCA", + tlsServerCreds: &configtls.TLSServerSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "server.crt"), + KeyFile: filepath.Join("testdata", "server.key"), + }, + ClientCAFile: filepath.Join("testdata", "server.crt"), + }, + tlsClientCreds: &configtls.TLSClientSetting{ + TLSSetting: configtls.TLSSetting{ + CAFile: filepath.Join("testdata", "ca.crt"), + CertFile: filepath.Join("testdata", "client.crt"), + KeyFile: filepath.Join("testdata", "client.key"), + }, + ServerName: "localhost", + }, + hasError: true, + }, + } + // prepare + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "localhost:0", + Transport: "tcp", + }, + TLSSetting: test.tlsServerCreds, + } + ln, err := gss.NetAddr.Listen(context.Background()) + assert.NoError(t, err) + s, err := gss.ToServerContext(context.Background(), componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + ptraceotlp.RegisterGRPCServer(s, &grpcTraceServer{}) + + go func() { + _ = s.Serve(ln) + }() + + gcs := &ClientConfig{ + Endpoint: ln.Addr().String(), + TLSSetting: *test.tlsClientCreds, + } + grpcClientConn, errClient := gcs.ToClientConn(context.Background(), componenttest.NewNopHost(), tt.TelemetrySettings()) + assert.NoError(t, errClient) + defer func() { assert.NoError(t, grpcClientConn.Close()) }() + c := ptraceotlp.NewGRPCClient(grpcClientConn) + ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Second) + resp, errResp := c.Export(ctx, ptraceotlp.NewExportRequest(), grpc.WaitForReady(true)) + if test.hasError { + assert.Error(t, errResp) + } else { + assert.NoError(t, errResp) + assert.NotNil(t, resp) + } + cancelFunc() + s.Stop() + }) + } +} + +func TestReceiveOnUnixDomainSocket(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows") + } + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, tt.Shutdown(context.Background())) }) + + socketName := tempSocketName(t) + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: socketName, + Transport: "unix", + }, + } + ln, err := gss.NetAddr.Listen(context.Background()) + assert.NoError(t, err) + srv, err := gss.ToServerContext(context.Background(), componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + assert.NoError(t, err) + ptraceotlp.RegisterGRPCServer(srv, &grpcTraceServer{}) + + go func() { + _ = srv.Serve(ln) + }() + + gcs := &ClientConfig{ + Endpoint: "unix://" + ln.Addr().String(), + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + } + grpcClientConn, errClient := gcs.ToClientConn(context.Background(), componenttest.NewNopHost(), tt.TelemetrySettings()) + assert.NoError(t, errClient) + defer func() { assert.NoError(t, grpcClientConn.Close()) }() + c := ptraceotlp.NewGRPCClient(grpcClientConn) + ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Second) + resp, errResp := c.Export(ctx, ptraceotlp.NewExportRequest(), grpc.WaitForReady(true)) + assert.NoError(t, errResp) + assert.NotNil(t, resp) + cancelFunc() + srv.Stop() +} + +func TestContextWithClient(t *testing.T) { + testCases := []struct { + desc string + input context.Context + doMetadata bool + expected client.Info + }{ + { + desc: "no peer information, empty client", + input: context.Background(), + expected: client.Info{}, + }, + { + desc: "existing client with IP, no peer information", + input: client.NewContext(context.Background(), client.Info{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 4), + }, + }), + expected: client.Info{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 4), + }, + }, + }, + { + desc: "empty client, with peer information", + input: peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 4), + }, + }), + expected: client.Info{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 4), + }, + }, + }, + { + desc: "existing client, existing IP gets overridden with peer information", + input: peer.NewContext(client.NewContext(context.Background(), client.Info{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 4), + }, + }), &peer.Peer{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 5), + }, + }), + expected: client.Info{ + Addr: &net.IPAddr{ + IP: net.IPv4(1, 2, 3, 5), + }, + }, + }, + { + desc: "existing client with metadata", + input: client.NewContext(context.Background(), client.Info{ + Metadata: client.NewMetadata(map[string][]string{"test-metadata-key": {"test-value"}}), + }), + doMetadata: true, + expected: client.Info{ + Metadata: client.NewMetadata(map[string][]string{"test-metadata-key": {"test-value"}}), + }, + }, + { + desc: "existing client with metadata in context", + input: metadata.NewIncomingContext( + client.NewContext(context.Background(), client.Info{}), + metadata.Pairs("test-metadata-key", "test-value"), + ), + doMetadata: true, + expected: client.Info{ + Metadata: client.NewMetadata(map[string][]string{"test-metadata-key": {"test-value"}}), + }, + }, + { + desc: "existing client with metadata in context, no metadata processing", + input: metadata.NewIncomingContext( + client.NewContext(context.Background(), client.Info{}), + metadata.Pairs("test-metadata-key", "test-value"), + ), + expected: client.Info{}, + }, + { + desc: "existing client with Host and metadata", + input: metadata.NewIncomingContext( + client.NewContext(context.Background(), client.Info{}), + metadata.Pairs("test-metadata-key", "test-value", ":authority", "localhost:55443"), + ), + doMetadata: true, + expected: client.Info{ + Metadata: client.NewMetadata(map[string][]string{"test-metadata-key": {"test-value"}, ":authority": {"localhost:55443"}, "Host": {"localhost:55443"}}), + }, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + cl := client.FromContext(contextWithClient(tC.input, tC.doMetadata)) + assert.Equal(t, tC.expected, cl) + }) + } +} + +func TestStreamInterceptorEnhancesClient(t *testing.T) { + // prepare + inCtx := peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.IPAddr{IP: net.IPv4(1, 1, 1, 1)}, + }) + var outContext context.Context + + stream := &mockedStream{ + ctx: inCtx, + } + + handler := func(_ any, stream grpc.ServerStream) error { + outContext = stream.Context() + return nil + } + + // test + err := enhanceStreamWithClientInformation(false)(nil, stream, nil, handler) + + // verify + assert.NoError(t, err) + + cl := client.FromContext(outContext) + assert.Equal(t, "1.1.1.1", cl.Addr.String()) +} + +type mockedStream struct { + ctx context.Context + grpc.ServerStream +} + +func (ms *mockedStream) Context() context.Context { + return ms.ctx +} + +func TestClientInfoInterceptors(t *testing.T) { + testCases := []struct { + desc string + tester func(context.Context, ptraceotlp.GRPCClient) + }{ + { + // we only have unary services, we don't have any clients we could use + // to test with streaming services + desc: "unary", + tester: func(ctx context.Context, cl ptraceotlp.GRPCClient) { + resp, errResp := cl.Export(ctx, ptraceotlp.NewExportRequest()) + require.NoError(t, errResp) + require.NotNil(t, resp) + }, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + mock := &grpcTraceServer{} + var l net.Listener + + // prepare the server + { + gss := &ServerConfig{ + NetAddr: confignet.NetAddr{ + Endpoint: "localhost:0", + Transport: "tcp", + }, + } + srv, err := gss.ToServerContext(context.Background(), componenttest.NewNopHost(), componenttest.NewNopTelemetrySettings()) + require.NoError(t, err) + ptraceotlp.RegisterGRPCServer(srv, mock) + + defer srv.Stop() + + l, err = gss.NetAddr.Listen(context.Background()) + require.NoError(t, err) + + go func() { + _ = srv.Serve(l) + }() + } + + // prepare the client and execute a RPC + { + gcs := &ClientConfig{ + Endpoint: l.Addr().String(), + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + } + + tt, err := componenttest.SetupTelemetry(componentID) + require.NoError(t, err) + defer func() { + require.NoError(t, tt.Shutdown(context.Background())) + }() + + grpcClientConn, errClient := gcs.ToClientConn(context.Background(), componenttest.NewNopHost(), tt.TelemetrySettings()) + require.NoError(t, errClient) + defer func() { assert.NoError(t, grpcClientConn.Close()) }() + + cl := ptraceotlp.NewGRPCClient(grpcClientConn) + ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelFunc() + + // test + tC.tester(ctx, cl) + } + + // verify + cl := client.FromContext(mock.recordedContext) + + // the client address is something like 127.0.0.1:41086 + assert.Contains(t, cl.Addr.String(), "127.0.0.1") + }) + } +} + +func TestDefaultUnaryInterceptorAuthSucceeded(t *testing.T) { + // prepare + handlerCalled := false + authCalled := false + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + authCalled = true + ctx := client.NewContext(context.Background(), client.Info{ + Addr: &net.IPAddr{IP: net.IPv4(1, 2, 3, 4)}, + }) + + return ctx, nil + } + handler := func(ctx context.Context, _ any) (any, error) { + handlerCalled = true + cl := client.FromContext(ctx) + assert.Equal(t, "1.2.3.4", cl.Addr.String()) + return nil, nil + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) + + // test + res, err := authUnaryServerInterceptor(ctx, nil, &grpc.UnaryServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.Nil(t, res) + assert.NoError(t, err) + assert.True(t, authCalled) + assert.True(t, handlerCalled) +} + +func TestDefaultUnaryInterceptorAuthFailure(t *testing.T) { + // prepare + authCalled := false + expectedErr := errors.New("not authenticated") + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + authCalled = true + return context.Background(), expectedErr + } + handler := func(context.Context, any) (any, error) { + assert.FailNow(t, "the handler should not have been called on auth failure!") + return nil, nil + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) + + // test + res, err := authUnaryServerInterceptor(ctx, nil, &grpc.UnaryServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.Nil(t, res) + assert.Equal(t, expectedErr, err) + assert.True(t, authCalled) +} + +func TestDefaultUnaryInterceptorMissingMetadata(t *testing.T) { + // prepare + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + assert.FailNow(t, "the auth func should not have been called!") + return context.Background(), nil + } + handler := func(context.Context, any) (any, error) { + assert.FailNow(t, "the handler should not have been called!") + return nil, nil + } + + // test + res, err := authUnaryServerInterceptor(context.Background(), nil, &grpc.UnaryServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.Nil(t, res) + assert.Equal(t, errMetadataNotFound, err) +} + +func TestDefaultStreamInterceptorAuthSucceeded(t *testing.T) { + // prepare + handlerCalled := false + authCalled := false + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + authCalled = true + ctx := client.NewContext(context.Background(), client.Info{ + Addr: &net.IPAddr{IP: net.IPv4(1, 2, 3, 4)}, + }) + return ctx, nil + } + handler := func(_ any, stream grpc.ServerStream) error { + // ensure that the client information is propagated down to the underlying stream + cl := client.FromContext(stream.Context()) + assert.Equal(t, "1.2.3.4", cl.Addr.String()) + handlerCalled = true + return nil + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) + streamServer := &mockServerStream{ + ctx: ctx, + } + + // test + err := authStreamServerInterceptor(nil, streamServer, &grpc.StreamServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.NoError(t, err) + assert.True(t, authCalled) + assert.True(t, handlerCalled) +} + +func TestDefaultStreamInterceptorAuthFailure(t *testing.T) { + // prepare + authCalled := false + expectedErr := errors.New("not authenticated") + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + authCalled = true + return context.Background(), expectedErr + } + handler := func(any, grpc.ServerStream) error { + assert.FailNow(t, "the handler should not have been called on auth failure!") + return nil + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) + streamServer := &mockServerStream{ + ctx: ctx, + } + + // test + err := authStreamServerInterceptor(nil, streamServer, &grpc.StreamServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.Equal(t, expectedErr, err) + assert.True(t, authCalled) +} + +func TestDefaultStreamInterceptorMissingMetadata(t *testing.T) { + // prepare + authFunc := func(context.Context, map[string][]string) (context.Context, error) { + assert.FailNow(t, "the auth func should not have been called!") + return context.Background(), nil + } + handler := func(any, grpc.ServerStream) error { + assert.FailNow(t, "the handler should not have been called!") + return nil + } + streamServer := &mockServerStream{ + ctx: context.Background(), + } + + // test + err := authStreamServerInterceptor(nil, streamServer, &grpc.StreamServerInfo{}, handler, auth.NewServer(auth.WithServerAuthenticate(authFunc))) + + // verify + assert.Equal(t, errMetadataNotFound, err) +} + +type mockServerStream struct { + grpc.ServerStream + ctx context.Context +} + +func (m *mockServerStream) Context() context.Context { + return m.ctx +} + +type grpcTraceServer struct { + ptraceotlp.UnimplementedGRPCServer + recordedContext context.Context +} + +func (gts *grpcTraceServer) Export(ctx context.Context, _ ptraceotlp.ExportRequest) (ptraceotlp.ExportResponse, error) { + gts.recordedContext = ctx + return ptraceotlp.NewExportResponse(), nil +} + +// tempSocketName provides a temporary Unix socket name for testing. +func tempSocketName(t *testing.T) string { + tmpfile, err := os.CreateTemp("", "sock") + require.NoError(t, err) + require.NoError(t, tmpfile.Close()) + socket := tmpfile.Name() + require.NoError(t, os.Remove(socket)) + return socket +} + +type mockHost struct { + component.Host + ext map[component.ID]component.Component +} + +func (nh *mockHost) GetExtensions() map[component.ID]component.Component { + return nh.ext +} \ No newline at end of file diff --git a/collector/config/configgrpc/testdata/ca.crt b/collector/config/configgrpc/testdata/ca.crt new file mode 100644 index 00000000..4bee456e --- /dev/null +++ b/collector/config/configgrpc/testdata/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNjCCAh4CCQC0I5IQT7eziDANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJB +VTESMBAGA1UECAwJQXVzdHJhbGlhMQ8wDQYDVQQHDAZTeWRuZXkxEjAQBgNVBAoM +CU15T3JnTmFtZTEVMBMGA1UEAwwMTXlDb21tb25OYW1lMB4XDTIyMDgwMzA0MTky +MVoXDTMyMDczMTA0MTkyMVowXTELMAkGA1UEBhMCQVUxEjAQBgNVBAgMCUF1c3Ry +YWxpYTEPMA0GA1UEBwwGU3lkbmV5MRIwEAYDVQQKDAlNeU9yZ05hbWUxFTATBgNV +BAMMDE15Q29tbW9uTmFtZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AMhGP0dy3zvkdx9zI+/XVjPOWlER0OUp7Sgzidc3nLOk42+bH4ofIVNtOFVqlNKi +O1bImu238VdBhd6R5IZZ1ZdIMcCeDgSJYu2X9wA3m4PKz8IdXo5ly2OHghhmCvqG +WxgqDj5wPXiczQwuf1EcDMtRWbXJ6Z/XH1U68R/kRdNLkiZ2LwtjoQpis5XYckLL +CrdF+AL6GeDIe0Mh9QGs26Vux+2kvaOGNUWRPE6Wt4GkqyKqmzYfR9HbflJ4xHT2 +I+jE1lg+jMBeom7z8Z90RE4GGcHjO+Vens/88r5EAjTnFj1Kb5gL2deSHY1m/++R +Z/kRyg+zQJyw4fAzlAA4+VkCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAM3gRdTKX +eGwGYVmmKqA2vTxeigQYLHml7OSopcWj2wJfxfp49HXPRuvgpQn9iubxO3Zmhd83 +2X1E+T0A8oy5CfxgpAhHb3lY0jm3TjKXm6m+dSODwL3uND8tX+SqR8sRTFxPvPuo +pmvhdTZoRI3EzIiHLTgCuSU25JNP/vrVoKk0JvCkDYTU/WcVfj0v95DTMoWR4JGz +mtBwrgD0EM2XRw5ZMc7sMPli1gqmCbCQUrDZ+rPB78WDCBILBd8Cz75qYTUp98BY +akJyBckdJHAdyEQYDKa9HpmpexOO7IhSXCTEN1DEBgpZgEi/lBDRG/b0OzenUUgt +LUABtWt3pNQ9HA== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/collector/config/configgrpc/testdata/client.crt b/collector/config/configgrpc/testdata/client.crt new file mode 100644 index 00000000..c10abc1b --- /dev/null +++ b/collector/config/configgrpc/testdata/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVTCCAj2gAwIBAgIJANt5fkUlfxyiMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAkFVMRIwEAYDVQQIDAlBdXN0cmFsaWExDzANBgNVBAcMBlN5ZG5leTESMBAG +A1UECgwJTXlPcmdOYW1lMRUwEwYDVQQDDAxNeUNvbW1vbk5hbWUwHhcNMjIwODAz +MDQxOTIxWhcNMzIwNzMxMDQxOTIxWjBdMQswCQYDVQQGEwJBVTESMBAGA1UECAwJ +QXVzdHJhbGlhMQ8wDQYDVQQHDAZTeWRuZXkxEjAQBgNVBAoMCU15T3JnTmFtZTEV +MBMGA1UEAwwMTXlDb21tb25OYW1lMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAm/gURxkdWTDS0TyL2j920SfOtOZIo7DjubWLbZtNLrNCZNBsV+8c/ko/ +wleWmUJQRHeiZkNFs8TK6d8Grks6ta9oNO4CiCCO1kz4QidA827cL5+WaKWEVn8Y +Z8aiEMjDOnpYnb/ycsXpERN/P22jHpFD3DKSwLXoXQvasbSJsZro+AIaPAurFB7W +rMagCptwzGQDzryqVKEmXo+eN4XRxsoE8yroHsGbQ8GCZ+neftgV3Jhi1qcXZ//A +3ApY5lg06n1A03fYBlXE5L9tYKpIRNl2kq45mJ8DX6Tdp4Z1Y15+keIIyQpx4LRf +rtdbMQNJhBFOwpAajTmaKXxeICFRHQIDAQABoxgwFjAUBgNVHREEDTALgglsb2Nh +bGhvc3QwDQYJKoZIhvcNAQELBQADggEBAKWrbMxms658R/wYwLxzWPrZVKFswOJX +TpSkXGkyRnrhhZi3I8EhLZhlpZ9k8dplcvseVAUdX9hJu0BaDWBiW/VlPVUkWpWR +QZzrssAKhmSYMgl3OiayU30vL9bxYsAX9KeOJfnJ4kWoBpnguToED7wrC1lbzrVK +Vj1AiI3hBdKUdPNO0hyb8yfxbP3MOottMkk89DIebtOhqj2KEU7sKrhW9a5P5D7d +0A+0kf/IunUZ4IYFfha6qy0gRMyayfm9ttrPAY6q3faqtWR7nY87/T/7wHr1LQ1/ +Q622p7v3j3y75lGN50kFnSd77ykag/8avEKxOTFoGOQc5VCRYJnJwb4= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/collector/config/configgrpc/testdata/client.key b/collector/config/configgrpc/testdata/client.key new file mode 100644 index 00000000..31192c20 --- /dev/null +++ b/collector/config/configgrpc/testdata/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAm/gURxkdWTDS0TyL2j920SfOtOZIo7DjubWLbZtNLrNCZNBs +V+8c/ko/wleWmUJQRHeiZkNFs8TK6d8Grks6ta9oNO4CiCCO1kz4QidA827cL5+W +aKWEVn8YZ8aiEMjDOnpYnb/ycsXpERN/P22jHpFD3DKSwLXoXQvasbSJsZro+AIa +PAurFB7WrMagCptwzGQDzryqVKEmXo+eN4XRxsoE8yroHsGbQ8GCZ+neftgV3Jhi +1qcXZ//A3ApY5lg06n1A03fYBlXE5L9tYKpIRNl2kq45mJ8DX6Tdp4Z1Y15+keII +yQpx4LRfrtdbMQNJhBFOwpAajTmaKXxeICFRHQIDAQABAoIBAQCWxrT7omi/vzYd +9dUQ8Acx3LS0JmaUb71F2x3loJt1iO+nO+FxBIPXw/ltK3U3xWaJOcnx6Biq15R9 +kBAKUEl6OA6aFHi4FhlfS9s3QHFGo6YSF8m0ckXDxGvYbqpfZWVt07Z1EYkUsQRF +cL6zl454T1/1r6I0z+XIhVwuLGRsHt2+GCwSrLMnF9aTUJvPFy5G7YlxmL5q1BFu +F70AK9FLZcYqa5nP1F1HbIQB/zsQ8admpKIy5tjaZiLgctXv2GTzzXDEwEnaJMrq +SPr1dGDhdGs5iYRMOMT5Pp9dIG2+ZSSMHFAn4IRoB/cPJbNEUkgwQOPmDYETqSg5 +tSjfIUw1AoGBAMjE6PlT1/orlHW4QvKmV73YtKPVfi1Coo/F8G45qFaHDkc6bI9W +ySrnvqWcPs++xOZMoGtLuESw/LEluFo8vMX8aQYrVSz4Pb7AvuYbBRE0EVVui7YB +3B1O0c7QTabmfQYeATYD7qSShLccSpUE+FQa6NdrJOxddJLM0Z9K73q7AoGBAMbg +I2+NYB6XME7tyStOS4pkA4y7brG/M8BCaY34nfOJT4Qh6pqZRnDJ4ReopGoXEqWg +hwFRsBNhsji4GGejRBPnYcfJTSuMXSPromgoH1tR0OQbMJB0pCTavbI9j+endlv4 +/P+KV3ZMYOLhL/gaaTG+o6Wh2ehnE/8/rmqGpoIHAoGAHXVG+c5jkkFytxMiP5hI +p4J0ftWEff+Y+p+Ad6veF1QZtDnOU/nX6oO2ZXZXgQPswB3eK+AgWXPen994/USM +LkCq6EzTYpXJ+YMuf3TXeX66TF68ASiks2gtQLsvqZ2IGq2sX9CT43HcJ0Hvb44b +IbwRDgqakFPmFuQWndjQ6qECgYA9bOlFATOY/zWKi2NBHvOyEOYPx6yO9fF0Bo83 +rHyMxfJra1Zc3c6l85S0jAAMTIgT5BsOyz5JHjm/zwyqpgDW7PaEkKZnNvllqNgG +t63HtOOCMOu1EnHIeE9zCBS0hkLGcYcjHoWZIkoiiU8ZoH6xQKKm+/CkGYJRqkei +22f+bQKBgGHq2/ZzgxfblD3blKWp8mh5Kw7c/2VwJRvLEMlgzrnRgF7QNhEcH3Jm +aD/pqzAkqHnLVVQ5ogMKrrLl11jQp4kX74+Ps2Yul7UgzXFYy020mQSpJF/FMjrl +PEqwfCiOT2nLyE30x9VClUOGXy1CxH52Yn/g81ENq3jKTptwh+fI +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/collector/config/configgrpc/testdata/server.crt b/collector/config/configgrpc/testdata/server.crt new file mode 100644 index 00000000..87472e32 --- /dev/null +++ b/collector/config/configgrpc/testdata/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVTCCAj2gAwIBAgIJANt5fkUlfxyhMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAkFVMRIwEAYDVQQIDAlBdXN0cmFsaWExDzANBgNVBAcMBlN5ZG5leTESMBAG +A1UECgwJTXlPcmdOYW1lMRUwEwYDVQQDDAxNeUNvbW1vbk5hbWUwHhcNMjIwODAz +MDQxOTIxWhcNMzIwNzMxMDQxOTIxWjBdMQswCQYDVQQGEwJBVTESMBAGA1UECAwJ +QXVzdHJhbGlhMQ8wDQYDVQQHDAZTeWRuZXkxEjAQBgNVBAoMCU15T3JnTmFtZTEV +MBMGA1UEAwwMTXlDb21tb25OYW1lMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAv1Pm2elKl/IpJlX5NqQjRTlA1rHws8F7v1IuaXB2qfk1MsDCt37OvlbR +4ARrY6zdUIrEQ/wrhQsZ2M5/yaj0rfeCgd/SDUKMAqDvXQXBY2AaLubTAIEMs4rF +R5Zq/pcBNz1zu8kRvRgvVuOpTCPR1kRvKFWAp689lXZhUU/BrQQXhdA993xVMRM9 +u4fZuJLxNGGR/EhqTec4Z65jAZiUfO7ID94PtaxTrzR/Kjr2CiceR5hwdY40Bcme +D3IAd0J6nN1zIihe+Nqg/ImOG7YS+efQIEWJ8eHOoCK5knFBXRy6WwxeCyAPXyIb +DTrqTy67eTDYc0XZ24F/5Q3GSvfMDwIDAQABoxgwFjAUBgNVHREEDTALgglsb2Nh +bGhvc3QwDQYJKoZIhvcNAQELBQADggEBAKeFHP5rQasRS/XGbPkobfbFyTdGnLay +0Vr6+Rs5+4siKlAIhuUP9A/De61CEkFj8NFi2bmXYv8q3qP/z0lrjw7btrvD7Qc7 +lth73k3U2sUVZoqbYQZz0GHCWfZm8yXjP63SKI+81LHbS40ArO0R44BLc9TbbRiR +/LwO/x2+cxs28KdsEkU6jQ6Ly5jyoxw1ysoIeRfIk+FnQD4w29TyGgtX/G15/NN0 +ytByIZ8wdbUciunQc3nPXoPc41N+hyi2GZaXMuJ4VlsNmgY+wPmp4y3pl4l0bgCb +1FR8Vvtsi8jLH8J15oAMWdmHQKcoJDE49llx+bQGpNekp6mlfX1DIPI= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/collector/config/configgrpc/testdata/server.key b/collector/config/configgrpc/testdata/server.key new file mode 100644 index 00000000..58aab75e --- /dev/null +++ b/collector/config/configgrpc/testdata/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAv1Pm2elKl/IpJlX5NqQjRTlA1rHws8F7v1IuaXB2qfk1MsDC +t37OvlbR4ARrY6zdUIrEQ/wrhQsZ2M5/yaj0rfeCgd/SDUKMAqDvXQXBY2AaLubT +AIEMs4rFR5Zq/pcBNz1zu8kRvRgvVuOpTCPR1kRvKFWAp689lXZhUU/BrQQXhdA9 +93xVMRM9u4fZuJLxNGGR/EhqTec4Z65jAZiUfO7ID94PtaxTrzR/Kjr2CiceR5hw +dY40BcmeD3IAd0J6nN1zIihe+Nqg/ImOG7YS+efQIEWJ8eHOoCK5knFBXRy6Wwxe +CyAPXyIbDTrqTy67eTDYc0XZ24F/5Q3GSvfMDwIDAQABAoIBAQC/BuxlAhKiJvyC +9DABKFy2zvU35y3mq/X8Dfec+tbf2pwM8nz3bLrLPDAMNR1rxbqqogJXxr1E9tJ1 +r6fTFshFsewx8+DrsFfOgBS9kfOGXvuFfJ2L0U13LcTPNxXY37gtCUQ2aAk3/Z+2 +Z1QvW0w1XNqHMOdlhQg95JZB8xnyvXs1niLT/I9d7KbPBmOWkB5Jp7+JaebmWqNS +alxnNqYnhXcrNSAbuR4bgz0l4I+Jprms26C6sakmgCfeMjfWbd2k3tp06vKXmT6q +qKa0855axP9wuSbKbscTDW5RFYTYnu/CSYJ4nZtzSS8a559iG3m61EgPOoVTnTX6 +0t0I+kwRAoGBAN403NO4FfHG8k2bFpbATQkmC9UwjMbl5RIEL0fFhNVsuM5jTwHc +0wlFm9tMN8xqg66OFCimC/mUNPWX8nrb/MwrAw6/50rbyqOBFnmFKIfVf4ftpLzt +BLhEg7a/FPgdDgldQD+C6XbMyBA1AF3nbpTnbnj5WVQTl672s9teegSzAoGBANxs +1y6Nfh2DyyU16p61376AAP3WfHvuBgJAC0xGCqoTrbyzl4/r06BTMl51PbWJLDjm +FryTtgM7a8XO1jwfWJno71dnT7Bsy+wYnmJ5+9XHwgO9oZfSFUk+ELrEImI/4NZX +dJLkc0SuCG/wa3Wa76+sFNlzAzBBs83RE2j432E1AoGAF+x5GhJnynAxBkn8VJ6/ +rIx8GafwgDmgQCBTNtb9Rj0+aHoot3qe/hCQhzvdhhSxuMlzQi0efPCIAyko4jFt +Nk4rNhtTO6wOVSxAzzSW+Ij0Ah6D7hNWvsAhrjtEdrIqILf5gt0FZdUGdTg/odyY ++08vhbbS90pkumG1W5kAaiECgYEAqjk3eBD26u4jjIn1tTk5H9GUcnMYUVCAvW4e +C3ovtCZcTlTW3+M73B1D0aRy0mWrjAlMV7cuoZJa6TiRQ37lmn5Dj1kONm3ekWZ1 +shEIBZEtaFwila88lwJiQwlCkGNKS9zf/qyDw+8uPtwI8JqFLUIUG9VxCexDYddr +SO6g+10CgYEAwUs2BRJb6Od+8XtH32+8DDOnpfWJARY0CwogN2k+D1dbAB8Wkda1 +BMADasAcjDFRX6xvyyDlqxcDIoCI1JvpS82I/PTNHeT8pEr5Caln7OHD/BtnPwmI +YR0bvKkoN0jdQdjifpMVXEbJS1VfFLdQYQ8iQMwZfkFmzIkYpvqWVtw= +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/collector/config/configgrpc/wrappedstream.go b/collector/config/configgrpc/wrappedstream.go new file mode 100644 index 00000000..6b229d3b --- /dev/null +++ b/collector/config/configgrpc/wrappedstream.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package configgrpc // import "go.opentelemetry.io/collector/config/configgrpc" + +import ( + "context" + + "google.golang.org/grpc" +) + +// this functionality was originally copied from grpc-ecosystem/go-grpc-middleware project + +// wrappedServerStream is a thin wrapper around grpc.ServerStream that allows modifying context. +type wrappedServerStream struct { + grpc.ServerStream + // wrappedContext is the wrapper's own Context. You can assign it. + wrappedCtx context.Context +} + +// Context returns the wrapper's wrappedContext, overwriting the nested grpc.ServerStream.Context() +func (w *wrappedServerStream) Context() context.Context { + return w.wrappedCtx +} + +// wrapServerStream returns a ServerStream with the new context. +func wrapServerStream(wrappedCtx context.Context, stream grpc.ServerStream) *wrappedServerStream { + if existing, ok := stream.(*wrappedServerStream); ok { + existing.wrappedCtx = wrappedCtx + return existing + } + return &wrappedServerStream{ServerStream: stream, wrappedCtx: wrappedCtx} +} \ No newline at end of file diff --git a/collector/config/configgrpc/wrappedstream_test.go b/collector/config/configgrpc/wrappedstream_test.go new file mode 100644 index 00000000..3e4554bf --- /dev/null +++ b/collector/config/configgrpc/wrappedstream_test.go @@ -0,0 +1,43 @@ +// Copyright The OpenTelemetry Authors +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package configgrpc // import "go.opentelemetry.io/collector/internal/middleware" + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" +) + +type ctxKey struct{} + +var oneCtxKey = ctxKey{} +var otherCtxKey = ctxKey{} + +func TestWrapServerStream(t *testing.T) { + ctx := context.WithValue(context.TODO(), oneCtxKey, 1) + fake := &fakeServerStream{ctx: ctx} + assert.NotNil(t, fake.Context().Value(oneCtxKey), "values from fake must propagate to wrapper") + wrapped := wrapServerStream(context.WithValue(fake.Context(), otherCtxKey, 2), fake) + assert.NotNil(t, wrapped.Context().Value(oneCtxKey), "values from wrapper must be set") + assert.NotNil(t, wrapped.Context().Value(otherCtxKey), "values from wrapper must be set") +} + +func TestDoubleWrapping(t *testing.T) { + fake := &fakeServerStream{ctx: context.Background()} + wrapped := wrapServerStream(fake.Context(), fake) + assert.Same(t, wrapped, wrapServerStream(wrapped.Context(), wrapped)) // should be noop + assert.Equal(t, fake, wrapped.ServerStream) +} + +type fakeServerStream struct { + grpc.ServerStream + ctx context.Context +} + +func (f *fakeServerStream) Context() context.Context { + return f.ctx +} \ No newline at end of file From 2abc058f498fbb2e92b30be6d2dd470a9536b2df Mon Sep 17 00:00:00 2001 From: moh-osman3 Date: Wed, 3 Apr 2024 23:18:40 -0400 Subject: [PATCH 2/4] updates --- collector/config/configgrpc/configgrpc.go | 37 ++---- .../config/configgrpc/configgrpc_test.go | 112 ++++++------------ 2 files changed, 49 insertions(+), 100 deletions(-) diff --git a/collector/config/configgrpc/configgrpc.go b/collector/config/configgrpc/configgrpc.go index 10f78f1d..ff3111be 100644 --- a/collector/config/configgrpc/configgrpc.go +++ b/collector/config/configgrpc/configgrpc.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package configgrpc // import "go.opentelemetry.io/collector/config/configgrpc" +package configgrpc // import "github.com/open-telemetry/otel-arrow/collector/config/configgrpc" import ( "context" @@ -128,7 +128,7 @@ type KeepaliveEnforcementPolicy struct { // ServerConfig defines common settings for a gRPC server configuration. type ServerConfig struct { // Server net.Addr config. For transport only "tcp" and "unix" are valid options. - NetAddr confignet.NetAddr `mapstructure:",squash"` + NetAddr confignet.AddrConfig `mapstructure:",squash"` // Configures the protocol to use TLS. // The default value is nil, which will cause the protocol to not use TLS. @@ -162,7 +162,11 @@ type ServerConfig struct { MemoryLimiter *component.ID `mapstructure:"memory_limiter"` } -type memoryLimiterExtension = interface{ MustRefuse() bool } +type memoryLimiterExtension = interface{ + MustRefuse() bool + UnaryInterceptorGenerator() grpc.UnaryServerInterceptor + StreamInterceptorGenerator() grpc.StreamServerInterceptor +} // SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.ClientConfig.Endpoint. func (gcs *ClientConfig) SanitizedEndpoint() string { @@ -283,7 +287,7 @@ func validateBalancerName(balancerName string) bool { // ToServer returns a grpc.Server for the configuration // Deprecated: [0.96.0] Use ToServerContext instead. -func (gss *ServerConfig) ToServer(host component.Host, settings component.TelemetrySettings, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { +func (gss *ServerConfig) ToServer(_ context.Context, host component.Host, settings component.TelemetrySettings, extraOpts ...grpc.ServerOption) (*grpc.Server, error) { return gss.ToServerContext(context.Background(), host, settings, extraOpts...) } @@ -417,12 +421,8 @@ func (gss *ServerConfig) toServerOption(host component.Host, settings component. return nil, err } - uInterceptors = append(uInterceptors, func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { - return memoryLimiterUnaryServerInterceptor(ctx, req, info, handler, memoryLimiter) - }) - sInterceptors = append(sInterceptors, func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - return memoryLimiterStreamServerInterceptor(srv, ss, info, handler, memoryLimiter) - }) + uInterceptors = append(uInterceptors, memoryLimiter.UnaryInterceptorGenerator()) + sInterceptors = append(sInterceptors, memoryLimiter.StreamInterceptorGenerator()) } otelOpts := []otelgrpc.Option{ @@ -441,23 +441,6 @@ func (gss *ServerConfig) toServerOption(host component.Host, settings component. return opts, nil } -func memoryLimiterUnaryServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ml memoryLimiterExtension) (any, error) { - if ml.MustRefuse() { - return nil, errMemoryLimitReached - } - - return handler(ctx, req) -} - -func memoryLimiterStreamServerInterceptor(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler, ml memoryLimiterExtension) error { - ctx := stream.Context() - if ml.MustRefuse() { - return errMemoryLimitReached - } - - return handler(srv, wrapServerStream(ctx, stream)) -} - func getMemoryLimiterExtension(extID *component.ID, extensions map[component.ID]component.Component) (memoryLimiterExtension, error) { if ext, found := extensions[*extID]; found { if server, ok := ext.(memoryLimiterExtension); ok { diff --git a/collector/config/configgrpc/configgrpc_test.go b/collector/config/configgrpc/configgrpc_test.go index d6ebdd17..444d4f15 100644 --- a/collector/config/configgrpc/configgrpc_test.go +++ b/collector/config/configgrpc/configgrpc_test.go @@ -183,7 +183,7 @@ func TestAllGrpcClientSettings(t *testing.T) { func TestDefaultGrpcServerSettings(t *testing.T) { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "0.0.0.0:1234", }, } @@ -194,7 +194,7 @@ func TestDefaultGrpcServerSettings(t *testing.T) { func TestAllGrpcServerSettingsExceptAuth(t *testing.T) { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "localhost:1234", Transport: "tcp", }, @@ -227,7 +227,7 @@ func TestAllGrpcServerSettingsExceptAuth(t *testing.T) { func TestGrpcServerAuthSettings(t *testing.T) { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "0.0.0.0:1234", }, } @@ -246,7 +246,7 @@ func TestGrpcServerAuthSettings(t *testing.T) { func TestGrpcServerAuthSettings_Deprecated(t *testing.T) { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "0.0.0.0:1234", }, } @@ -258,7 +258,7 @@ func TestGrpcServerAuthSettings_Deprecated(t *testing.T) { mockID: auth.NewServer(), }, } - srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + srv, err := gss.ToServer(context.Background(), host, componenttest.NewNopTelemetrySettings()) assert.NoError(t, err) assert.NotNil(t, srv) } @@ -274,14 +274,29 @@ func (mml *mockMemoryLimiterExtension) MustRefuse() bool { return mml.mustRefuse } +func (mml *mockMemoryLimiterExtension) UnaryInterceptorGenerator() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + if mml.MustRefuse() { + return nil, errMemoryLimitReached + } + return nil, nil + } +} + +func (mml *mockMemoryLimiterExtension) StreamInterceptorGenerator() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return nil + } +} + func TestGetMemoryLimiterExtension(t *testing.T) { - badID := component.NewID("badmemlimiter") + badID := component.NewID(component.MustNewType("badmemlimiter")) notMLExtensionErr := fmt.Errorf("requested MemoryLimiter, %s, is not a memoryLimiterExtension", badID) - missingID := component.NewID("missingmemlimiter") + missingID := component.NewID(component.MustNewType("missingmemlimiter")) missingMLExtensionErr := fmt.Errorf("failed to resolve memoryLimiterExtension %q: %s", missingID, "memory limiter extension not found") - validID := component.NewID("memorylimiter") + validID := component.NewID(component.MustNewType("memorylimiter")) tests := []struct { name string @@ -309,7 +324,7 @@ func TestGetMemoryLimiterExtension(t *testing.T) { t.Run(test.name, func(t *testing.T) { comID := test.componentID gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "localhost:0", Transport: "tcp", }, @@ -320,7 +335,7 @@ func TestGetMemoryLimiterExtension(t *testing.T) { assert.NoError(t, err) ml := &mockMemoryLimiterExtension{iD: comID} extList := map[component.ID]component.Component{ - component.NewID("memorylimiter"): ml, + component.NewID(component.MustNewType("memorylimiter")): ml, badID: nopExt, } @@ -329,7 +344,7 @@ func TestGetMemoryLimiterExtension(t *testing.T) { } // ToServer calls getMemoryLimiterExtension(). - srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + srv, err := gss.ToServer(context.Background(), host, componenttest.NewNopTelemetrySettings()) // desired extension was not found. if test.getExtErr != nil { @@ -341,7 +356,6 @@ func TestGetMemoryLimiterExtension(t *testing.T) { assert.NoError(t, err) }) } - } func TestGrpcUnaryServerMemoryLimiterSettings(t *testing.T) { @@ -364,9 +378,9 @@ func TestGrpcUnaryServerMemoryLimiterSettings(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - comID := component.NewID("memorylimiter") + comID := component.NewID(component.MustNewType("memorylimiter")) gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "localhost:0", Transport: "tcp", }, @@ -382,7 +396,7 @@ func TestGrpcUnaryServerMemoryLimiterSettings(t *testing.T) { ext: extList, } - srv, err := gss.ToServer(host, componenttest.NewNopTelemetrySettings()) + srv, err := gss.ToServer(context.Background(), host, componenttest.NewNopTelemetrySettings()) // found extension so finish setting up server and client to test interceptor. assert.NoError(t, err) @@ -434,54 +448,6 @@ func TestGrpcUnaryServerMemoryLimiterSettings(t *testing.T) { } } -func TestGrpcStreamServerMemoryLimiterSettings(t *testing.T) { - tests := []struct { - name string - refused bool - // interceptErr refers to whether memorylimiterextension allowed the request. - interceptErr error - }{ - { - name: "stream memory limiter extension accept", - refused: false, - interceptErr: nil, - }, - { - name: "stream memory limiter extension refuse", - refused: true, - interceptErr: errMemoryLimitReached, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - comID := component.NewID("memorylimiter") - - ml := &mockMemoryLimiterExtension{iD: comID, mustRefuse: test.refused} - handlerCalled := false - handler := func(_ any, _ grpc.ServerStream) error { - handlerCalled = true - return nil - } - ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "some-auth-data")) - streamServer := &mockServerStream{ - ctx: ctx, - } - - // test - err := memoryLimiterStreamServerInterceptor(nil, streamServer, &grpc.StreamServerInfo{}, handler, ml) - - // verify - assert.ErrorIs(t, test.interceptErr, err) - if test.refused { - assert.False(t, handlerCalled) - } else { - assert.True(t, handlerCalled) - } - }) - } - -} - func TestGRPCClientSettingsError(t *testing.T) { tt, err := componenttest.SetupTelemetry(componentID) require.NoError(t, err) @@ -630,7 +596,7 @@ func TestGRPCServerWarning(t *testing.T) { }{ { settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "0.0.0.0:1234", Transport: "tcp", }, @@ -639,7 +605,7 @@ func TestGRPCServerWarning(t *testing.T) { }, { settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "127.0.0.1:1234", Transport: "tcp", }, @@ -648,7 +614,7 @@ func TestGRPCServerWarning(t *testing.T) { }, { settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "0.0.0.0:1234", Transport: "unix", }, @@ -681,7 +647,7 @@ func TestGRPCServerSettingsError(t *testing.T) { { err: "^failed to load TLS config: failed to load CA CertPool File: failed to load cert /doesnt/exist:", settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "127.0.0.1:1234", Transport: "tcp", }, @@ -695,7 +661,7 @@ func TestGRPCServerSettingsError(t *testing.T) { { err: "^failed to load TLS config: failed to load TLS cert and key: for auth via TLS, provide both certificate and key, or neither", settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "127.0.0.1:1234", Transport: "tcp", }, @@ -709,7 +675,7 @@ func TestGRPCServerSettingsError(t *testing.T) { { err: "^failed to load client CA CertPool: failed to load CA /doesnt/exist:", settings: ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "127.0.0.1:1234", Transport: "tcp", }, @@ -729,7 +695,7 @@ func TestGRPCServerSettingsError(t *testing.T) { func TestGRPCServerSettings_ToListener_Error(t *testing.T) { settings := ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "127.0.0.1:1234567", Transport: "tcp", }, @@ -850,7 +816,7 @@ func TestHttpReception(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "localhost:0", Transport: "tcp", }, @@ -898,7 +864,7 @@ func TestReceiveOnUnixDomainSocket(t *testing.T) { socketName := tempSocketName(t) gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: socketName, Transport: "unix", }, @@ -1094,7 +1060,7 @@ func TestClientInfoInterceptors(t *testing.T) { // prepare the server { gss := &ServerConfig{ - NetAddr: confignet.NetAddr{ + NetAddr: confignet.AddrConfig{ Endpoint: "localhost:0", Transport: "tcp", }, From 01a90306649e7b781418b0aaadc5ef3ce9489fa1 Mon Sep 17 00:00:00 2001 From: moh-osman3 Date: Thu, 4 Apr 2024 00:43:37 -0400 Subject: [PATCH 3/4] add blockingmemorylimiterextension --- collector/config/configgrpc/configgrpc.go | 2 - .../blockingmemorylimiter.go | 73 ++++++ .../blockingmemorylimiter_test.go | 213 ++++++++++++++++++ .../blockingmemorylimiterextension/config.go | 11 + .../blockingmemorylimiterextension/factory.go | 38 ++++ .../factory_test.go | 50 ++++ .../generated_component_test.go | 51 +++++ .../blockingmemorylimiterextension/go.mod | 66 ++++++ .../blockingmemorylimiterextension/go.sum | 62 +++++ .../internal/metadata/generated_status.go | 26 +++ .../metadata.yaml | 13 ++ collector/go.mod | 17 +- collector/go.sum | 13 ++ go.work | 5 +- 14 files changed, 633 insertions(+), 7 deletions(-) create mode 100644 collector/extension/blockingmemorylimiterextension/blockingmemorylimiter.go create mode 100644 collector/extension/blockingmemorylimiterextension/blockingmemorylimiter_test.go create mode 100644 collector/extension/blockingmemorylimiterextension/config.go create mode 100644 collector/extension/blockingmemorylimiterextension/factory.go create mode 100644 collector/extension/blockingmemorylimiterextension/factory_test.go create mode 100644 collector/extension/blockingmemorylimiterextension/generated_component_test.go create mode 100644 collector/extension/blockingmemorylimiterextension/go.mod create mode 100644 collector/extension/blockingmemorylimiterextension/go.sum create mode 100644 collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go create mode 100644 collector/extension/blockingmemorylimiterextension/metadata.yaml diff --git a/collector/config/configgrpc/configgrpc.go b/collector/config/configgrpc/configgrpc.go index ff3111be..e16a7ed9 100644 --- a/collector/config/configgrpc/configgrpc.go +++ b/collector/config/configgrpc/configgrpc.go @@ -165,7 +165,6 @@ type ServerConfig struct { type memoryLimiterExtension = interface{ MustRefuse() bool UnaryInterceptorGenerator() grpc.UnaryServerInterceptor - StreamInterceptorGenerator() grpc.StreamServerInterceptor } // SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.ClientConfig.Endpoint. @@ -422,7 +421,6 @@ func (gss *ServerConfig) toServerOption(host component.Host, settings component. } uInterceptors = append(uInterceptors, memoryLimiter.UnaryInterceptorGenerator()) - sInterceptors = append(sInterceptors, memoryLimiter.StreamInterceptorGenerator()) } otelOpts := []otelgrpc.Option{ diff --git a/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter.go b/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter.go new file mode 100644 index 00000000..66025490 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter.go @@ -0,0 +1,73 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package blockingmemorylimiterextension // import "github.com/open-telemetry/otel-arrow/collector/blockingmemorylimiterextension" + +import ( + "context" + "fmt" + "time" + + "golang.org/x/sync/semaphore" + "go.uber.org/zap" + "google.golang.org/grpc" + + "go.opentelemetry.io/collector/component" +) + +type blockingMLExtension struct { + limitBytes int64 + sem *semaphore.Weighted + logger *zap.Logger + timeout time.Duration +} + +// This interface is meant to access the size of a +// ExportTraceServiceRequest, ExportMetricsServiceRequest, ExportLogsServicesRequest +type telemetryServiceRequest = interface { + Size() int +} + + +// newMemoryLimiter returns a new memorylimiter extension. +func newBlockingMLExtension(cfg *Config, logger *zap.Logger) (*blockingMLExtension, error) { + limitBytes := int64(cfg.MemoryLimitMiB) << 20 + return &blockingMLExtension{ + limitBytes: limitBytes, + sem: semaphore.NewWeighted(limitBytes), + timeout: cfg.Timeout, + logger: logger, + }, nil +} + +func (bml *blockingMLExtension) Start(ctx context.Context, host component.Host) error { + return nil +} + +func (bml *blockingMLExtension) Shutdown(ctx context.Context) error { + return nil +} + +func (bml *blockingMLExtension) MustRefuse() bool { + return false +} + +func (bml *blockingMLExtension) UnaryInterceptorGenerator() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + a := req.(telemetryServiceRequest) + requestSize := int64(a.Size()) + + semCtx, cancel := context.WithTimeout(context.Background(), bml.timeout) + defer cancel() + + err = bml.sem.Acquire(semCtx, requestSize) + if err != nil { + return nil, fmt.Errorf("not enough memory available to process request, %w", err) + } + + resp, err = handler(ctx, req) + bml.sem.Release(requestSize) + + return resp, err + } +} diff --git a/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter_test.go b/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter_test.go new file mode 100644 index 00000000..7326bacf --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/blockingmemorylimiter_test.go @@ -0,0 +1,213 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package blockingmemorylimiterextension + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/multierr" + "go.uber.org/zap" + "google.golang.org/grpc" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configtls" + "github.com/open-telemetry/otel-arrow/collector/config/configgrpc" + "github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor/testdata" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" +) + +type mockMemoryLimiterExtension struct { + *blockingMLExtension + componentID component.ID + blockCh chan struct{} +} + +func (mockmle *mockMemoryLimiterExtension) UnaryInterceptorGenerator() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + inner := mockmle.blockingMLExtension.UnaryInterceptorGenerator() + blockHandler := func(ctx context.Context, req any) (any, error) { + <-mockmle.blockCh + return handler(ctx, req) + } + return inner(ctx, req, info, blockHandler) + } +} + +type mockHost struct { + component.Host + ext map[component.ID]component.Component +} + +func (nh *mockHost) GetExtensions() map[component.ID]component.Component { + return nh.ext +} +type grpcTraceServer struct { + ptraceotlp.UnimplementedGRPCServer + recordedContext context.Context +} + +func (gts *grpcTraceServer) Export(ctx context.Context, _ ptraceotlp.ExportRequest) (ptraceotlp.ExportResponse, error) { + gts.recordedContext = ctx + return ptraceotlp.NewExportResponse(), nil +} + +func TestUnaryInterceptorMemoryLimited(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + mlCfg *Config + numTraces int + numIter int + expectError bool + sleep time.Duration + }{ + { + name: "below memory limit", + mlCfg: &Config{ + MemoryLimitMiB: 1, + Timeout: 1 * time.Nanosecond, + }, + sleep: 1 * time.Second, + numTraces: 100, + numIter: 1, + expectError: false, + }, + { + name: "above memory limit", + mlCfg: &Config{ + MemoryLimitMiB: 1, + Timeout: 1 * time.Nanosecond, + }, + sleep: 1 * time.Second, + numTraces: 30000, + numIter: 1, + expectError: true, + }, + { + name: "multiple requests timeout short", + mlCfg: &Config{ + MemoryLimitMiB: 1, + Timeout: 1 * time.Nanosecond, + }, + sleep: 1 * time.Second, + numTraces: 5000, + numIter: 2, + expectError: true, + }, + { + name: "multiple requests timeout long", + mlCfg: &Config{ + MemoryLimitMiB: 1, + Timeout: 2 * time.Second, + }, + sleep: 1 * time.Second, + numTraces: 5000, + numIter: 2, + expectError: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ml, err := newBlockingMLExtension(tt.mlCfg, zap.NewNop()) + comID := component.NewID(component.MustNewType("memorylimiter")) + mockML := &mockMemoryLimiterExtension{ + componentID: comID, + blockingMLExtension: ml, + blockCh: make(chan struct{}), + } + assert.NoError(t, err) + + assert.NoError(t, ml.Start(ctx, &mockHost{})) + + gss := &configgrpc.ServerConfig{ + NetAddr: confignet.AddrConfig{ + Endpoint: "localhost:0", + Transport: "tcp", + }, + MemoryLimiter: &comID, + } + extList := map[component.ID]component.Component{ + comID: mockML, + } + + host := &mockHost{ + ext: extList, + } + + srv, err := gss.ToServer(context.Background(), host, componenttest.NewNopTelemetrySettings()) + + // found extension so finish setting up server and client to test interceptor. + assert.NoError(t, err) + mock := &grpcTraceServer{} + + ptraceotlp.RegisterGRPCServer(srv, mock) + + defer srv.Stop() + + l, err := gss.NetAddr.Listen(context.Background()) + require.NoError(t, err) + + go func() { + _ = srv.Serve(l) + }() + + // setup client + gcs := &configgrpc.ClientConfig{ + Endpoint: l.Addr().String(), + TLSSetting: configtls.TLSClientSetting{ + Insecure: true, + }, + } + + tel, err := componenttest.SetupTelemetry(comID) + require.NoError(t, err) + defer func() { + require.NoError(t, tel.Shutdown(context.Background())) + }() + + grpcClientConn, errClient := gcs.ToClientConn(context.Background(), componenttest.NewNopHost(), tel.TelemetrySettings()) + require.NoError(t, errClient) + defer func() { assert.NoError(t, grpcClientConn.Close()) }() + + basecl := ptraceotlp.NewGRPCClient(grpcClientConn) + + ctx, cancelFunc := context.WithTimeout(context.Background(), 20*time.Second) + defer cancelFunc() + + var wg sync.WaitGroup + var retErr error + for i := 0; i < tt.numIter; i++ { + wg.Add(1) + go func() { + traces := testdata.GenerateTraces(tt.numTraces) + _, errResp := basecl.Export(ctx, ptraceotlp.NewExportRequestFromTraces(traces)) + + retErr = multierr.Append(retErr, errResp) + wg.Done() + }() + } + + // sleep so multiple requests have time to be blocked after calling sem.Acquire() + time.Sleep(tt.sleep) + close(mockML.blockCh) + wg.Wait() + + if tt.expectError { + assert.ErrorContains(t, retErr, "not enough memory available to process request") + } else { + assert.NoError(t, retErr) + } + + assert.NoError(t, ml.Shutdown(ctx)) + }) + } +} \ No newline at end of file diff --git a/collector/extension/blockingmemorylimiterextension/config.go b/collector/extension/blockingmemorylimiterextension/config.go new file mode 100644 index 00000000..1ec6ab13 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/config.go @@ -0,0 +1,11 @@ +package blockingmemorylimiterextension // import "github.com/open-telemetry/otel-arrow/collector/blockingmemorylimiterextension" + +import ( + "time" +) + +type Config struct { + MemoryLimitMiB uint32 `mapstructure:"memory_limit_mib"` + // Timeout is the amount of time to wait for request to be accepted. + Timeout time.Duration `mapstructure:timeout` +} \ No newline at end of file diff --git a/collector/extension/blockingmemorylimiterextension/factory.go b/collector/extension/blockingmemorylimiterextension/factory.go new file mode 100644 index 00000000..4dae7322 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/factory.go @@ -0,0 +1,38 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package blockingmemorylimiterextension // import "github.com/open-telemetry/otel-arrow/collector/blockingmemorylimiterextension" + +//go:generate mdatagen metadata.yaml + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + + "github.com/open-telemetry/otel-arrow/collector/extension/blockingmemorylimiterextension/internal/metadata" +) + +// NewFactory returns a new factory for the Memory Limiter extension. +func NewFactory() extension.Factory { + return extension.NewFactory( + metadata.Type, + createDefaultConfig, + createExtension, + metadata.ExtensionStability) +} + +// CreateDefaultConfig creates the default configuration for extension. Notice +// that the default configuration is expected to fail for this extension. +func createDefaultConfig() component.Config { + return &Config{ + MemoryLimitMiB: 32, + Timeout: 1 * time.Nanosecond, + } +} + +func createExtension(_ context.Context, set extension.CreateSettings, cfg component.Config) (extension.Extension, error) { + return newBlockingMLExtension(cfg.(*Config), set.TelemetrySettings.Logger) +} \ No newline at end of file diff --git a/collector/extension/blockingmemorylimiterextension/factory_test.go b/collector/extension/blockingmemorylimiterextension/factory_test.go new file mode 100644 index 00000000..9b9ce413 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/factory_test.go @@ -0,0 +1,50 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package memorylimiterextension + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/extension/extensiontest" + "go.opentelemetry.io/collector/internal/memorylimiter" +) + +func TestCreateDefaultConfig(t *testing.T) { + factory := NewFactory() + require.NotNil(t, factory) + + cfg := factory.CreateDefaultConfig() + assert.NotNil(t, cfg, "failed to create default config") + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestCreateExtension(t *testing.T) { + factory := NewFactory() + require.NotNil(t, factory) + + cfg := factory.CreateDefaultConfig() + + // Create extension with a valid config. + pCfg := cfg.(*Config) + pCfg.MemoryLimitMiB = 5722 + pCfg.MemorySpikeLimitMiB = 1907 + pCfg.CheckInterval = 100 * time.Millisecond + + tp, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + assert.NoError(t, err) + assert.NotNil(t, tp) + // test if we can shutdown a monitoring routine that has not started + assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted) + assert.NoError(t, tp.Start(context.Background(), componenttest.NewNopHost())) + + assert.NoError(t, tp.Shutdown(context.Background())) + // verify that no monitoring routine is running + assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted) +} \ No newline at end of file diff --git a/collector/extension/blockingmemorylimiterextension/generated_component_test.go b/collector/extension/blockingmemorylimiterextension/generated_component_test.go new file mode 100644 index 00000000..53a0b75a --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/generated_component_test.go @@ -0,0 +1,51 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package blockingmemorylimiterextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + + "go.opentelemetry.io/collector/extension/extensiontest" + + "go.opentelemetry.io/collector/confmap/confmaptest" +) + +func TestCheckConfigStruct(t *testing.T) { + componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig()) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, component.UnmarshalConfig(sub, cfg)) + + t.Run("shutdown", func(t *testing.T) { + e, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + err = e.Shutdown(context.Background()) + require.NoError(t, err) + }) + + t.Run("lifecycle", func(t *testing.T) { + + firstExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + require.NoError(t, firstExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, firstExt.Shutdown(context.Background())) + + secondExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, secondExt.Shutdown(context.Background())) + }) +} diff --git a/collector/extension/blockingmemorylimiterextension/go.mod b/collector/extension/blockingmemorylimiterextension/go.mod new file mode 100644 index 00000000..fb05ac83 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/go.mod @@ -0,0 +1,66 @@ +module github.com/open-telemetry/otel-arrow/collector/blockingmemorylimiterextension + +go 1.22.1 + +require ( + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector v0.97.0 + go.opentelemetry.io/collector/component v0.97.0 + go.opentelemetry.io/collector/config/confignet v0.97.0 + go.opentelemetry.io/collector/config/configtls v0.97.0 + go.opentelemetry.io/collector/confmap v0.97.0 + go.opentelemetry.io/collector/extension v0.97.0 + go.opentelemetry.io/collector/pdata v1.4.0 + go.opentelemetry.io/otel/metric v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + golang.org/x/sync v0.6.0 + google.golang.org/grpc v1.62.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/collector/config/configopaque v1.4.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.97.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/collector/extension/blockingmemorylimiterextension/go.sum b/collector/extension/blockingmemorylimiterextension/go.sum new file mode 100644 index 00000000..db8044a3 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/go.sum @@ -0,0 +1,62 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +go.opentelemetry.io/collector v0.97.0 h1:qyOju13byHIKEK/JehmTiGMj4pFLa4kDyrOCtTmjHU0= +go.opentelemetry.io/collector/component v0.97.0 h1:vanKhXl5nptN8igRH4PqVYHOILif653vaPIKv6LCZCI= +go.opentelemetry.io/collector/config/confignet v0.97.0 h1:KJjv10/YVMslSSLVWW/IIjpLM3JiO3rWvw5dK/t1H7g= +go.opentelemetry.io/collector/config/configopaque v1.4.0 h1:5KgD9oLN+N07HqDsLzUrU0mE2pC8cMhrCSC1Nf8CEO4= +go.opentelemetry.io/collector/config/configtelemetry v0.97.0 h1:JS/WxK09A9m39D5OqsAWaoRe4tG7ESMnzDNIbZ5bD6c= +go.opentelemetry.io/collector/config/configtls v0.97.0 h1:wmXj/rKQUGMZzbHVCTyB+xUWImsGxnLqhivwjBE0FdI= +go.opentelemetry.io/collector/confmap v0.97.0 h1:0CGSk7YW9rPc6jCwJteJzHzN96HRoHTfuqI7J/EmZsg= +go.opentelemetry.io/collector/extension v0.97.0 h1:LpjZ4KQgnhLG/u3l69QgWkX8qMqeS8IFKWMoDtbPIeE= +go.opentelemetry.io/collector/pdata v1.4.0 h1:cA6Pr7Z2V7mE+i7FmYpavX7nefzd6H4CICgW0T9aJX0= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go b/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go new file mode 100644 index 00000000..0b085ecf --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go @@ -0,0 +1,26 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +var ( + Type = component.MustNewType("blocking_memory_limiter") + scopeName = "github.com/open-telemetry/otel-arrow/collector/extension/blockingmemorylimiterextension" +) + +const ( + ExtensionStability = component.StabilityLevelDevelopment +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter(scopeName) +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer(scopeName) +} diff --git a/collector/extension/blockingmemorylimiterextension/metadata.yaml b/collector/extension/blockingmemorylimiterextension/metadata.yaml new file mode 100644 index 00000000..20a912f1 --- /dev/null +++ b/collector/extension/blockingmemorylimiterextension/metadata.yaml @@ -0,0 +1,13 @@ +type: blocking_memory_limiter + +status: + class: extension + stability: + development: [extension] + distributions: [] + +tests: + config: + check_interval: 5s + limit_mib: 400 + spike_limit_mib: 50 \ No newline at end of file diff --git a/collector/go.mod b/collector/go.mod index 6dc5b384..b27b9dff 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -6,17 +6,29 @@ toolchain go1.21.4 require ( github.com/klauspost/compress v1.17.7 + github.com/mostynb/go-grpc-compression v1.2.2 github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector v0.97.0 go.opentelemetry.io/collector/component v0.97.0 + go.opentelemetry.io/collector/config/configauth v0.97.0 + go.opentelemetry.io/collector/config/configcompression v1.4.0 + go.opentelemetry.io/collector/config/confignet v0.97.0 + go.opentelemetry.io/collector/config/configopaque v1.4.0 go.opentelemetry.io/collector/config/configtelemetry v0.97.0 + go.opentelemetry.io/collector/config/configtls v0.97.0 go.opentelemetry.io/collector/exporter v0.97.0 + go.opentelemetry.io/collector/extension v0.97.0 + go.opentelemetry.io/collector/extension/auth v0.97.0 + go.opentelemetry.io/collector/pdata v1.4.0 go.opentelemetry.io/collector/receiver v0.97.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/metric v1.24.0 go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/sdk/metric v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 google.golang.org/grpc v1.62.1 ) @@ -24,11 +36,14 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect @@ -45,9 +60,7 @@ require ( github.com/rogpeppe/go-internal v1.11.0 // indirect go.opentelemetry.io/collector/confmap v0.97.0 // indirect go.opentelemetry.io/collector/consumer v0.97.0 // indirect - go.opentelemetry.io/collector/pdata v1.4.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/collector/go.sum b/collector/go.sum index e3fe405b..825258b6 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -5,6 +5,7 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -17,10 +18,12 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -46,6 +49,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mostynb/go-grpc-compression v1.2.2 h1:XaDbnRvt2+1vgr0b/l0qh4mJAfIxE0bKXtz2Znl3GGI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= @@ -64,20 +68,29 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.97.0 h1:qyOju13byHIKEK/JehmTiGMj4pFLa4kDyrOCtTmjHU0= go.opentelemetry.io/collector/component v0.97.0 h1:vanKhXl5nptN8igRH4PqVYHOILif653vaPIKv6LCZCI= go.opentelemetry.io/collector/component v0.97.0/go.mod h1:F/m3HMlkb16RKI7wJjgbECK1IZkAcmB8bu7yD8XOkwM= +go.opentelemetry.io/collector/config/configauth v0.97.0 h1:38M2uUsBzgD7sdJPPXUsOq1BFr6X6P4A5VFg+MOcRNY= +go.opentelemetry.io/collector/config/configcompression v1.4.0 h1:qWRKdl49lBvPUr6UWmyf1pR4EOBHN+66pDeGtfQ1Mbk= +go.opentelemetry.io/collector/config/confignet v0.97.0 h1:KJjv10/YVMslSSLVWW/IIjpLM3JiO3rWvw5dK/t1H7g= +go.opentelemetry.io/collector/config/configopaque v1.4.0 h1:5KgD9oLN+N07HqDsLzUrU0mE2pC8cMhrCSC1Nf8CEO4= go.opentelemetry.io/collector/config/configtelemetry v0.97.0 h1:JS/WxK09A9m39D5OqsAWaoRe4tG7ESMnzDNIbZ5bD6c= go.opentelemetry.io/collector/config/configtelemetry v0.97.0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/config/configtls v0.97.0 h1:wmXj/rKQUGMZzbHVCTyB+xUWImsGxnLqhivwjBE0FdI= go.opentelemetry.io/collector/confmap v0.97.0 h1:0CGSk7YW9rPc6jCwJteJzHzN96HRoHTfuqI7J/EmZsg= go.opentelemetry.io/collector/confmap v0.97.0/go.mod h1:AnJmZcZoOLuykSXGiAf3shi11ZZk5ei4tZd9dDTTpWE= go.opentelemetry.io/collector/consumer v0.97.0 h1:S0BZQtJQxSHT156S8a5rLt3TeWYP8Rq+jn8QEyWQUYk= go.opentelemetry.io/collector/consumer v0.97.0/go.mod h1:1D06LURiZ/1KA2OnuKNeSn9bvFmJ5ZWe6L8kLu0osSY= go.opentelemetry.io/collector/exporter v0.97.0 h1:kw/fQrpkhTz0/3I/Z0maRj0S8Mi0NK50/WwFuWrRYPc= go.opentelemetry.io/collector/exporter v0.97.0/go.mod h1:EJYc4biKWxq3kD4Xh4SUSFbZ2lMsxjzwiCozikEDMjk= +go.opentelemetry.io/collector/extension v0.97.0 h1:LpjZ4KQgnhLG/u3l69QgWkX8qMqeS8IFKWMoDtbPIeE= +go.opentelemetry.io/collector/extension/auth v0.97.0 h1:2AYGxSbsi1KC2DOOFbAe7valrERb86m7TfRY85X8hSE= go.opentelemetry.io/collector/pdata v1.4.0 h1:cA6Pr7Z2V7mE+i7FmYpavX7nefzd6H4CICgW0T9aJX0= go.opentelemetry.io/collector/pdata v1.4.0/go.mod h1:0Ttp4wQinhV5oJTd9MjyvUegmZBO9O0nrlh/+EDLw+Q= go.opentelemetry.io/collector/receiver v0.97.0 h1:ozzE5MhIPtfnYA/UKB/NCcgxSmeLqdwErboi6B/IpLQ= go.opentelemetry.io/collector/receiver v0.97.0/go.mod h1:1TCN9DRuB45+xKqlwv4BMQR6qXgaJeSSNezFTJhmDUo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= diff --git a/go.work b/go.work index f010cb72..1aa050c2 100644 --- a/go.work +++ b/go.work @@ -1,6 +1,4 @@ -go 1.21 - -toolchain go1.21.4 +go 1.22.1 use ( . @@ -10,6 +8,7 @@ use ( ./collector/examples/printer ./collector/exporter/fileexporter ./collector/exporter/otelarrowexporter + ./collector/extension/blockingmemorylimiterextension ./collector/processor/concurrentbatchprocessor ./collector/processor/experimentprocessor ./collector/processor/obfuscationprocessor From 677a399c211de8a3dbfdbcd4a52e310c8e0f92f0 Mon Sep 17 00:00:00 2001 From: moh-osman3 Date: Thu, 4 Apr 2024 01:46:35 -0400 Subject: [PATCH 4/4] grr gomod remove all annoying files --- .../blockingmemorylimiterextension/factory.go | 8 +-- .../factory_test.go | 50 ------------------ .../generated_component_test.go | 51 ------------------- .../internal/metadata/generated_status.go | 26 ---------- .../metadata.yaml | 13 ----- 5 files changed, 4 insertions(+), 144 deletions(-) delete mode 100644 collector/extension/blockingmemorylimiterextension/factory_test.go delete mode 100644 collector/extension/blockingmemorylimiterextension/generated_component_test.go delete mode 100644 collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go delete mode 100644 collector/extension/blockingmemorylimiterextension/metadata.yaml diff --git a/collector/extension/blockingmemorylimiterextension/factory.go b/collector/extension/blockingmemorylimiterextension/factory.go index 4dae7322..2b41bf85 100644 --- a/collector/extension/blockingmemorylimiterextension/factory.go +++ b/collector/extension/blockingmemorylimiterextension/factory.go @@ -11,17 +11,17 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" - - "github.com/open-telemetry/otel-arrow/collector/extension/blockingmemorylimiterextension/internal/metadata" ) +var Type = component.MustNewType("blocking_memory_limiter") +const ExtensionStability = component.StabilityLevelDevelopment // NewFactory returns a new factory for the Memory Limiter extension. func NewFactory() extension.Factory { return extension.NewFactory( - metadata.Type, + Type, createDefaultConfig, createExtension, - metadata.ExtensionStability) + ExtensionStability) } // CreateDefaultConfig creates the default configuration for extension. Notice diff --git a/collector/extension/blockingmemorylimiterextension/factory_test.go b/collector/extension/blockingmemorylimiterextension/factory_test.go deleted file mode 100644 index 9b9ce413..00000000 --- a/collector/extension/blockingmemorylimiterextension/factory_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package memorylimiterextension - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/collector/extension/extensiontest" - "go.opentelemetry.io/collector/internal/memorylimiter" -) - -func TestCreateDefaultConfig(t *testing.T) { - factory := NewFactory() - require.NotNil(t, factory) - - cfg := factory.CreateDefaultConfig() - assert.NotNil(t, cfg, "failed to create default config") - assert.NoError(t, componenttest.CheckConfigStruct(cfg)) -} - -func TestCreateExtension(t *testing.T) { - factory := NewFactory() - require.NotNil(t, factory) - - cfg := factory.CreateDefaultConfig() - - // Create extension with a valid config. - pCfg := cfg.(*Config) - pCfg.MemoryLimitMiB = 5722 - pCfg.MemorySpikeLimitMiB = 1907 - pCfg.CheckInterval = 100 * time.Millisecond - - tp, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) - assert.NoError(t, err) - assert.NotNil(t, tp) - // test if we can shutdown a monitoring routine that has not started - assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted) - assert.NoError(t, tp.Start(context.Background(), componenttest.NewNopHost())) - - assert.NoError(t, tp.Shutdown(context.Background())) - // verify that no monitoring routine is running - assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted) -} \ No newline at end of file diff --git a/collector/extension/blockingmemorylimiterextension/generated_component_test.go b/collector/extension/blockingmemorylimiterextension/generated_component_test.go deleted file mode 100644 index 53a0b75a..00000000 --- a/collector/extension/blockingmemorylimiterextension/generated_component_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package blockingmemorylimiterextension - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/component/componenttest" - - "go.opentelemetry.io/collector/extension/extensiontest" - - "go.opentelemetry.io/collector/confmap/confmaptest" -) - -func TestCheckConfigStruct(t *testing.T) { - componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig()) -} - -func TestComponentLifecycle(t *testing.T) { - factory := NewFactory() - - cm, err := confmaptest.LoadConf("metadata.yaml") - require.NoError(t, err) - cfg := factory.CreateDefaultConfig() - sub, err := cm.Sub("tests::config") - require.NoError(t, err) - require.NoError(t, component.UnmarshalConfig(sub, cfg)) - - t.Run("shutdown", func(t *testing.T) { - e, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) - require.NoError(t, err) - err = e.Shutdown(context.Background()) - require.NoError(t, err) - }) - - t.Run("lifecycle", func(t *testing.T) { - - firstExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) - require.NoError(t, err) - require.NoError(t, firstExt.Start(context.Background(), componenttest.NewNopHost())) - require.NoError(t, firstExt.Shutdown(context.Background())) - - secondExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) - require.NoError(t, err) - require.NoError(t, secondExt.Start(context.Background(), componenttest.NewNopHost())) - require.NoError(t, secondExt.Shutdown(context.Background())) - }) -} diff --git a/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go b/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go deleted file mode 100644 index 0b085ecf..00000000 --- a/collector/extension/blockingmemorylimiterextension/internal/metadata/generated_status.go +++ /dev/null @@ -1,26 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" -) - -var ( - Type = component.MustNewType("blocking_memory_limiter") - scopeName = "github.com/open-telemetry/otel-arrow/collector/extension/blockingmemorylimiterextension" -) - -const ( - ExtensionStability = component.StabilityLevelDevelopment -) - -func Meter(settings component.TelemetrySettings) metric.Meter { - return settings.MeterProvider.Meter(scopeName) -} - -func Tracer(settings component.TelemetrySettings) trace.Tracer { - return settings.TracerProvider.Tracer(scopeName) -} diff --git a/collector/extension/blockingmemorylimiterextension/metadata.yaml b/collector/extension/blockingmemorylimiterextension/metadata.yaml deleted file mode 100644 index 20a912f1..00000000 --- a/collector/extension/blockingmemorylimiterextension/metadata.yaml +++ /dev/null @@ -1,13 +0,0 @@ -type: blocking_memory_limiter - -status: - class: extension - stability: - development: [extension] - distributions: [] - -tests: - config: - check_interval: 5s - limit_mib: 400 - spike_limit_mib: 50 \ No newline at end of file