diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 31375626..f722e90d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -136,7 +136,7 @@ jobs:
TRACE_ID=$(grep "Location: /trace/" curl-output.http | cut -d/ -f3 | tr -d '\r')
mkdir -p output/api/traces
- for mode in ff{0,1,2,3}{0,1}00000{0,1}; do
+ for mode in ff{0,1,2,3}{0,1,5,6}000000; do
mode_trace=${mode}${TRACE_ID:10}
curl -o output/api/traces/$mode_trace http://localhost:16686/api/traces/$mode_trace
done
@@ -172,12 +172,12 @@ jobs:
Redirecting to ${basename}
-
+
EOF
- mkdir -p output/trace/ff20000000$(cat $trace_dir/trace_id)
- cp spa.html output/trace/ff20000000$(cat $trace_dir/trace_id)/index.html
+ mkdir -p output/trace/ff26000000$(cat $trace_dir/trace_id)
+ cp spa.html output/trace/ff26000000$(cat $trace_dir/trace_id)/index.html
cp $trace_dir/api/traces/* output/api/traces/
done
mv output/dot-usage/* output/
@@ -261,7 +261,7 @@ jobs:
promises.push((async () => {
const traceName = await fs.readFile(`pages/${trace}/trace_name`, {encoding: "utf8"})
const partialTraceId = await fs.readFile(`pages/${trace}/trace_id`, {encoding: "utf8"})
- const traceId = "ff20000000" + partialTraceId
+ const traceId = "ff26000000" + partialTraceId
console.log(`Loading trace ${traceName}`)
diff --git a/Makefile b/Makefile
index 6d92e630..5b928d5a 100644
--- a/Makefile
+++ b/Makefile
@@ -34,6 +34,8 @@ else
LOG_FILE_ARG ?=
endif
+LINKER_WORKER_COUNT ?= 1
+
CONTROLLERS ?= audit-consumer,audit-producer,audit-webhook,event-informer,annotation-linker,owner-linker,resource-object-tag,resource-event-tag,diff-decorator,diff-controller,diff-api,pprof,jaeger-storage-plugin,jaeger-redirect-server,kelemetrix
ifeq ($(CONTROLLERS),)
ENABLE_ARGS ?=
@@ -69,7 +71,7 @@ endif
.PHONY: run dump-rotate test usage dot kind stack pre-commit
run: output/kelemetry $(DUMP_ROTATE_DEP)
GIN_MODE=debug \
- ./output/kelemetry \
+ $(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) \
--mq=local \
--audit-consumer-partition=$(PARTITIONS) \
--http-address=0.0.0.0 \
@@ -85,6 +87,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
--log-file=$(LOG_FILE) \
--aggregator-pseudo-span-global-tags=runId=$(RUN_ID) \
--aggregator-event-span-global-tags=run=$(RUN_ID) \
+ --linker-worker-count=$(LINKER_WORKER_COUNT) \
--pprof-addr=:6030 \
--diff-cache=$(ETCD_OR_LOCAL) \
--diff-cache-etcd-endpoints=127.0.0.1:2379 \
@@ -95,6 +98,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
--span-cache-etcd-endpoints=127.0.0.1:2379 \
--tracer-otel-endpoint=$(OTEL_EXPORTER_OTLP_ENDPOINT) \
--tracer-otel-insecure \
+ --object-cache-size=16777216 \
--jaeger-cluster-names=$(CLUSTER_NAME) \
--jaeger-storage-plugin-address=0.0.0.0:17271 \
--jaeger-backend=jaeger-storage \
@@ -117,15 +121,15 @@ test:
go test -v -race -coverpkg=./pkg/... -coverprofile=coverage.out $(INTEGRATION_ARG) $(BUILD_ARGS) ./pkg/...
usage: output/kelemetry
- ./output/kelemetry --usage=USAGE.txt
+ $(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --usage=USAGE.txt
dot: output/kelemetry
- ./output/kelemetry --dot=depgraph.dot
+ $(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --dot=depgraph.dot
dot -Tpng depgraph.dot >depgraph.png
dot -Tsvg depgraph.dot >depgraph.svg
output/kelemetry: go.mod go.sum $(shell find -type f -name "*.go")
- go build -v $(RACE_ARG) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
+ go build -v $(RACE_ARG) -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
kind:
kind delete cluster --name tracetest
diff --git a/charts/kelemetry/templates/_helpers.yaml b/charts/kelemetry/templates/_helpers.yaml
index 2524c062..16ac93ee 100644
--- a/charts/kelemetry/templates/_helpers.yaml
+++ b/charts/kelemetry/templates/_helpers.yaml
@@ -111,6 +111,7 @@ span-cache-etcd-prefix: {{ .Values.aggregator.spanCache.etcd.prefix | toJson }}
{{- end }}
{{/* LINKERS */}}
+linker-worker-count: {{ .Values.linkers.workerCount }}
annotation-linker-enable: {{ .Values.linkers.annotation }}
owner-linker-enable: {{ .Values.linkers.ownerReference }}
diff --git a/charts/kelemetry/values.yaml b/charts/kelemetry/values.yaml
index c29405f5..1c1654fd 100644
--- a/charts/kelemetry/values.yaml
+++ b/charts/kelemetry/values.yaml
@@ -323,6 +323,9 @@ aggregator:
# Linkers associated objects together.
linkers:
+ # Maximum number of concurrent link jobs.
+ # Each link job runs each of the linkers for a single object in series.
+ workerCount: 8
# Enable the owner linker, which links objects based on native owner references.
ownerReference: true
# Enable the annotation linker, which links objects based on the `kelemetry.kubewharf.io/parent-link` annotation.
diff --git a/dev.docker-compose.yaml b/dev.docker-compose.yaml
index 8f66a2b3..c951a95d 100644
--- a/dev.docker-compose.yaml
+++ b/dev.docker-compose.yaml
@@ -54,6 +54,7 @@ services:
- 127.0.0.1:17272:17271
volumes:
- badger:/mnt/badger
+ restart: always
# Web frontend for raw trace database view.
jaeger-query-raw:
diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
index ea1417fd..dc6b026b 100644
--- a/docs/USER_GUIDE.md
+++ b/docs/USER_GUIDE.md
@@ -20,8 +20,12 @@ The "Service" field selects one of the display modes:
Additional information is available in event tags.
- `timeline`: All events are displayed as children of the root object.
-By default, the whole trace is displayed, including parent and sibling objects of the searched object.
-Enabling the `exclusive` option limits the output to the subtree under the object matched in the search.
+By default, only the trace for a single object is displayed.
+More traces are available by configuration:
+
+- `full tree`: view the full tree from the deepest ancestor
+- `ancestors`: include transitive owners
+- `children`: include child objects
### Cluster
diff --git a/hack/tfconfig.yaml b/hack/tfconfig.yaml
index 2be80224..b8ddc198 100644
--- a/hack/tfconfig.yaml
+++ b/hack/tfconfig.yaml
@@ -13,9 +13,8 @@ configs:
- kind: Batch
batchName: initial
- kind: ExtractNestingVisitor
- matchesNestLevel:
- oneOf: []
- negate: true
+ matchesPseudoType:
+ oneOf: ["object"]
- kind: Batch
batchName: final
- id: "20000000"
@@ -23,12 +22,12 @@ configs:
steps:
- kind: Batch
batchName: initial
- - kind: ExtractNestingVisitor
- matchesNestLevel:
- oneOf: ["object"]
- negate: true
- kind: Batch
batchName: collapse
+ - kind: ExtractNestingVisitor
+ matchesPseudoType:
+ oneOf: ["linkClass"]
+ matchesName:
- kind: CompactDurationVisitor
- kind: Batch
batchName: final
@@ -37,28 +36,48 @@ configs:
steps:
- kind: Batch
batchName: initial
- - kind: ExtractNestingVisitor
- matchesNestLevel:
- oneOf: ["object"]
- negate: true
- kind: Batch
batchName: collapse
- kind: GroupByTraceSourceVisitor
shouldBeGrouped:
oneOf: ["event"]
- negate: true
+ then: false
- kind: CompactDurationVisitor
- kind: Batch
batchName: final
modifiers:
+ # Multiple active link-selector modifiers are additive (union)
"01000000":
- displayName: exclusive
- modifierName: exclusive
+ # the entire tree under the deepest (up to 3 levels) ancestor
+ displayName: full tree
+ modifierName: link-selector
+ args:
+ modifierClass: owner-ref
+ includeSiblings: false
+ upwardDistance: 3
+ downwardDistance: 3
+ "02000000":
+ # include all ancestors (up to 3) but not siblings of ancestors
+ displayName: ancestors
+ modifierName: link-selector
+ args:
+ modifierClass: owner-ref
+ includeSiblings: true
+ upwardDistance: 3
+ "04000000":
+ # the entire subtree under this object
+ displayName: owned objects
+ modifierName: link-selector
+ args:
+ modifierClass: owner-ref
+ ifAll:
+ - linkClass: children
+ fromChild: false
+ downwardDistance: 3
# Uncomment to enable extension trace from apiserver
# "00000001":
-# # We want to run extension modifiers after exclusive modifier to avoid fetching unused traces
# displayName: apiserver trace
# modifierName: extension
# args:
@@ -122,9 +141,6 @@ batches:
- name: collapse
steps:
- kind: CollapseNestingVisitor
- shouldCollapse:
- oneOf: []
- negate: true
tagMappings:
"audit":
- fromSpanTag: "userAgent"
diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go
index 4c7c976c..ae7d5f73 100644
--- a/pkg/aggregator/aggregator.go
+++ b/pkg/aggregator/aggregator.go
@@ -21,13 +21,12 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
- "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"k8s.io/utils/clock"
"github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent"
"github.com/kubewharf/kelemetry/pkg/aggregator/eventdecorator"
- "github.com/kubewharf/kelemetry/pkg/aggregator/linker"
+ linkjob "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job"
"github.com/kubewharf/kelemetry/pkg/aggregator/objectspandecorator"
"github.com/kubewharf/kelemetry/pkg/aggregator/spancache"
"github.com/kubewharf/kelemetry/pkg/aggregator/tracer"
@@ -42,14 +41,11 @@ func init() {
}
type options struct {
- reserveTtl time.Duration
- spanTtl time.Duration
- spanFollowTtl time.Duration
- spanExtraTtl time.Duration
- globalPseudoTags map[string]string
- globalEventTags map[string]string
- subObjectPrimaryPollInterval time.Duration
- subObjectPrimaryPollTimeout time.Duration
+ reserveTtl time.Duration
+ spanTtl time.Duration
+ spanExtraTtl time.Duration
+ globalPseudoTags map[string]string
+ globalEventTags map[string]string
}
func (options *options) Setup(fs *pflag.FlagSet) {
@@ -65,15 +61,10 @@ func (options *options) Setup(fs *pflag.FlagSet) {
time.Minute*30,
"duration of each span",
)
- fs.DurationVar(&options.spanFollowTtl,
- "aggregator-span-follow-ttl",
- 0,
- "duration after expiry of previous span within which new spans are considered FollowsFrom",
- )
fs.DurationVar(&options.spanExtraTtl,
"aggregator-span-extra-ttl",
0,
- "duration for which an object span is retained after the FollowsFrom period has ended",
+ "duration for which an object span is retained in cache after its duration has elapsed",
)
fs.StringToStringVar(&options.globalPseudoTags,
"aggregator-pseudo-span-global-tags",
@@ -85,17 +76,6 @@ func (options *options) Setup(fs *pflag.FlagSet) {
map[string]string{},
"tags applied to all event spans",
)
- fs.DurationVar(&options.subObjectPrimaryPollInterval,
- "aggregator-sub-object-primary-poll-interval",
- time.Second*5,
- "interval to poll primary event before promoting non-primary events",
- )
- fs.DurationVar(&options.subObjectPrimaryPollTimeout,
- "aggregator-sub-object-primary-poll-timeout",
- time.Second*5,
- "timeout to wait for primary event before promoting non-primary events "+
- "(increasing this timeout may lead to indefinite consumer lag",
- )
}
func (options *options) EnableFlag() *bool { return nil }
@@ -104,27 +84,36 @@ type Aggregator interface {
manager.Component
// Send sends an event to the tracer backend.
- // The sub-object ID is an optional identifier that associates the event with an object-scoped context (e.g. resource version).
- // If an event is created with the same sub-object ID with Primary=false,
- // it waits for the primary event to be created and takes it as the parent.
- // If the primary event does not get created after options.subObjectPrimaryBackoff, this event is promoted as primary.
- // If multiple primary events are sent, the slower one (by SpanCache-authoritative timing) is demoted.
- Send(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event, subObjectId *SubObjectId) error
-}
-
-type SubObjectId struct {
- Id string
- Primary bool
+ Send(ctx context.Context, object utilobject.Rich, event *aggregatorevent.Event) error
+
+ // EnsureObjectSpan creates a pseudospan for the object, and triggers any possible relevant linkers.
+ EnsureObjectSpan(
+ ctx context.Context,
+ object utilobject.Rich,
+ eventTime time.Time,
+ ) (tracer.SpanContext, error)
+
+ // GetOrCreatePseudoSpan creates a span following the pseudospan standard with the required tags.
+ GetOrCreatePseudoSpan(
+ ctx context.Context,
+ object utilobject.Rich,
+ pseudoType zconstants.PseudoTypeValue,
+ eventTime time.Time,
+ parent tracer.SpanContext,
+ followsFrom tracer.SpanContext,
+ extraTags map[string]string,
+ dedupId string,
+ ) (span tracer.SpanContext, isNew bool, err error)
}
type aggregator struct {
- options options
- Clock clock.Clock
- Linkers *manager.List[linker.Linker]
- Logger logrus.FieldLogger
- SpanCache spancache.Cache
- Tracer tracer.Tracer
- Metrics metrics.Client
+ options options
+ Clock clock.Clock
+ Logger logrus.FieldLogger
+ SpanCache spancache.Cache
+ Tracer tracer.Tracer
+ Metrics metrics.Client
+ LinkJobPublisher linkjob.Publisher
EventDecorators *manager.List[eventdecorator.Decorator]
ObjectSpanDecorators *manager.List[objectspandecorator.Decorator]
@@ -136,13 +125,10 @@ type aggregator struct {
}
type sendMetric struct {
- Cluster string
- TraceSource string
- HasSubObjectId bool
- Primary bool // whether the subObjectId is primary or not
- PrimaryChanged bool // whether the primary got demoted or non-primary got promoted
- Success bool
- Error metrics.LabeledError
+ Cluster string
+ TraceSource string
+ Success bool
+ Error metrics.LabeledError
}
func (*sendMetric) MetricName() string { return "aggregator_send" }
@@ -155,8 +141,9 @@ type sinceEventMetric struct {
func (*sinceEventMetric) MetricName() string { return "aggregator_send_since_event" }
type lazySpanMetric struct {
- Cluster string
- Result string
+ Cluster string
+ PseudoType zconstants.PseudoTypeValue
+ Result string
}
func (*lazySpanMetric) MetricName() string { return "aggregator_lazy_span" }
@@ -169,13 +156,7 @@ func (aggregator *aggregator) Options() manager.Options {
return &aggregator.options
}
-func (aggregator *aggregator) Init() error {
- if aggregator.options.spanFollowTtl > aggregator.options.spanTtl {
- return fmt.Errorf("invalid option: --span-ttl must not be shorter than --span-follow-ttl")
- }
-
- return nil
-}
+func (aggregator *aggregator) Init() error { return nil }
func (aggregator *aggregator) Start(ctx context.Context) error { return nil }
@@ -185,7 +166,6 @@ func (aggregator *aggregator) Send(
ctx context.Context,
object utilobject.Rich,
event *aggregatorevent.Event,
- subObjectId *SubObjectId,
) (err error) {
sendMetric := &sendMetric{Cluster: object.Cluster, TraceSource: event.TraceSource}
defer aggregator.SendMetric.DeferCount(aggregator.Clock.Now(), sendMetric)
@@ -194,137 +174,28 @@ func (aggregator *aggregator) Send(
With(&sinceEventMetric{Cluster: object.Cluster, TraceSource: event.TraceSource}).
Summary(float64(aggregator.Clock.Since(event.Time).Nanoseconds()))
- var parentSpan tracer.SpanContext
-
- type primaryReservation struct {
- cacheKey string
- uid spancache.Uid
- }
- var reservedPrimary *primaryReservation
-
- if subObjectId != nil {
- sendMetric.HasSubObjectId = true
- sendMetric.Primary = subObjectId.Primary
-
- cacheKey := aggregator.spanCacheKey(object, subObjectId.Id)
-
- if !subObjectId.Primary {
- pollCtx, cancelFunc := context.WithTimeout(ctx, aggregator.options.subObjectPrimaryPollTimeout)
- defer cancelFunc()
-
- if err := wait.PollUntilContextCancel(
- pollCtx,
- aggregator.options.subObjectPrimaryPollInterval,
- true,
- func(context.Context) (done bool, err error) {
- entry, err := aggregator.SpanCache.Fetch(pollCtx, cacheKey)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "PrimaryEventPoll")
- return false, fmt.Errorf("%w during primary event poll", err)
- }
-
- if entry != nil {
- parentSpan, err = aggregator.Tracer.ExtractCarrier(entry.Value)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "ExtractPrimaryCarrier")
- return false, fmt.Errorf("%w during decoding primary span", err)
- }
-
- return true, nil
- }
-
- return false, nil
- },
- ); err != nil {
- if !wait.Interrupted(err) {
- if sendMetric.Error == nil {
- sendMetric.Error = metrics.LabelError(err, "UnknownPrimaryPoll")
- aggregator.Logger.
- WithFields(object.AsFields("object")).
- WithField("event", event.Title).
- WithError(err).
- Warn("Unknown error for primary poll")
- }
- return err
- }
-
- sendMetric.PrimaryChanged = parentSpan == nil
-
- // primary poll timeout, parentSpan == nil, so promote to primary
- sendMetric.Error = nil
- }
- }
-
- if parentSpan == nil {
- // either object ID is primary, or primary poll expired, in which case we should promote
- if err := retry.OnError(retry.DefaultBackoff, spancache.ShouldRetry, func() error {
- entry, err := aggregator.SpanCache.FetchOrReserve(ctx, cacheKey, aggregator.options.reserveTtl)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "PrimaryReserve")
- return fmt.Errorf("%w during primary event fetch-or-reserve", err)
- }
-
- if entry.Value != nil {
- // another primary event was sent, demote this one
- sendMetric.PrimaryChanged = true
- event.Log(
- zconstants.LogTypeRealError,
- fmt.Sprintf("Kelemetry: multiple primary events for %s sent, demoted later event", subObjectId.Id),
- )
-
- parentSpan, err = aggregator.Tracer.ExtractCarrier(entry.Value)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "ExtractAltPrimaryCarrier")
- return fmt.Errorf("%w during decoding primary span", err)
- }
-
- return nil
- }
-
- reservedPrimary = &primaryReservation{
- cacheKey: cacheKey,
- uid: entry.LastUid,
- }
-
- return nil
- }); err != nil {
- if wait.Interrupted(err) {
- sendMetric.Error = metrics.LabelError(err, "PrimaryReserveTimeout")
- }
- return err
- }
- }
- }
-
- if parentSpan == nil {
- // there is no primary span to fallback to, so we are the primary
- parentSpan, err = aggregator.ensureObjectSpan(ctx, object, event.Time)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "EnsureObjectSpan")
- return fmt.Errorf("%w during fetching field span for primary span", err)
- }
+ parentSpan, err := aggregator.EnsureObjectSpan(ctx, object, event.Time)
+ if err != nil {
+ sendMetric.Error = metrics.LabelError(err, "EnsureObjectSpan")
+ return fmt.Errorf("%w during ensuring object span", err)
}
for _, decorator := range aggregator.EventDecorators.Impls {
decorator.Decorate(ctx, object, event)
}
+ tags := zconstants.VersionedKeyToSpanTags(object.VersionedKey)
+ tags[zconstants.NotPseudo] = zconstants.NotPseudo
+ tags[zconstants.TraceSource] = event.TraceSource
+
span := tracer.Span{
Type: event.TraceSource,
Name: event.Title,
StartTime: event.Time,
FinishTime: event.GetEndTime(),
Parent: parentSpan,
- Tags: map[string]string{
- "cluster": object.Cluster,
- "namespace": object.Namespace,
- "name": object.Name,
- "group": object.Group,
- "version": object.Version,
- "resource": object.Resource,
- zconstants.TraceSource: event.TraceSource,
- },
- Logs: event.Logs,
+ Tags: tags,
+ Logs: event.Logs,
}
for tagKey, tagValue := range event.Tags {
span.Tags[tagKey] = fmt.Sprint(tagValue)
@@ -333,31 +204,12 @@ func (aggregator *aggregator) Send(
span.Tags[tagKey] = tagValue
}
- sentSpan, err := aggregator.Tracer.CreateSpan(span)
+ _, err = aggregator.Tracer.CreateSpan(span)
if err != nil {
sendMetric.Error = metrics.LabelError(err, "CreateSpan")
return fmt.Errorf("cannot create span: %w", err)
}
- if reservedPrimary != nil {
- sentSpanRaw, err := aggregator.Tracer.InjectCarrier(sentSpan)
- if err != nil {
- sendMetric.Error = metrics.LabelError(err, "InjectCarrier")
- return fmt.Errorf("%w during serializing sent span ID", err)
- }
-
- if err := aggregator.SpanCache.SetReserved(
- ctx,
- reservedPrimary.cacheKey,
- sentSpanRaw,
- reservedPrimary.uid,
- aggregator.options.spanTtl,
- ); err != nil {
- sendMetric.Error = metrics.LabelError(err, "SetReserved")
- return fmt.Errorf("%w during persisting primary span ID", err)
- }
- }
-
sendMetric.Success = true
aggregator.Logger.WithFields(object.AsFields("object")).
@@ -368,211 +220,185 @@ func (aggregator *aggregator) Send(
return nil
}
-func (aggregator *aggregator) ensureObjectSpan(
+func (agg *aggregator) EnsureObjectSpan(
ctx context.Context,
object utilobject.Rich,
eventTime time.Time,
) (tracer.SpanContext, error) {
- return aggregator.getOrCreateSpan(ctx, object, eventTime, func() (_ tracer.SpanContext, err error) {
- // try to associate a parent object
- var parent *utilobject.Rich
-
- for _, linker := range aggregator.Linkers.Impls {
- parent = linker.Lookup(ctx, object)
- if parent != nil {
- break
- }
- }
+ span, isNew, err := agg.GetOrCreatePseudoSpan(ctx, object, zconstants.PseudoTypeObject, eventTime, nil, nil, nil, "object")
+ if err != nil {
+ return nil, err
+ }
- if parent == nil {
- return nil, nil
- }
+ if isNew {
+ agg.LinkJobPublisher.Publish(&linkjob.LinkJob{
+ Object: object,
+ EventTime: eventTime,
+ Span: span,
+ })
+ }
- // ensure parent object has a span
- return aggregator.ensureObjectSpan(ctx, *parent, eventTime)
- })
+ return span, nil
}
-func (aggregator *aggregator) getOrCreateSpan(
- ctx context.Context,
- object utilobject.Rich,
- eventTime time.Time,
- parentGetter func() (tracer.SpanContext, error),
-) (tracer.SpanContext, error) {
- lazySpanMetric := &lazySpanMetric{
- Cluster: object.Cluster,
- Result: "error",
- }
- defer aggregator.LazySpanMetric.DeferCount(aggregator.Clock.Now(), lazySpanMetric)
-
- cacheKey := aggregator.expiringSpanCacheKey(object, eventTime)
+type spanCreator struct {
+ cacheKey string
- logger := aggregator.Logger.
- WithField("step", "getOrCreateSpan").
- WithFields(object.AsFields("object"))
+ retries int32
+ fetchedSpan tracer.SpanContext
+ reserveUid spancache.Uid
+}
- var reserveUid spancache.Uid
- var returnSpan tracer.SpanContext
- var followsFrom tracer.SpanContext
+func (c *spanCreator) fetchOrReserve(
+ ctx context.Context,
+ agg *aggregator,
+) error {
+ c.retries += 1
- defer func() {
- logger.WithField("cacheKey", cacheKey).WithField("result", lazySpanMetric.Result).Debug("getOrCreateSpan")
- }()
+ entry, err := agg.SpanCache.FetchOrReserve(ctx, c.cacheKey, agg.options.reserveTtl)
+ if err != nil {
+ return metrics.LabelError(fmt.Errorf("%w during fetch-or-reserve of object span", err), "FetchOrReserve")
+ }
- retries := int64(0)
- if err := retry.OnError(retry.DefaultBackoff, spancache.ShouldRetry, func() error {
- retries += 1
- entry, err := aggregator.SpanCache.FetchOrReserve(ctx, cacheKey, aggregator.options.reserveTtl)
+ if entry.Value != nil {
+ // the entry already exists, no additional logic required
+ span, err := agg.Tracer.ExtractCarrier(entry.Value)
if err != nil {
- return metrics.LabelError(fmt.Errorf("%w during initial fetch-or-reserve", err), "FetchOrReserve")
+ return metrics.LabelError(fmt.Errorf("persisted span contains invalid data: %w", err), "BadCarrier")
}
- if entry.Value != nil {
- // the entry already exists, no additional logic required
- reserveUid = []byte{}
- followsFrom = nil
- returnSpan, err = aggregator.Tracer.ExtractCarrier(entry.Value)
- if err != nil {
- return metrics.LabelError(fmt.Errorf("persisted span contains invalid data: %w", err), "BadCarrier")
- }
-
- return nil
- }
-
- // we created a new reservation
- reserveUid = entry.LastUid
- returnSpan = nil
- followsFrom = nil
+ c.fetchedSpan = span
+ return nil
+ }
- // check if this new span is a follower of the previous one
- followsTime := eventTime.Add(-aggregator.options.spanFollowTtl)
- followsKey := aggregator.expiringSpanCacheKey(object, followsTime)
+ // else, a new reservation was created
+ c.reserveUid = entry.LastUid
+ return nil
+}
- if followsKey == cacheKey {
- // previous span expired
- return nil
- }
+func (agg *aggregator) GetOrCreatePseudoSpan(
+ ctx context.Context,
+ object utilobject.Rich,
+ pseudoType zconstants.PseudoTypeValue,
+ eventTime time.Time,
+ parent tracer.SpanContext,
+ followsFrom tracer.SpanContext,
+ extraTags map[string]string,
+ dedupId string,
+) (_span tracer.SpanContext, _isNew bool, _err error) {
+ lazySpanMetric := &lazySpanMetric{
+ Cluster: object.Cluster,
+ PseudoType: pseudoType,
+ Result: "error",
+ }
+ defer agg.LazySpanMetric.DeferCount(agg.Clock.Now(), lazySpanMetric)
- followsEntry, err := aggregator.SpanCache.Fetch(ctx, followsKey)
- if err != nil {
- return metrics.LabelError(fmt.Errorf("error fetching followed entry: %w", err), "FetchFollow")
- }
+ cacheKey := agg.expiringSpanCacheKey(object.Key, eventTime, dedupId)
- if followsEntry == nil {
- // no following target
- return nil
- }
+ logger := agg.Logger.
+ WithField("step", "GetOrCreatePseudoSpan").
+ WithField("dedupId", dedupId).
+ WithFields(object.AsFields("object"))
- if followsEntry.Value == nil {
- return metrics.LabelError(spancache.ErrAlreadyReserved, "FollowPending") // trigger retry
- }
+ defer func() {
+ logger.WithField("cacheKey", cacheKey).WithField("result", lazySpanMetric.Result).Debug("GetOrCreatePseudoSpan")
+ }()
- // we have a following target
- followsFrom, err = aggregator.Tracer.ExtractCarrier(followsEntry.Value)
- if err != nil {
- return metrics.LabelError(fmt.Errorf("followed persisted span contains invalid data: %w", err), "BadFollowCarrier")
- }
+ creator := &spanCreator{cacheKey: cacheKey}
- return nil
- }); err != nil {
- return nil, metrics.LabelError(fmt.Errorf("cannot reserve or fetch span %q: %w", cacheKey, err), "ReserveRetryLoop")
+ if err := retry.OnError(
+ retry.DefaultBackoff,
+ spancache.ShouldRetry,
+ func() error { return creator.fetchOrReserve(ctx, agg) },
+ ); err != nil {
+ return nil, false, metrics.LabelError(fmt.Errorf("cannot reserve or fetch span %q: %w", cacheKey, err), "ReserveRetryLoop")
}
retryCountMetric := lazySpanRetryCountMetric(*lazySpanMetric)
defer func() {
- aggregator.LazySpanRetryCountMetric.With(&retryCountMetric).Summary(float64(retries))
+ agg.LazySpanRetryCountMetric.With(&retryCountMetric).Summary(float64(creator.retries))
}() // take the value of lazySpanMetric later
logger = logger.
- WithField("returnSpan", returnSpan != nil).
- WithField("reserveUid", reserveUid).
- WithField("followsFrom", followsFrom != nil)
+ WithField("returnSpan", creator.fetchedSpan != nil).
+ WithField("reserveUid", creator.reserveUid)
- if returnSpan != nil {
+ if creator.fetchedSpan != nil {
lazySpanMetric.Result = "fetch"
- return returnSpan, nil
+ return creator.fetchedSpan, false, nil
}
// we have a new reservation, need to initialize it now
- startTime := aggregator.Clock.Now()
+ startTime := agg.Clock.Now()
- parent, err := parentGetter()
+ span, err := agg.CreatePseudoSpan(ctx, object, pseudoType, eventTime, parent, followsFrom, extraTags)
if err != nil {
- return nil, fmt.Errorf("cannot fetch parent object: %w", err)
+ return nil, false, metrics.LabelError(fmt.Errorf("cannot create span: %w", err), "CreateSpan")
}
- span, err := aggregator.createSpan(ctx, object, zconstants.NestLevelObject, eventTime, parent, followsFrom)
+ entryValue, err := agg.Tracer.InjectCarrier(span)
if err != nil {
- return nil, metrics.LabelError(fmt.Errorf("cannot create span: %w", err), "CreateSpan")
+ return nil, false, metrics.LabelError(fmt.Errorf("cannot serialize span context: %w", err), "InjectCarrier")
}
- entryValue, err := aggregator.Tracer.InjectCarrier(span)
+ totalTtl := agg.options.spanTtl + agg.options.spanExtraTtl
+ err = agg.SpanCache.SetReserved(ctx, cacheKey, entryValue, creator.reserveUid, totalTtl)
if err != nil {
- return nil, metrics.LabelError(fmt.Errorf("cannot serialize span context: %w", err), "InjectCarrier")
+ return nil, false, metrics.LabelError(fmt.Errorf("cannot persist reserved value: %w", err), "PersistCarrier")
}
- totalTtl := aggregator.options.spanTtl + aggregator.options.spanFollowTtl + aggregator.options.spanExtraTtl
- err = aggregator.SpanCache.SetReserved(ctx, cacheKey, entryValue, reserveUid, totalTtl)
- if err != nil {
- return nil, metrics.LabelError(fmt.Errorf("cannot persist reserved value: %w", err), "PersistCarrier")
- }
+ logger.WithField("duration", agg.Clock.Since(startTime)).Debug("Created new span")
- logger.WithField("duration", aggregator.Clock.Since(startTime)).Debug("Created new span")
+ lazySpanMetric.Result = "create"
- if followsFrom != nil {
- lazySpanMetric.Result = "renew"
- } else {
- lazySpanMetric.Result = "create"
- }
-
- return span, nil
+ return span, true, nil
}
-func (aggregator *aggregator) createSpan(
+func (agg *aggregator) CreatePseudoSpan(
ctx context.Context,
object utilobject.Rich,
- nestLevel string,
+ pseudoType zconstants.PseudoTypeValue,
eventTime time.Time,
parent tracer.SpanContext,
followsFrom tracer.SpanContext,
+ extraTags map[string]string,
) (tracer.SpanContext, error) {
- remainderSeconds := eventTime.Unix() % int64(aggregator.options.spanTtl.Seconds())
+ remainderSeconds := eventTime.Unix() % int64(agg.options.spanTtl.Seconds())
startTime := eventTime.Add(-time.Duration(remainderSeconds) * time.Second)
+
+ tags := zconstants.VersionedKeyToSpanTags(object.VersionedKey)
+ tags[zconstants.TraceSource] = zconstants.TraceSourceObject
+ tags[zconstants.PseudoType] = string(pseudoType)
+ tags["timeStamp"] = startTime.Format(time.RFC3339)
+
span := tracer.Span{
- Type: nestLevel,
- Name: fmt.Sprintf("%s/%s %s", object.Resource, object.Name, nestLevel),
+ Type: string(pseudoType),
+ Name: fmt.Sprintf("%s/%s", object.Resource, object.Name),
StartTime: startTime,
- FinishTime: startTime.Add(aggregator.options.spanTtl),
+ FinishTime: startTime.Add(agg.options.spanTtl),
Parent: parent,
Follows: followsFrom,
- Tags: map[string]string{
- "cluster": object.Cluster,
- "namespace": object.Namespace,
- "name": object.Name,
- "group": object.Group,
- "version": object.Version,
- "resource": object.Resource,
- zconstants.NestLevel: nestLevel,
- zconstants.TraceSource: zconstants.TraceSourceObject,
- "timeStamp": startTime.Format(time.RFC3339),
- },
+ Tags: tags,
}
- for tagKey, tagValue := range aggregator.options.globalPseudoTags {
+ for tagKey, tagValue := range agg.options.globalPseudoTags {
+ span.Tags[tagKey] = tagValue
+ }
+ for tagKey, tagValue := range extraTags {
span.Tags[tagKey] = tagValue
}
- if nestLevel == zconstants.NestLevelObject {
- for _, decorator := range aggregator.ObjectSpanDecorators.Impls {
+ if pseudoType == zconstants.PseudoTypeObject {
+ for _, decorator := range agg.ObjectSpanDecorators.Impls {
decorator.Decorate(ctx, object, span.Type, span.Tags)
}
}
- spanContext, err := aggregator.Tracer.CreateSpan(span)
+ spanContext, err := agg.Tracer.CreateSpan(span)
if err != nil {
return nil, metrics.LabelError(fmt.Errorf("cannot create span: %w", err), "CreateSpan")
}
- aggregator.Logger.
+ agg.Logger.
WithFields(object.AsFields("object")).
WithField("parent", parent).
Debug("CreateSpan")
@@ -580,11 +406,15 @@ func (aggregator *aggregator) createSpan(
return spanContext, nil
}
-func (aggregator *aggregator) expiringSpanCacheKey(object utilobject.Rich, timestamp time.Time) string {
+func (aggregator *aggregator) expiringSpanCacheKey(
+ object utilobject.Key,
+ timestamp time.Time,
+ subObject string,
+) string {
expiringWindow := timestamp.Unix() / int64(aggregator.options.spanTtl.Seconds())
- return aggregator.spanCacheKey(object, fmt.Sprintf("field=object,window=%d", expiringWindow))
+ return aggregator.spanCacheKey(object, fmt.Sprintf("field=%s,window=%d", subObject, expiringWindow))
}
-func (aggregator *aggregator) spanCacheKey(object utilobject.Rich, subObjectId string) string {
- return fmt.Sprintf("%s/%s", object.String(), subObjectId)
+func (aggregator *aggregator) spanCacheKey(object utilobject.Key, window string) string {
+ return fmt.Sprintf("%s/%s", object.String(), window)
}
diff --git a/pkg/aggregator/linker/job/interface.go b/pkg/aggregator/linker/job/interface.go
new file mode 100644
index 00000000..8c9979ca
--- /dev/null
+++ b/pkg/aggregator/linker/job/interface.go
@@ -0,0 +1,63 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linkjob
+
+import (
+ "context"
+ "time"
+
+ "github.com/kubewharf/kelemetry/pkg/aggregator/tracer"
+ "github.com/kubewharf/kelemetry/pkg/manager"
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+type Publisher interface {
+ Publish(job *LinkJob)
+}
+
+type Subscriber interface {
+ Subscribe(ctx context.Context, name string) <-chan *LinkJob
+}
+
+func init() {
+ manager.Global.Provide("linker-job-publisher", manager.Ptr[Publisher](&publisherMux{
+ Mux: manager.NewMux("linker-job-publisher", false),
+ }))
+ manager.Global.Provide("linker-job-subscriber", manager.Ptr[Subscriber](&subscriberMux{
+ Mux: manager.NewMux("linker-job-subscriber", false),
+ }))
+}
+
+type publisherMux struct {
+ *manager.Mux
+}
+
+func (mux *publisherMux) Publish(job *LinkJob) {
+ mux.Impl().(Publisher).Publish(job)
+}
+
+type subscriberMux struct {
+ *manager.Mux
+}
+
+func (mux *subscriberMux) Subscribe(ctx context.Context, name string) <-chan *LinkJob {
+ return mux.Impl().(Subscriber).Subscribe(ctx, name)
+}
+
+type LinkJob struct {
+ Object utilobject.Rich
+ EventTime time.Time
+ Span tracer.SpanContext
+}
diff --git a/pkg/aggregator/linker/job/local/local.go b/pkg/aggregator/linker/job/local/local.go
new file mode 100644
index 00000000..47310d5d
--- /dev/null
+++ b/pkg/aggregator/linker/job/local/local.go
@@ -0,0 +1,107 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linkerjoblocal
+
+import (
+ "context"
+ "sync"
+
+ "github.com/sirupsen/logrus"
+
+ linkjob "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job"
+ "github.com/kubewharf/kelemetry/pkg/manager"
+ "github.com/kubewharf/kelemetry/pkg/metrics"
+ "github.com/kubewharf/kelemetry/pkg/util/channel"
+)
+
+func init() {
+ manager.Global.Provide("linker-job-local-queue", manager.Ptr(&queue{}))
+ manager.Global.ProvideMuxImpl("linker-job-local-publisher", manager.Ptr(&publisher{}), linkjob.Publisher.Publish)
+ manager.Global.ProvideMuxImpl("linker-job-local-subscriber", manager.Ptr(&subscriber{}), linkjob.Subscriber.Subscribe)
+}
+
+type queue struct {
+ Logger logrus.FieldLogger
+
+ subscribersMu sync.RWMutex
+ subscribers map[*subscriberKey]*channel.UnboundedQueue[*linkjob.LinkJob]
+}
+
+type subscriberKey struct{}
+
+func (queue *queue) Options() manager.Options { return &manager.NoOptions{} }
+func (queue *queue) Init() error {
+ queue.subscribers = map[*subscriberKey]*channel.UnboundedQueue[*linkjob.LinkJob]{}
+ return nil
+}
+func (queue *queue) Start(ctx context.Context) error { return nil }
+func (queue *queue) Close(ctx context.Context) error { return nil }
+
+type publisher struct {
+ Queue *queue
+ manager.MuxImplBase
+}
+
+func (publisher *publisher) Options() manager.Options { return &manager.NoOptions{} }
+func (publisher *publisher) Init() error { return nil }
+func (publisher *publisher) Start(ctx context.Context) error { return nil }
+func (publisher *publisher) Close(ctx context.Context) error { return nil }
+func (publisher *publisher) MuxImplName() (name string, isDefault bool) { return "local", true }
+func (publisher *publisher) Publish(job *linkjob.LinkJob) {
+ publisher.Queue.subscribersMu.RLock()
+ defer publisher.Queue.subscribersMu.RUnlock()
+
+ for _, sub := range publisher.Queue.subscribers {
+ sub.Send(job)
+ }
+}
+
+type subscriber struct {
+ Queue *queue
+ Metrics metrics.Client
+ manager.MuxImplBase
+}
+
+type queueMetricTags struct {
+ Name string
+}
+
+func (*queueMetricTags) MetricName() string { return "linker_local_worker_lag" }
+
+func (subscriber *subscriber) Options() manager.Options { return &manager.NoOptions{} }
+func (subscriber *subscriber) Init() error { return nil }
+func (subscriber *subscriber) Start(ctx context.Context) error { return nil }
+func (subscriber *subscriber) Close(ctx context.Context) error { return nil }
+func (subscriber *subscriber) MuxImplName() (name string, isDefault bool) { return "local", true }
+func (subscriber *subscriber) Subscribe(ctx context.Context, name string) <-chan *linkjob.LinkJob {
+ queue := channel.NewUnboundedQueue[*linkjob.LinkJob](16)
+ channel.InitMetricLoop(queue, subscriber.Metrics, &queueMetricTags{Name: name})
+
+ subscriber.Queue.subscribersMu.Lock()
+ defer subscriber.Queue.subscribersMu.Unlock()
+
+ key := &subscriberKey{}
+ subscriber.Queue.subscribers[key] = queue
+
+ go func() {
+ <-ctx.Done()
+
+ subscriber.Queue.subscribersMu.Lock()
+ defer subscriber.Queue.subscribersMu.Unlock()
+ delete(subscriber.Queue.subscribers, key)
+ }()
+
+ return queue.Receiver()
+}
diff --git a/pkg/aggregator/linker/job/worker/worker.go b/pkg/aggregator/linker/job/worker/worker.go
new file mode 100644
index 00000000..c5ee82d8
--- /dev/null
+++ b/pkg/aggregator/linker/job/worker/worker.go
@@ -0,0 +1,168 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linkjobworker
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/sirupsen/logrus"
+ "github.com/spf13/pflag"
+ "k8s.io/utils/clock"
+ "k8s.io/utils/pointer"
+
+ "github.com/kubewharf/kelemetry/pkg/aggregator"
+ "github.com/kubewharf/kelemetry/pkg/aggregator/linker"
+ linkjob "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job"
+ "github.com/kubewharf/kelemetry/pkg/manager"
+ "github.com/kubewharf/kelemetry/pkg/metrics"
+ "github.com/kubewharf/kelemetry/pkg/util/shutdown"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
+)
+
+func init() {
+ manager.Global.Provide("linker-job-worker", manager.Ptr(&worker{}))
+}
+
+type workerOptions struct {
+ WorkerCount int
+}
+
+func (options *workerOptions) Setup(fs *pflag.FlagSet) {
+ fs.IntVar(&options.WorkerCount, "linker-worker-count", 0, "Number of workers to execute link jobs")
+}
+func (options *workerOptions) EnableFlag() *bool { return pointer.Bool(options.WorkerCount > 0) }
+
+type worker struct {
+ options workerOptions
+ Logger logrus.FieldLogger
+ Clock clock.Clock
+ Linkers *manager.List[linker.Linker]
+ Subscriber linkjob.Subscriber
+ Aggregator aggregator.Aggregator
+ ExecuteJobMetric *metrics.Metric[*executeJobMetric]
+
+ ch <-chan *linkjob.LinkJob
+}
+
+type executeJobMetric struct {
+ Linker string
+ Error metrics.LabeledError
+}
+
+func (*executeJobMetric) MetricName() string { return "linker_job_exec" }
+
+func (worker *worker) Options() manager.Options { return &worker.options }
+func (worker *worker) Init() error {
+ worker.ch = worker.Subscriber.Subscribe(context.Background(), "worker") // background context, never unsubscribe
+ return nil
+}
+
+func (worker *worker) Start(ctx context.Context) error {
+ for workerId := 0; workerId < worker.options.WorkerCount; workerId++ {
+ go func(workerId int) {
+ defer shutdown.RecoverPanic(worker.Logger)
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case job := <-worker.ch:
+ worker.executeJob(ctx, worker.Logger.WithFields(job.Object.AsFields("job")), job)
+ }
+ }
+ }(workerId)
+ }
+
+ return nil
+}
+func (worker *worker) Close(ctx context.Context) error { return nil }
+
+func (worker *worker) executeJob(ctx context.Context, logger logrus.FieldLogger, job *linkjob.LinkJob) {
+ for _, linker := range worker.Linkers.Impls {
+ linkerLogger := logger.WithField("linker", linker.LinkerName())
+ if err := worker.execute(ctx, linkerLogger, linker, job); err != nil {
+ linkerLogger.WithError(err).Error("generating links")
+ }
+ }
+}
+
+func (worker *worker) execute(ctx context.Context, logger logrus.FieldLogger, linker linker.Linker, job *linkjob.LinkJob) error {
+ logger.Debug("execute linker")
+ startTime := worker.Clock.Now()
+ links, err := linker.Lookup(ctx, job.Object)
+ worker.ExecuteJobMetric.DeferCount(startTime, &executeJobMetric{
+ Linker: linker.LinkerName(),
+ Error: err,
+ })
+ if err != nil {
+ return metrics.LabelError(fmt.Errorf("calling linker: %w", err), "CallLinker")
+ }
+
+ for _, link := range links {
+ linkedSpan, err := worker.Aggregator.EnsureObjectSpan(ctx, link.Object, job.EventTime)
+ if err != nil {
+ return metrics.LabelError(fmt.Errorf("creating object span: %w", err), "CreateLinkedObjectSpan")
+ }
+
+ forwardTags := map[string]string{}
+ zconstants.TagLinkedObject(forwardTags, zconstants.LinkRef{
+ Key: link.Object.Key,
+ Role: link.Role,
+ Class: link.Class,
+ })
+ _, _, err = worker.Aggregator.GetOrCreatePseudoSpan(
+ ctx,
+ job.Object,
+ zconstants.PseudoTypeLink,
+ job.EventTime,
+ job.Span,
+ nil,
+ forwardTags,
+ link.DedupId,
+ )
+ if err != nil {
+ return metrics.LabelError(
+ fmt.Errorf("creating link span from source object to linked object: %w", err),
+ "CreateForwardLinkSpan",
+ )
+ }
+
+ backwardTags := map[string]string{}
+ zconstants.TagLinkedObject(backwardTags, zconstants.LinkRef{
+ Key: job.Object.Key,
+ Role: zconstants.ReverseLinkRole(link.Role),
+ Class: link.Class,
+ })
+ _, _, err = worker.Aggregator.GetOrCreatePseudoSpan(
+ ctx,
+ link.Object,
+ zconstants.PseudoTypeLink,
+ job.EventTime,
+ linkedSpan,
+ nil,
+ backwardTags,
+ fmt.Sprintf("%s@%s", link.DedupId, job.Object.String()),
+ )
+ if err != nil {
+ return metrics.LabelError(
+ fmt.Errorf("creating link span from linked object to source object: %w", err),
+ "CreateBackwardLinkSpan",
+ )
+ }
+ }
+
+ return nil
+}
diff --git a/pkg/aggregator/linker/linker.go b/pkg/aggregator/linker/linker.go
index f648be25..2bf517ee 100644
--- a/pkg/aggregator/linker/linker.go
+++ b/pkg/aggregator/linker/linker.go
@@ -18,8 +18,17 @@ import (
"context"
utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
type Linker interface {
- Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich
+ LinkerName() string
+ Lookup(ctx context.Context, object utilobject.Rich) ([]LinkerResult, error)
+}
+
+type LinkerResult struct {
+ Object utilobject.Rich
+ Role zconstants.LinkRoleValue
+ Class string
+ DedupId string
}
diff --git a/pkg/annotationlinker/linker.go b/pkg/annotationlinker/linker.go
index 46841d36..33707d05 100644
--- a/pkg/annotationlinker/linker.go
+++ b/pkg/annotationlinker/linker.go
@@ -17,6 +17,7 @@ package annotationlinker
import (
"context"
"encoding/json"
+ "fmt"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
@@ -26,7 +27,9 @@ import (
"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
"github.com/kubewharf/kelemetry/pkg/manager"
+ "github.com/kubewharf/kelemetry/pkg/metrics"
utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
func init() {
@@ -58,7 +61,8 @@ func (ctrl *controller) Init() error { return nil }
func (ctrl *controller) Start(ctx context.Context) error { return nil }
func (ctrl *controller) Close(ctx context.Context) error { return nil }
-func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich {
+func (ctrl *controller) LinkerName() string { return "annotation-linker" }
+func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) ([]linker.LinkerResult, error) {
raw := object.Raw
logger := ctrl.Logger.WithFields(object.AsFields("object"))
@@ -70,13 +74,12 @@ func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
raw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey)
if err != nil {
- logger.WithError(err).Error("cannot fetch object value")
- return nil
+ return nil, metrics.LabelError(fmt.Errorf("cannot fetch object value: %w", err), "FetchCache")
}
if raw == nil {
logger.Debug("object no longer exists")
- return nil
+ return nil, nil
}
}
@@ -84,8 +87,7 @@ func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
ref := &ParentLink{}
err := json.Unmarshal([]byte(ann), ref)
if err != nil {
- logger.WithError(err).Error("cannot parse ParentLink annotation")
- return nil
+ return nil, metrics.LabelError(fmt.Errorf("cannot parse ParentLink annotation: %w", err), "ParseAnnotation")
}
if ref.Cluster == "" {
@@ -93,10 +95,14 @@ func (ctrl *controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
}
objectRef := ref.ToRich()
- logger.WithField("parent", objectRef).Debug("Resolved parent")
+ logger.WithFields(objectRef.AsFields("parent")).Debug("Resolved parent")
- return &objectRef
+ return []linker.LinkerResult{{
+ Object: objectRef,
+ Role: zconstants.LinkRoleParent,
+ DedupId: "annotation",
+ }}, nil
}
- return nil
+ return nil, nil
}
diff --git a/pkg/audit/consumer/consumer.go b/pkg/audit/consumer/consumer.go
index a937cb70..38519834 100644
--- a/pkg/audit/consumer/consumer.go
+++ b/pkg/audit/consumer/consumer.go
@@ -274,15 +274,7 @@ func (recv *receiver) handleItem(
Resource: objectRef.Resource,
}).Summary(float64(e2eLatency.Nanoseconds()))
- var subObjectId *aggregator.SubObjectId
- if recv.options.enableSubObject && (message.Verb == audit.VerbUpdate || message.Verb == audit.VerbPatch) {
- subObjectId = &aggregator.SubObjectId{
- Id: fmt.Sprintf("rv=%s", message.ObjectRef.ResourceVersion),
- Primary: message.ResponseStatus.Code < 300,
- }
- }
-
- err := recv.Aggregator.Send(ctx, objectRef, event, subObjectId)
+ err := recv.Aggregator.Send(ctx, objectRef, event)
if err != nil {
fieldLogger.WithError(err).Error()
} else {
diff --git a/pkg/event/controller.go b/pkg/event/controller.go
index e77fd5b6..669e1664 100644
--- a/pkg/event/controller.go
+++ b/pkg/event/controller.go
@@ -324,7 +324,7 @@ func (ctrl *controller) handleEvent(ctx context.Context, event *corev1.Event) {
Version: gvr.Version,
},
Uid: event.InvolvedObject.UID,
- }, aggregatorEvent, nil); err != nil {
+ }, aggregatorEvent); err != nil {
logger.WithError(err).Error("Cannot send trace")
metric.Error = metrics.LabelError(err, "SendTrace")
return
diff --git a/pkg/frontend/backend/interface.go b/pkg/frontend/backend/interface.go
index 82aedcff..c447cc79 100644
--- a/pkg/frontend/backend/interface.go
+++ b/pkg/frontend/backend/interface.go
@@ -23,6 +23,7 @@ import (
"github.com/jaegertracing/jaeger/storage/spanstore"
"k8s.io/utils/clock"
+ tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/manager"
"github.com/kubewharf/kelemetry/pkg/metrics"
)
@@ -38,7 +39,6 @@ type Backend interface {
List(
ctx context.Context,
query *spanstore.TraceQueryParameters,
- exclusive bool,
) ([]*TraceThumbnail, error)
// Gets the full tree of a trace based on the identifier returned from a prvious call to List.
@@ -59,9 +59,13 @@ type TraceThumbnail struct {
// Identifier is a serializable object that identifies the trace in GetTrace calls.
Identifier any
- Spans []*model.Span
+ Spans *tftree.SpanTree
}
+func (tt *TraceThumbnail) GetSpans() *tftree.SpanTree { return tt.Spans }
+func (tt *TraceThumbnail) GetMetadata() any { return tt.Identifier }
+func (tt *TraceThumbnail) FromThumbnail(src *TraceThumbnail) { *tt = *src }
+
type mux struct {
*manager.Mux
Clock clock.Clock
@@ -81,10 +85,9 @@ func (*getMetric) MetricName() string { return "jaeger_backend_get" }
func (mux *mux) List(
ctx context.Context,
query *spanstore.TraceQueryParameters,
- exclusive bool,
) ([]*TraceThumbnail, error) {
defer mux.ListMetric.DeferCount(mux.Clock.Now(), &listMetric{})
- return mux.Impl().(Backend).List(ctx, query, exclusive)
+ return mux.Impl().(Backend).List(ctx, query)
}
func (mux *mux) Get(
diff --git a/pkg/frontend/backend/jaeger-storage/backend.go b/pkg/frontend/backend/jaeger-storage/backend.go
index 8cea1ee2..dcad2262 100644
--- a/pkg/frontend/backend/jaeger-storage/backend.go
+++ b/pkg/frontend/backend/jaeger-storage/backend.go
@@ -31,13 +31,11 @@ import (
"github.com/spf13/pflag"
"github.com/spf13/viper"
"go.uber.org/zap"
- "k8s.io/apimachinery/pkg/util/sets"
jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/manager"
utiljaeger "github.com/kubewharf/kelemetry/pkg/util/jaeger"
- utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
"github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
@@ -152,123 +150,44 @@ func (backend *Backend) Close(ctx context.Context) error { return nil }
func (backend *Backend) List(
ctx context.Context,
params *spanstore.TraceQueryParameters,
- exclusive bool,
) ([]*jaegerbackend.TraceThumbnail, error) {
- filterTags := map[string]string{}
- for key, val := range params.Tags {
- filterTags[key] = val
- }
- if len(params.OperationName) > 0 {
- filterTags["cluster"] = params.OperationName
- }
-
- // TODO support additional user-defined trace sources
- var traces []*model.Trace
+ traceThumbnails := []*jaegerbackend.TraceThumbnail{}
for _, traceSource := range zconstants.KnownTraceSources(false) {
- if len(traces) >= params.NumTraces {
+ if len(traceThumbnails) >= params.NumTraces {
break
}
newParams := &spanstore.TraceQueryParameters{
ServiceName: traceSource,
- Tags: filterTags,
+ Tags: params.Tags,
StartTimeMin: params.StartTimeMin,
StartTimeMax: params.StartTimeMax,
DurationMin: params.DurationMin,
DurationMax: params.DurationMax,
- NumTraces: params.NumTraces - len(traces),
+ NumTraces: params.NumTraces - len(traceThumbnails),
}
- newTraces, err := backend.reader.FindTraces(ctx, newParams)
+ traces, err := backend.reader.FindTraces(ctx, newParams)
if err != nil {
return nil, fmt.Errorf("find traces from backend err: %w", err)
}
- traces = append(traces, newTraces...)
- }
-
- var traceThumbnails []*jaegerbackend.TraceThumbnail
-
- // a stateful function that determines only returns true for each valid resultant root span the first time
- var deduplicator func(*model.Span) bool
- if exclusive {
- // exclusive mode, each object under trace should have a list entry
- type objectInTrace struct {
- traceId model.TraceID
- key utilobject.Key
- }
- seenObjects := sets.New[objectInTrace]()
- deduplicator = func(span *model.Span) bool {
- key, hasKey := utilobject.FromSpan(span)
- if !hasKey {
- return false // not a root
- }
-
- field, hasField := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel)
- if !hasField || field.VStr != zconstants.NestLevelObject {
- return false // not an object root
+ for _, trace := range traces {
+ if len(trace.Spans) == 0 {
+ continue
}
- fullKey := objectInTrace{
- traceId: span.TraceID,
- key: key,
- }
-
- if seenObjects.Has(fullKey) {
- return false // a known root
- }
+ tree := tftree.NewSpanTree(trace.Spans)
- for reqKey, reqValue := range filterTags {
- if value, exists := model.KeyValues(span.Tags).FindByKey(reqKey); !exists || value.VStr != reqValue {
- return false // not a matched root
- }
+ thumbnail := &jaegerbackend.TraceThumbnail{
+ Identifier: identifier{
+ TraceId: tree.Root.TraceID,
+ SpanId: tree.Root.SpanID,
+ },
+ Spans: tree,
}
- seenObjects.Insert(fullKey)
- return true
- }
- } else {
- // non exclusive mode, display full trace, so we want each full trace to display exactly once.
- seenTraces := sets.New[model.TraceID]()
- deduplicator = func(span *model.Span) bool {
- if len(span.References) > 0 {
- return false // we only want the root
- }
-
- if seenTraces.Has(span.TraceID) {
- return false
- }
-
- seenTraces.Insert(span.TraceID)
- return true
- }
- }
-
- for _, trace := range traces {
- if len(trace.Spans) == 0 {
- continue
- }
-
- for _, span := range trace.Spans {
- if deduplicator(span) {
- tree := tftree.NewSpanTree(trace.Spans)
- if err := tree.SetRoot(span.SpanID); err != nil {
- return nil, fmt.Errorf("unexpected SetRoot error for span ID from trace: %w", err)
- }
-
- thumbnail := &jaegerbackend.TraceThumbnail{
- Identifier: identifier{
- TraceId: span.TraceID,
- SpanId: span.SpanID,
- },
- Spans: tree.GetSpans(),
- }
- traceThumbnails = append(traceThumbnails, thumbnail)
-
- backend.Logger.WithField("ident", thumbnail.Identifier).
- WithField("filteredSpans", len(thumbnail.Spans)).
- Debug("matched trace")
- }
+ traceThumbnails = append(traceThumbnails, thumbnail)
}
}
diff --git a/pkg/frontend/http/trace/server.go b/pkg/frontend/http/trace/server.go
index 2f5466d2..980e8251 100644
--- a/pkg/frontend/http/trace/server.go
+++ b/pkg/frontend/http/trace/server.go
@@ -106,7 +106,7 @@ func (server *server) handleTrace(ctx *gin.Context, metric *requestMetric) (code
}
if query.DisplayMode == "" {
- query.DisplayMode = "tracing [exclusive]"
+ query.DisplayMode = "tracing"
}
trace, code, err := server.findTrace(metric, query.DisplayMode, query)
diff --git a/pkg/frontend/reader/merge.go b/pkg/frontend/reader/merge.go
deleted file mode 100644
index d35d5c38..00000000
--- a/pkg/frontend/reader/merge.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2023 The Kelemetry Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package jaegerreader
-
-import (
- "github.com/jaegertracing/jaeger/model"
- "k8s.io/apimachinery/pkg/util/sets"
-
- jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
- utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
-)
-
-type mergeMap struct {
- ptrSet sets.Set[*mergeEntry]
- fromKeys map[utilobject.Key]*mergeEntry
-}
-
-type mergeEntry struct {
- keys sets.Set[utilobject.Key]
- identifiers []any
- spans []*model.Span
-}
-
-func singletonMerged(keys sets.Set[utilobject.Key], thumbnail *jaegerbackend.TraceThumbnail) *mergeEntry {
- return &mergeEntry{
- keys: keys,
- identifiers: []any{thumbnail.Identifier},
- spans: thumbnail.Spans,
- }
-}
-
-func (entry *mergeEntry) join(other *mergeEntry) {
- for key := range other.keys {
- entry.keys.Insert(key)
- }
-
- entry.identifiers = append(entry.identifiers, other.identifiers...)
- entry.spans = append(entry.spans, other.spans...)
-}
-
-// add a thumbnail with a preferred root key.
-func (m *mergeMap) add(keys sets.Set[utilobject.Key], thumbnail *jaegerbackend.TraceThumbnail) {
- entry := singletonMerged(keys.Clone(), thumbnail)
- m.ptrSet.Insert(entry)
-
- dups := sets.New[*mergeEntry]()
-
- for key := range keys {
- if prev, hasPrev := m.fromKeys[key]; hasPrev {
- dups.Insert(prev)
- }
- }
-
- for dup := range dups {
- entry.join(dup)
- m.ptrSet.Delete(dup)
- }
-
- for key := range entry.keys {
- // including all new and joined keys
- m.fromKeys[key] = entry
- }
-}
-
-func mergeSegments(thumbnails []*jaegerbackend.TraceThumbnail) []*mergeEntry {
- m := mergeMap{
- ptrSet: sets.New[*mergeEntry](),
- fromKeys: map[utilobject.Key]*mergeEntry{},
- }
-
- for _, thumbnail := range thumbnails {
- keys := utilobject.FromSpans(thumbnail.Spans)
- m.add(keys, thumbnail)
- }
-
- return m.ptrSet.UnsortedList()
-}
diff --git a/pkg/frontend/reader/merge/merge.go b/pkg/frontend/reader/merge/merge.go
new file mode 100644
index 00000000..88bb6bdb
--- /dev/null
+++ b/pkg/frontend/reader/merge/merge.go
@@ -0,0 +1,628 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package merge
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "sync/atomic"
+ "time"
+
+ "github.com/jaegertracing/jaeger/model"
+ "k8s.io/apimachinery/pkg/util/sets"
+
+ jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
+ tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
+ tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ reflectutil "github.com/kubewharf/kelemetry/pkg/util/reflect"
+ "github.com/kubewharf/kelemetry/pkg/util/semaphore"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
+)
+
+type objKey = utilobject.Key
+
+type Merger[M any] struct {
+ objects map[objKey]*object[M]
+}
+
+type TraceWithMetadata[M any] struct {
+ Tree *tftree.SpanTree
+ Metadata M
+}
+
+type RawTree struct {
+ Tree *tftree.SpanTree
+}
+
+func (tr RawTree) GetSpans() *tftree.SpanTree { return tr.Tree }
+func (tr RawTree) GetMetadata() struct{} { return struct{}{} }
+func (tr RawTree) FromThumbnail(self *RawTree, tt *jaegerbackend.TraceThumbnail) {
+ self.Tree = tt.Spans
+}
+
+func (merger *Merger[M]) AddTraces(trees []TraceWithMetadata[M]) (_affected sets.Set[objKey], _err error) {
+ if merger.objects == nil {
+ merger.objects = make(map[objKey]*object[M])
+ }
+
+ affected := sets.New[objKey]()
+ for _, trace := range trees {
+ key := zconstants.ObjectKeyFromSpan(trace.Tree.Root)
+ affected.Insert(key)
+
+ if obj, hasPrev := merger.objects[key]; hasPrev {
+ if err := obj.merge(trace.Tree, trace.Metadata); err != nil {
+ return nil, err
+ }
+ } else {
+ obj, err := newObject[M](key, trace.Tree, trace.Metadata)
+ if err != nil {
+ return nil, err
+ }
+
+ merger.objects[key] = obj
+ }
+ }
+
+ for key := range affected {
+ merger.objects[key].identifyLinks()
+ }
+
+ return affected, nil
+}
+
+type followLinkPool[M any] struct {
+ sem *semaphore.Semaphore
+ knownKeys sets.Set[objKey]
+ lister ListFunc[M]
+ startTime, endTime time.Time
+ merger *Merger[M]
+}
+
+func (fl *followLinkPool[M]) scheduleFrom(obj *object[M], followLimit *atomic.Int32, linkSelector tfconfig.LinkSelector) {
+ admittedLinks := []TargetLink{}
+
+ for _, link := range obj.links {
+ if _, known := fl.knownKeys[link.Key]; known {
+ admittedLinks = append(admittedLinks, link)
+ continue
+ }
+ if followLimit.Add(-1) < 0 {
+ continue
+ }
+
+ parentKey, childKey, parentIsSource := obj.key, link.Key, true
+ if link.Role == zconstants.LinkRoleParent {
+ parentKey, childKey, parentIsSource = link.Key, obj.key, false
+ }
+
+ subSelector := linkSelector.Admit(parentKey, childKey, parentIsSource, link.Class)
+ if subSelector != nil {
+ admittedLinks = append(admittedLinks, link)
+ fl.knownKeys.Insert(link.Key)
+ fl.schedule(link.Key, subSelector, followLimit, int32(fl.endTime.Sub(fl.startTime)/(time.Minute*30)))
+ }
+ }
+
+ obj.links = admittedLinks
+}
+
+func (fl *followLinkPool[M]) schedule(key objKey, linkSelector tfconfig.LinkSelector, followLimit *atomic.Int32, traceLimit int32) {
+ fl.sem.Schedule(func(ctx context.Context) (semaphore.Publish, error) {
+ thumbnails, err := fl.lister(ctx, key, fl.startTime, fl.endTime, int(traceLimit))
+ if err != nil {
+ return nil, fmt.Errorf("fetching linked traces: %w", err)
+ }
+
+ return func() error {
+ affected, err := fl.merger.AddTraces(thumbnails)
+ if err != nil {
+ return err
+ }
+
+ for key := range affected {
+ fl.scheduleFrom(fl.merger.objects[key], followLimit, linkSelector)
+ }
+
+ return nil
+ }, nil
+ })
+}
+
+type ListFunc[M any] func(
+ ctx context.Context,
+ key objKey,
+ startTime, endTime time.Time,
+ limit int,
+) ([]TraceWithMetadata[M], error)
+
+func (merger *Merger[M]) FollowLinks(
+ ctx context.Context,
+ linkSelector tfconfig.LinkSelector,
+ startTime, endTime time.Time,
+ lister ListFunc[M],
+ concurrency int,
+ limit int32,
+ limitIsGlobal bool,
+) error {
+ fl := &followLinkPool[M]{
+ sem: semaphore.New(concurrency),
+ knownKeys: sets.New[objKey](),
+ lister: lister,
+ startTime: startTime,
+ endTime: endTime,
+ merger: merger,
+ }
+
+ for _, obj := range merger.objects {
+ fl.knownKeys.Insert(obj.key)
+ }
+
+ globalLimit := new(atomic.Int32)
+ globalLimit.Store(limit)
+
+ for _, obj := range merger.objects {
+ var remainingLimit *atomic.Int32
+ if limitIsGlobal {
+ remainingLimit = globalLimit
+ } else {
+ remainingLimit = new(atomic.Int32)
+ remainingLimit.Store(limit)
+ }
+
+ fl.scheduleFrom(obj, remainingLimit, linkSelector)
+ }
+
+ if err := fl.sem.Run(ctx); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (merger *Merger[M]) MergeTraces() ([]*MergeTree[M], error) {
+ abLinks := abLinkMap{}
+
+ for _, obj := range merger.objects {
+ for _, link := range obj.links {
+ abLink := abLinkFromTargetLink(obj.key, link)
+ abLinks.insert(abLink)
+ }
+ }
+
+ connectedComps := merger.findConnectedComponents(merger.objects, abLinks)
+ mergeTrees := make([]*MergeTree[M], 0, len(connectedComps))
+ for _, keys := range connectedComps {
+ var members []*object[M]
+ for _, key := range keys {
+ members = append(members, merger.objects[key])
+ }
+
+ mergeTree, err := newMergeTree(members, abLinks)
+ if err != nil {
+ return nil, err
+ }
+
+ mergeTrees = append(mergeTrees, mergeTree)
+ }
+
+ return mergeTrees, nil
+}
+
+type object[M any] struct {
+ key objKey
+ metadata []M
+ tree *tftree.SpanTree
+
+ links []TargetLink
+}
+
+func newObject[M any](key objKey, trace *tftree.SpanTree, metadata M) (*object[M], error) {
+ clonedTree, err := trace.Clone()
+ if err != nil {
+ return nil, fmt.Errorf("clone spans: %w", err)
+ }
+ obj := &object[M]{
+ key: key,
+ metadata: []M{metadata},
+ tree: clonedTree,
+ }
+ return obj, nil
+}
+
+func (obj *object[M]) merge(trace *tftree.SpanTree, metadata M) error {
+ obj.metadata = append(obj.metadata, metadata)
+
+ mergeRoot(obj.tree.Root, trace.Root)
+
+ copyVisitor := ©TreeVisitor{to: obj.tree, toParent: obj.tree.Root.SpanID}
+ trace.Visit(copyVisitor)
+ if copyVisitor.err != nil {
+ return copyVisitor.err
+ }
+
+ return nil
+}
+
+func mergeRoot(base *model.Span, tail *model.Span) {
+ mergeRootInterval(base, tail)
+ mergeRootTags(base, tail)
+ mergeRootLogs(base, tail)
+}
+
+func mergeRootInterval(base *model.Span, tail *model.Span) {
+ startTime := base.StartTime
+ if tail.StartTime.Before(startTime) {
+ startTime = tail.StartTime
+ }
+
+ endTime := base.StartTime.Add(base.Duration)
+ tailEndTime := tail.StartTime.Add(tail.Duration)
+ if tailEndTime.After(endTime) {
+ endTime = tailEndTime
+ }
+
+ base.StartTime = startTime
+ base.Duration = endTime.Sub(startTime)
+}
+
+func mergeRootTags(base *model.Span, tail *model.Span) {
+ tagPos := map[string]int{}
+ for pos, tag := range base.Tags {
+ tagPos[tag.Key] = pos
+ }
+
+ for _, tag := range tail.Tags {
+ if pos, hasTag := tagPos[tag.Key]; hasTag {
+ if tail.StartTime.After(base.StartTime) {
+ // the newer value wins
+ base.Tags[pos] = tag
+ }
+ } else {
+ base.Tags = append(base.Tags, tag)
+ }
+ }
+}
+
+func mergeRootLogs(base *model.Span, tail *model.Span) {
+ base.Logs = append(base.Logs, tail.Logs...)
+}
+
+func (obj *object[M]) identifyLinks() {
+ for spanId := range obj.tree.Children(obj.tree.Root.SpanID) {
+ span := obj.tree.Span(spanId)
+ pseudoType, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType)
+ if !(isPseudo && pseudoType.VStr == string(zconstants.PseudoTypeLink)) {
+ continue
+ }
+
+ target, hasTarget := zconstants.LinkedKeyFromSpan(span)
+ if !hasTarget {
+ continue
+ }
+
+ linkRoleTag, hasLinkRole := model.KeyValues(span.Tags).FindByKey(zconstants.LinkRole)
+ if !hasLinkRole {
+ continue
+ }
+ linkRole := linkRoleTag.VStr
+
+ linkClassTag, hasLinkClass := model.KeyValues(span.Tags).FindByKey(zconstants.LinkClass)
+ linkClass := ""
+ if hasLinkClass {
+ linkClass = linkClassTag.VStr
+ }
+
+ obj.links = append(obj.links, TargetLink{
+ Key: target,
+ Role: zconstants.LinkRoleValue(linkRole),
+ Class: linkClass,
+ })
+ }
+}
+
+type TargetLink struct {
+ Key objKey
+ Role zconstants.LinkRoleValue
+ Class string
+}
+
+type copyTreeVisitor struct {
+ to *tftree.SpanTree
+ toParent model.SpanID
+ err error
+}
+
+func (visitor *copyTreeVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor {
+ if span.SpanID != tree.Root.SpanID {
+ spanCopy, err := tftree.CopySpan(span)
+ if err != nil {
+ visitor.err = err
+ return nil
+ }
+
+ visitor.to.Add(spanCopy, visitor.toParent)
+
+ return ©TreeVisitor{
+ to: visitor.to,
+ toParent: spanCopy.SpanID,
+ }
+ }
+
+ return visitor
+}
+
+func (visitor *copyTreeVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {}
+
+type abLink struct {
+ alpha, beta objKey
+
+ alphaIsParent bool // this needs to be changed if there are link roles other than parent and child
+ class string
+}
+
+func (link abLink) isParent(key objKey) bool {
+ if link.alphaIsParent {
+ return link.alpha == key
+ } else {
+ return link.beta == key
+ }
+}
+
+func abLinkFromTargetLink(subject objKey, link TargetLink) abLink {
+ if groupingKeyLess(subject, link.Key) {
+ return abLink{
+ alpha: subject,
+ beta: link.Key,
+ alphaIsParent: link.Role == zconstants.LinkRoleChild,
+ class: link.Class,
+ }
+ } else {
+ return abLink{
+ beta: subject,
+ alpha: link.Key,
+ alphaIsParent: link.Role != zconstants.LinkRoleChild,
+ class: link.Class,
+ }
+ }
+}
+
+func groupingKeyLess(left, right objKey) bool {
+ if left.Group != right.Group {
+ return left.Group < right.Group
+ }
+
+ if left.Resource != right.Resource {
+ return left.Resource < right.Resource
+ }
+
+ if left.Cluster != right.Cluster {
+ return left.Cluster < right.Cluster
+ }
+
+ if left.Namespace != right.Namespace {
+ return left.Namespace < right.Namespace
+ }
+
+ if left.Name != right.Name {
+ return left.Name < right.Name
+ }
+
+ return false
+}
+
+type abLinkMap map[objKey]map[objKey]abLink
+
+func (m abLinkMap) insert(link abLink) {
+ m.insertDirected(link.alpha, link.beta, link)
+ m.insertDirected(link.beta, link.alpha, link)
+}
+
+func (m abLinkMap) insertDirected(k1, k2 objKey, link abLink) {
+ v1, hasK1 := m[k1]
+ if !hasK1 {
+ v1 = map[objKey]abLink{}
+ m[k1] = v1
+ }
+ v1[k2] = link
+}
+
+func (m abLinkMap) detectRoot(seed objKey, vertexFilter func(objKey) bool) (_root objKey, _hasCycle bool) {
+ visited := sets.New[objKey]()
+ return m.dfsRoot(visited, seed, vertexFilter)
+}
+
+func (m abLinkMap) dfsRoot(visited sets.Set[objKey], key objKey, vertexFilter func(objKey) bool) (_root objKey, _hasCycle bool) {
+ if visited.Has(key) {
+ return key, true
+ }
+ visited.Insert(key) // avoid infinite recursion
+
+ for peer, link := range m[key] {
+ if !vertexFilter(peer) {
+ continue
+ }
+
+ if link.isParent(peer) {
+ return m.dfsRoot(visited, peer, vertexFilter)
+ }
+ }
+
+ return key, false // key has no parent, so key is root
+}
+
+type componentTaint = int
+
+type connectedComponent = []objKey
+
+func (*Merger[M]) findConnectedComponents(objects map[objKey]*object[M], abLinks abLinkMap) []connectedComponent {
+ objectKeys := make(sets.Set[objKey], len(objects))
+ for gk := range objects {
+ objectKeys.Insert(gk)
+ }
+
+ var taintCounter componentTaint
+
+ taints := map[objKey]componentTaint{}
+
+ for {
+ seed, hasMore := peekArbitraryFromSet(objectKeys)
+ if !hasMore {
+ break
+ }
+
+ dfsTaint(objectKeys, abLinks, taints, taintCounter, seed)
+ taintCounter += 1
+ }
+
+ components := make([]connectedComponent, taintCounter)
+ for key, taint := range taints {
+ components[taint] = append(components[taint], key)
+ }
+
+ return components
+}
+
+func peekArbitraryFromSet[T comparable](set sets.Set[T]) (T, bool) {
+ for value := range set {
+ return value, true
+ }
+
+ return reflectutil.ZeroOf[T](), false
+}
+
+func dfsTaint(
+ keys sets.Set[objKey],
+ abLinks abLinkMap,
+ taints map[objKey]componentTaint,
+ taintId componentTaint,
+ seed objKey,
+) {
+ taints[seed] = taintId
+ delete(keys, seed) // delete before diving in to avoid recursing backwards
+
+ for peer := range abLinks[seed] {
+ if _, remaining := keys[peer]; !remaining {
+ continue // this should be unreachable
+ }
+
+ dfsTaint(keys, abLinks, taints, taintId, peer)
+ }
+}
+
+type MergeTree[M any] struct {
+ Metadata []M
+
+ Tree *tftree.SpanTree
+}
+
+func newMergeTree[M any](
+ members []*object[M],
+ abLinks abLinkMap,
+) (*MergeTree[M], error) {
+ metadata := []M{}
+
+ for _, member := range members {
+ metadata = append(metadata, member.metadata...)
+ }
+
+ merged, err := mergeLinkedTraces(members, abLinks)
+ if err != nil {
+ return nil, err
+ }
+
+ return &MergeTree[M]{
+ Metadata: metadata,
+ Tree: merged,
+ }, nil
+}
+
+func mergeLinkedTraces[M any](objects []*object[M], abLinks abLinkMap) (*tftree.SpanTree, error) {
+ trees := make(map[objKey]*object[M], len(objects))
+ for _, obj := range objects {
+ trees[obj.key] = obj
+ }
+
+ rootKey, _ := abLinks.detectRoot(objects[0].key, func(key objKey) bool {
+ _, hasTree := trees[key]
+ return hasTree
+ })
+
+ tree := trees[rootKey].tree
+ treeObjects := sets.New(rootKey)
+
+ pendingObjects := []objKey{rootKey}
+ for len(pendingObjects) > 0 {
+ subj := pendingObjects[len(pendingObjects)-1]
+ pendingObjects = pendingObjects[:len(pendingObjects)-1]
+
+ for _, link := range trees[subj].links {
+ if link.Role != zconstants.LinkRoleChild {
+ continue
+ }
+
+ parentSpan := trees[subj].tree.Root
+ if link.Class != "" {
+ virtualSpan := createVirtualSpan(tree.Root.TraceID, parentSpan, "", link.Class)
+ tree.Add(virtualSpan, parentSpan.SpanID)
+ parentSpan = virtualSpan
+ }
+
+ if treeObjects.Has(link.Key) {
+ parentSpan.Warnings = append(parentSpan.Warnings, fmt.Sprintf("repeated object %v omitted", link.Key))
+ // duplicate
+ continue
+ }
+
+ subtree, hasSubtree := trees[link.Key]
+ if !hasSubtree {
+ // this link was not fetched, e.g. because of fetch limit or link selector
+ continue
+ }
+
+ tree.AddTree(subtree.tree, parentSpan.SpanID)
+ treeObjects.Insert(link.Key)
+ pendingObjects = append(pendingObjects, link.Key)
+ }
+ }
+
+ return tree, nil
+}
+
+func createVirtualSpan(traceId model.TraceID, span *model.Span, opName string, svcName string) *model.Span {
+ spanId := model.SpanID(rand.Uint64())
+
+ return &model.Span{
+ TraceID: traceId,
+ SpanID: spanId,
+ OperationName: opName,
+ Flags: 0,
+ StartTime: span.StartTime,
+ Duration: span.Duration,
+ Tags: []model.KeyValue{
+ {
+ Key: zconstants.PseudoType,
+ VType: model.StringType,
+ VStr: string(zconstants.PseudoTypeLinkClass),
+ },
+ },
+ Process: &model.Process{
+ ServiceName: svcName,
+ },
+ ProcessID: "1",
+ }
+}
diff --git a/pkg/frontend/reader/merge/merge_test.go b/pkg/frontend/reader/merge/merge_test.go
new file mode 100644
index 00000000..f66574b0
--- /dev/null
+++ b/pkg/frontend/reader/merge/merge_test.go
@@ -0,0 +1,318 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package merge_test
+
+import (
+ "context"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/jaegertracing/jaeger/model"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/kubewharf/kelemetry/pkg/frontend/reader/merge"
+ tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
+ tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
+)
+
+func newTrace(id uint64, key utilobject.Key, startTime int64, endTime int64, links []merge.TargetLink) merge.TraceWithMetadata[uint64] {
+ traceId := model.NewTraceID(id, id)
+ objectSpan := &model.Span{
+ TraceID: traceId,
+ SpanID: model.SpanID(id | (1 << 16)),
+ StartTime: time.Time{}.Add(time.Duration(startTime)),
+ Duration: time.Duration(endTime - startTime),
+ Tags: append(
+ mapToTags(zconstants.KeyToSpanTags(key)),
+ model.String(zconstants.TraceSource, zconstants.TraceSourceObject),
+ model.String(zconstants.PseudoType, string(zconstants.PseudoTypeObject)),
+ ),
+ }
+ spans := []*model.Span{objectSpan}
+
+ for i, link := range links {
+ tags := zconstants.KeyToSpanTags(key)
+ zconstants.TagLinkedObject(tags, zconstants.LinkRef{
+ Key: link.Key,
+ Role: link.Role,
+ Class: link.Class,
+ })
+
+ spans = append(spans, &model.Span{
+ TraceID: traceId,
+ SpanID: model.SpanID(id | (3 << 16) | (uint64(i) << 20)),
+ StartTime: objectSpan.StartTime,
+ Duration: objectSpan.Duration,
+ Tags: append(
+ mapToTags(tags),
+ model.String(zconstants.TraceSource, zconstants.TraceSourceObject),
+ model.String(zconstants.PseudoType, string(zconstants.PseudoTypeLink)),
+ ),
+ References: []model.SpanRef{model.NewChildOfRef(traceId, objectSpan.SpanID)},
+ })
+ }
+
+ spans = append(spans, &model.Span{
+ TraceID: traceId,
+ SpanID: model.SpanID(id | (2 << 16)),
+ StartTime: time.Time{}.Add(time.Duration(startTime+endTime) / 2),
+ Duration: time.Duration(endTime-startTime) / 4,
+ Tags: append(
+ mapToTags(zconstants.KeyToSpanTags(key)),
+ model.String(zconstants.TraceSource, zconstants.TraceSourceEvent),
+ model.String(zconstants.NotPseudo, zconstants.NotPseudo),
+ ),
+ References: []model.SpanRef{model.NewChildOfRef(traceId, objectSpan.SpanID)},
+ })
+
+ tree := tftree.NewSpanTree(spans)
+
+ return merge.TraceWithMetadata[uint64]{
+ Tree: tree,
+ Metadata: id,
+ }
+}
+
+func mapToTags(m map[string]string) (out []model.KeyValue) {
+ for key, value := range m {
+ out = append(out, model.String(key, value))
+ }
+
+ return out
+}
+
+type traceList []merge.TraceWithMetadata[uint64]
+
+func (list traceList) append(idLow uint32, key utilobject.Key, links []merge.TargetLink) traceList {
+ for time := 0; time < 4; time++ {
+ id := uint64(idLow) | (uint64(time) << 8)
+ list = append(list, newTrace(
+ id, key,
+ int64(time*10), int64((time+1)*10),
+ links,
+ ))
+ }
+ return list
+}
+
+// IDs:
+// - rs = 0x10, 0x11
+// - dp = 0x20
+// - pod = 0x30 | rs | (replica*2)
+// - node = 0x40
+func sampleTraces(withPods bool, withNode bool) traceList {
+ rsKeys := []utilobject.Key{
+ {
+ Cluster: "test",
+ Group: "apps",
+ Resource: "replicasets",
+ Namespace: "default",
+ Name: "dp-spec1",
+ },
+ {
+ Cluster: "test",
+ Group: "apps",
+ Resource: "replicasets",
+ Namespace: "default",
+ Name: "dp-spec2",
+ },
+ }
+ dpKey := utilobject.Key{
+ Cluster: "test",
+ Group: "apps",
+ Resource: "deployments",
+ Namespace: "default",
+ Name: "dp",
+ }
+ podKeys := [][]utilobject.Key{
+ {
+ {
+ Cluster: "test",
+ Group: "",
+ Resource: "pods",
+ Namespace: "default",
+ Name: "dp-spec1-replica1",
+ },
+ {
+ Cluster: "test",
+ Group: "",
+ Resource: "pods",
+ Namespace: "default",
+ Name: "dp-spec1-replica2",
+ },
+ },
+ {
+ {
+ Cluster: "test",
+ Group: "",
+ Resource: "pods",
+ Namespace: "default",
+ Name: "dp-spec2-replica1",
+ },
+ {
+ Cluster: "test",
+ Group: "",
+ Resource: "pods",
+ Namespace: "default",
+ Name: "dp-spec2-replica2",
+ },
+ },
+ }
+ nodeKey := utilobject.Key{
+ Cluster: "test",
+ Group: "",
+ Resource: "nodes",
+ Namespace: "",
+ Name: "node",
+ }
+
+ list := traceList{}
+ for spec := uint32(0); spec < 2; spec++ {
+ rsLinks := []merge.TargetLink{
+ {Key: dpKey, Role: zconstants.LinkRoleParent, Class: "children"},
+ }
+ if withPods {
+ rsLinks = append(rsLinks,
+ merge.TargetLink{Key: podKeys[spec][0], Role: zconstants.LinkRoleChild, Class: "children"},
+ merge.TargetLink{Key: podKeys[spec][1], Role: zconstants.LinkRoleChild, Class: "children"},
+ )
+ }
+ list = list.append(0x10|spec, rsKeys[spec], rsLinks)
+ }
+ list = list.append(0x20, dpKey, []merge.TargetLink{
+ {Key: rsKeys[0], Role: zconstants.LinkRoleChild, Class: "children"},
+ {Key: rsKeys[1], Role: zconstants.LinkRoleChild, Class: "children"},
+ })
+
+ nodeLinks := []merge.TargetLink{}
+ if withPods {
+ for spec := uint32(0); spec < 2; spec++ {
+ for replica := uint32(0); replica < 2; replica++ {
+ podLinks := []merge.TargetLink{
+ {Key: rsKeys[spec], Role: zconstants.LinkRoleParent, Class: "children"},
+ }
+ if withNode {
+ podLinks = append(podLinks, merge.TargetLink{Key: nodeKey, Role: zconstants.LinkRoleChild, Class: "node"})
+ }
+ list = list.append(0x30|spec|(replica<<1), podKeys[spec][replica], podLinks)
+ nodeLinks = append(nodeLinks, merge.TargetLink{Key: podKeys[spec][replica], Role: zconstants.LinkRoleParent, Class: "node"})
+ }
+ }
+ }
+ if withNode {
+ list = list.append(0x40, nodeKey, nodeLinks)
+ }
+ return list
+}
+
+func do(
+ t *testing.T,
+ clipTimeStart, clipTimeEnd int64,
+ traces traceList,
+ activePrefixLength int,
+ linkSelector tfconfig.LinkSelector,
+ expectGroupSizes []int,
+ expectObjectCounts []int,
+) {
+ t.Helper()
+
+ assert := assert.New(t)
+
+ active := []merge.TraceWithMetadata[uint64]{}
+ for _, trace := range traces[:activePrefixLength] {
+ traceTime := int64(trace.Tree.Root.StartTime.Sub(time.Time{}))
+ if clipTimeStart <= traceTime && traceTime < clipTimeEnd {
+ active = append(active, trace)
+ }
+ }
+
+ merger := merge.Merger[uint64]{}
+ _, err := merger.AddTraces(active)
+ assert.NoError(err)
+
+ assert.NoError(merger.FollowLinks(
+ context.Background(),
+ linkSelector,
+ time.Time{}.Add(time.Duration(clipTimeStart)),
+ time.Time{}.Add(time.Duration(clipTimeEnd)),
+ func(
+ ctx context.Context,
+ key utilobject.Key,
+ startTime, endTime time.Time,
+ limit int,
+ ) (out []merge.TraceWithMetadata[uint64], _ error) {
+ for _, trace := range traces {
+ traceKey := zconstants.ObjectKeyFromSpan(trace.Tree.Root)
+ traceTime := int64(trace.Tree.Root.StartTime.Sub(time.Time{}))
+ if key == traceKey && clipTimeStart <= traceTime && traceTime < clipTimeEnd {
+ out = append(out, trace)
+ }
+ }
+
+ return out, nil
+ },
+ len(traces),
+ int32(len(traces)),
+ false,
+ ))
+
+ result, err := merger.MergeTraces()
+ assert.NoError(err)
+ assert.Len(result, len(expectGroupSizes))
+
+ sort.Ints(expectGroupSizes)
+ actualGroupSizes := make([]int, len(result))
+ for i, group := range result {
+ actualGroupSizes[i] = len(group.Metadata)
+ }
+ sort.Ints(actualGroupSizes)
+ assert.Equal(expectGroupSizes, actualGroupSizes)
+
+ actualObjectCounts := []int{}
+ for _, group := range result {
+ objectCount := 0
+ for _, span := range group.Tree.GetSpans() {
+ pseudoTag, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType)
+ if isPseudo && pseudoTag.VStr == string(zconstants.PseudoTypeObject) {
+ objectCount += 1
+ }
+ }
+
+ actualObjectCounts = append(actualObjectCounts, objectCount)
+ }
+ sort.Ints(actualObjectCounts)
+ assert.Equal(expectObjectCounts, actualObjectCounts)
+}
+
+func TestFullTree(t *testing.T) {
+ do(t, 10, 30, sampleTraces(true, true), 4, tfconfig.ConstantLinkSelector(true), []int{2 * (1 + 2 + 4 + 1)}, []int{1 + 2 + 4 + 1})
+}
+
+func TestFilteredTree(t *testing.T) {
+ do(t, 10, 30, sampleTraces(true, true), 4, rsPodLinksOnly{}, []int{2 * (1 + 2)}, []int{1 + 2})
+}
+
+type rsPodLinksOnly struct{}
+
+func (rsPodLinksOnly) Admit(parent, child utilobject.Key, parentIsSource bool, class string) tfconfig.LinkSelector {
+ if parent.Resource == "replicasets" && child.Resource == "pods" {
+ return rsPodLinksOnly{}
+ } else {
+ return nil
+ }
+}
diff --git a/pkg/frontend/reader/reader.go b/pkg/frontend/reader/reader.go
index 08b86ebf..2f611e40 100644
--- a/pkg/frontend/reader/reader.go
+++ b/pkg/frontend/reader/reader.go
@@ -29,11 +29,14 @@ import (
jaegerbackend "github.com/kubewharf/kelemetry/pkg/frontend/backend"
"github.com/kubewharf/kelemetry/pkg/frontend/clusterlist"
+ "github.com/kubewharf/kelemetry/pkg/frontend/reader/merge"
transform "github.com/kubewharf/kelemetry/pkg/frontend/tf"
tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
+ tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/frontend/tracecache"
"github.com/kubewharf/kelemetry/pkg/manager"
utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ reflectutil "github.com/kubewharf/kelemetry/pkg/util/reflect"
"github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
@@ -46,7 +49,10 @@ type Interface interface {
}
type options struct {
- cacheExtensions bool
+ cacheExtensions bool
+ followLinkConcurrency int
+ followLinkLimit int32
+ followLinksInList bool
}
func (options *options) Setup(fs *pflag.FlagSet) {
@@ -56,6 +62,24 @@ func (options *options) Setup(fs *pflag.FlagSet) {
false,
"cache extension trace search result, otherwise trace is searched again every time result is reloaded",
)
+ fs.IntVar(
+ &options.followLinkConcurrency,
+ "frontend-follow-link-concurrency",
+ 20,
+ "number of concurrent trace per request to follow links",
+ )
+ fs.Int32Var(
+ &options.followLinkLimit,
+ "frontend-follow-link-limit",
+ 10,
+ "maximum number of linked objects to follow per search result",
+ )
+ fs.BoolVar(
+ &options.followLinksInList,
+ "frontend-follow-links-in-list",
+ true,
+ "whether links should be recursed into when listing traces",
+ )
}
func (options *options) EnableFlag() *bool { return nil }
@@ -127,78 +151,73 @@ func (reader *spanReader) FindTraces(ctx context.Context, query *spanstore.Trace
}
reader.Logger.WithField("query", query).
- WithField("exclusive", config.UseSubtree).
WithField("config", config.Name).
Debug("start trace list")
- thumbnails, err := reader.Backend.List(ctx, query, config.UseSubtree)
+
+ if len(query.OperationName) > 0 {
+ if query.Tags == nil {
+ query.Tags = map[string]string{}
+ }
+ query.Tags["cluster"] = query.OperationName
+ }
+
+ tts, err := reader.Backend.List(ctx, query)
if err != nil {
return nil, err
}
+ twmList := make([]merge.TraceWithMetadata[any], len(tts))
+ for i, tt := range tts {
+ twmList[i] = merge.TraceWithMetadata[any]{
+ Tree: tt.Spans,
+ Metadata: tt.Identifier,
+ }
+ }
+
+ merger := merge.Merger[any]{}
+ if _, err := merger.AddTraces(twmList); err != nil {
+ return nil, fmt.Errorf("group traces by object: %w", err)
+ }
+
+ if reader.options.followLinksInList {
+ if err := merger.FollowLinks(
+ ctx,
+ config.LinkSelector,
+ query.StartTimeMin, query.StartTimeMax,
+ mergeListWithBackend[any](reader.Backend, reflectutil.Identity[any], OriginalTraceRequest{FindTraces: query}),
+ reader.options.followLinkConcurrency, reader.options.followLinkLimit, false,
+ ); err != nil {
+ return nil, fmt.Errorf("follow links: %w", err)
+ }
+ }
+
+ mergeTrees, err := merger.MergeTraces()
+ if err != nil {
+ return nil, fmt.Errorf("merging split and linked traces: %w", err)
+ }
+
var rootKey *utilobject.Key
if rootKeyValue, ok := utilobject.FromMap(query.Tags); ok {
rootKey = &rootKeyValue
}
- mergedEntries := mergeSegments(thumbnails)
-
cacheEntries := []tracecache.Entry{}
traces := []*model.Trace{}
- for _, entry := range mergedEntries {
+ for _, mergeTree := range mergeTrees {
cacheId := generateCacheId(config.Id)
- for _, span := range entry.spans {
- span.TraceID = cacheId
- for i := range span.References {
- span.References[i].TraceID = cacheId
- }
- }
-
- entry.spans = filterTimeRange(entry.spans, query.StartTimeMin, query.StartTimeMax)
-
- trace := &model.Trace{
- ProcessMap: []model.Trace_ProcessMapping{{
- ProcessID: "0",
- Process: model.Process{},
- }},
- Spans: entry.spans,
+ trace, extensionCache, err := reader.prepareEntry(ctx, rootKey, query, mergeTree.Tree, cacheId)
+ if err != nil {
+ return nil, err
}
- displayMode := extractDisplayMode(cacheId)
-
- extensions := &transform.FetchExtensionsAndStoreCache{}
-
- if err := reader.Transformer.Transform(
- ctx, trace, rootKey, displayMode,
- extensions,
- query.StartTimeMin, query.StartTimeMax,
- ); err != nil {
- return nil, fmt.Errorf("trace transformation failed: %w", err)
- }
traces = append(traces, trace)
- identifiers := make([]json.RawMessage, len(entry.identifiers))
- for i, identifier := range entry.identifiers {
- idJson, err := json.Marshal(identifier)
- if err != nil {
- return nil, fmt.Errorf("thumbnail identifier marshal: %w", err)
- }
-
- identifiers[i] = json.RawMessage(idJson)
+ cacheEntry, err := reader.prepareCache(rootKey, query, mergeTree.Metadata, cacheId, extensionCache)
+ if err != nil {
+ return nil, err
}
- cacheEntry := tracecache.Entry{
- LowId: cacheId.Low,
- Value: tracecache.EntryValue{
- Identifiers: identifiers,
- StartTime: query.StartTimeMin,
- EndTime: query.StartTimeMax,
- RootObject: rootKey,
- },
- }
- if reader.options.cacheExtensions {
- cacheEntry.Value.Extensions = extensions.Cache
- }
cacheEntries = append(cacheEntries, cacheEntry)
}
@@ -213,6 +232,80 @@ func (reader *spanReader) FindTraces(ctx context.Context, query *spanstore.Trace
return traces, nil
}
+func (reader *spanReader) prepareEntry(
+ ctx context.Context,
+ rootKey *utilobject.Key,
+ query *spanstore.TraceQueryParameters,
+ tree *tftree.SpanTree,
+ cacheId model.TraceID,
+) (*model.Trace, []tracecache.ExtensionCache, error) {
+ spans := tree.GetSpans()
+
+ for _, span := range spans {
+ span.TraceID = cacheId
+ for i := range span.References {
+ span.References[i].TraceID = cacheId
+ }
+ }
+
+ spans = filterTimeRange(spans, query.StartTimeMin, query.StartTimeMax)
+
+ trace := &model.Trace{
+ ProcessMap: []model.Trace_ProcessMapping{{
+ ProcessID: "0",
+ Process: model.Process{},
+ }},
+ Spans: spans,
+ }
+
+ displayMode := extractDisplayMode(cacheId)
+
+ extensions := &transform.FetchExtensionsAndStoreCache{}
+
+ if err := reader.Transformer.Transform(
+ ctx, trace, rootKey, displayMode,
+ extensions,
+ query.StartTimeMin, query.StartTimeMax,
+ ); err != nil {
+ return nil, nil, fmt.Errorf("trace transformation failed: %w", err)
+ }
+
+ return trace, extensions.Cache, nil
+}
+
+func (reader *spanReader) prepareCache(
+ rootKey *utilobject.Key,
+ query *spanstore.TraceQueryParameters,
+ identifiers []any,
+ cacheId model.TraceID,
+ extensionCache []tracecache.ExtensionCache,
+) (tracecache.Entry, error) {
+ identifiersJson := make([]json.RawMessage, len(identifiers))
+ for i, identifier := range identifiers {
+ idJson, err := json.Marshal(identifier)
+ if err != nil {
+ return tracecache.Entry{}, fmt.Errorf("thumbnail identifier marshal: %w", err)
+ }
+
+ identifiersJson[i] = json.RawMessage(idJson)
+ }
+
+ cacheEntry := tracecache.Entry{
+ LowId: cacheId.Low,
+ Value: tracecache.EntryValue{
+ Identifiers: identifiersJson,
+ StartTime: query.StartTimeMin,
+ EndTime: query.StartTimeMax,
+ RootObject: rootKey,
+ },
+ }
+ if reader.options.cacheExtensions {
+ cacheEntry.Value.Extensions = extensionCache
+ }
+
+ return cacheEntry, nil
+}
+
func (reader *spanReader) GetTrace(ctx context.Context, cacheId model.TraceID) (*model.Trace, error) {
entry, err := reader.TraceCache.Fetch(ctx, cacheId.Low)
if err != nil {
@@ -222,13 +315,9 @@ func (reader *spanReader) GetTrace(ctx context.Context, cacheId model.TraceID) (
return nil, fmt.Errorf("trace %v not found", cacheId)
}
- aggTrace := &model.Trace{
- ProcessMap: []model.Trace_ProcessMapping{{
- ProcessID: "0",
- Process: model.Process{},
- }},
- }
+ displayMode := extractDisplayMode(cacheId)
+ traces := make([]merge.TraceWithMetadata[struct{}], 0, len(entry.Identifiers))
for _, identifier := range entry.Identifiers {
trace, err := reader.Backend.Get(ctx, identifier, cacheId, entry.StartTime, entry.EndTime)
if err != nil {
@@ -236,7 +325,43 @@ func (reader *spanReader) GetTrace(ctx context.Context, cacheId model.TraceID) (
}
clipped := filterTimeRange(trace.Spans, entry.StartTime, entry.EndTime)
- aggTrace.Spans = append(aggTrace.Spans, clipped...)
+ traces = append(traces, merge.TraceWithMetadata[struct{}]{Tree: tftree.NewSpanTree(clipped)})
+ }
+
+ merger := merge.Merger[struct{}]{}
+ if _, err := merger.AddTraces(traces); err != nil {
+ return nil, fmt.Errorf("grouping traces by object: %w", err)
+ }
+
+ displayConfig := reader.TransformConfigs.GetById(displayMode)
+ if displayConfig == nil {
+ return nil, fmt.Errorf("display mode %x does not exist", displayMode)
+ }
+
+ if err := merger.FollowLinks(
+ ctx,
+ displayConfig.LinkSelector,
+ entry.StartTime, entry.EndTime,
+ mergeListWithBackend[struct{}](reader.Backend, func(any) struct{} { return struct{}{} }, OriginalTraceRequest{GetTrace: &cacheId}),
+ reader.options.followLinkConcurrency, reader.options.followLinkLimit, true,
+ ); err != nil {
+ return nil, fmt.Errorf("cannot follow links: %w", err)
+ }
+ mergedTrees, err := merger.MergeTraces()
+ if err != nil {
+ return nil, fmt.Errorf("merging linked trees: %w", err)
+ }
+
+ // if spans were connected, they should continue to be connected since link spans cannot be deleted, so assume there is only one trace
+ if len(mergedTrees) != 1 {
+ return nil, fmt.Errorf("inconsistent linked trace count %d", len(mergedTrees))
+ }
+ mergedTree := mergedTrees[0]
+ aggTrace := &model.Trace{
+ ProcessMap: []model.Trace_ProcessMapping{{
+ ProcessID: "0",
+ }},
+ Spans: mergedTree.Tree.GetSpans(),
}
var extensions transform.ExtensionProcessor = &transform.FetchExtensionsAndStoreCache{}
@@ -244,7 +369,6 @@ func (reader *spanReader) GetTrace(ctx context.Context, cacheId model.TraceID) (
extensions = &transform.LoadExtensionCache{Cache: entry.Extensions}
}
- displayMode := extractDisplayMode(cacheId)
if err := reader.Transformer.Transform(
ctx, aggTrace, entry.RootObject, displayMode,
extensions,
@@ -299,3 +423,45 @@ func filterTimeRange(spans []*model.Span, startTime, endTime time.Time) []*model
return retained
}
+
+type (
+ OriginalTraceRequestKey struct{}
+ OriginalTraceRequest struct {
+ GetTrace *model.TraceID
+ FindTraces *spanstore.TraceQueryParameters
+ }
+)
+
+func mergeListWithBackend[M any](backend jaegerbackend.Backend, convertMetadata func(any) M, otr OriginalTraceRequest) merge.ListFunc[M] {
+ return func(
+ ctx context.Context,
+ key utilobject.Key,
+ startTime time.Time, endTime time.Time,
+ limit int,
+ ) ([]merge.TraceWithMetadata[M], error) {
+ tags := zconstants.KeyToSpanTags(key)
+
+ tts, err := backend.List(
+ context.WithValue(ctx, OriginalTraceRequestKey{}, otr),
+ &spanstore.TraceQueryParameters{
+ Tags: tags,
+ StartTimeMin: startTime,
+ StartTimeMax: endTime,
+ NumTraces: limit,
+ },
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ twmList := make([]merge.TraceWithMetadata[M], len(tts))
+ for i, tt := range tts {
+ twmList[i] = merge.TraceWithMetadata[M]{
+ Tree: tt.Spans,
+ Metadata: convertMetadata(tt.Identifier),
+ }
+ }
+
+ return twmList, nil
+ }
+}
diff --git a/pkg/frontend/tf/config/config.go b/pkg/frontend/tf/config/config.go
index b5bd4849..57524228 100644
--- a/pkg/frontend/tf/config/config.go
+++ b/pkg/frontend/tf/config/config.go
@@ -15,7 +15,12 @@
package tfconfig
import (
+ "fmt"
+ "sort"
"strconv"
+ "strings"
+
+ "k8s.io/apimachinery/pkg/util/sets"
"github.com/kubewharf/kelemetry/pkg/frontend/extension"
"github.com/kubewharf/kelemetry/pkg/manager"
@@ -51,24 +56,43 @@ type Config struct {
Id Id
// The config name, used in search page display.
Name string
- // If true, only displays the spans below the matched span.
- // If false, displays the whole trace including parent and sibling spans.
- UseSubtree bool
+ // Base config name without modifiers, used to help reconstruct the name.
+ BaseName string
+ // Names of modifiers, used to help reconstruct the name.
+ ModifierNames sets.Set[string]
+ // Only links with roles in this set are followed.
+ LinkSelector LinkSelector
// The extension traces for this config.
Extensions []extension.Provider
// The steps to transform the tree
Steps []Step
}
+func (config *Config) RecomputeName() {
+ modifiers := config.ModifierNames.UnsortedList()
+ sort.Strings(modifiers)
+ if len(modifiers) > 0 {
+ config.Name = fmt.Sprintf("%s [%s]", config.BaseName, strings.Join(modifiers, "+"))
+ } else {
+ config.Name = config.BaseName
+ }
+}
+
func (config *Config) Clone() *Config {
steps := make([]Step, len(config.Steps))
copy(steps, config.Steps) // no need to deep clone each step
+ extensions := make([]extension.Provider, len(config.Extensions))
+ copy(extensions, config.Extensions)
+
return &Config{
- Id: config.Id,
- Name: config.Name,
- UseSubtree: config.UseSubtree,
- Steps: steps,
+ Id: config.Id,
+ Name: config.Name,
+ BaseName: config.BaseName,
+ ModifierNames: config.ModifierNames.Clone(),
+ LinkSelector: config.LinkSelector, // modifier changes LinkSelector by wrapping the previous value
+ Extensions: extensions,
+ Steps: steps,
}
}
diff --git a/pkg/frontend/tf/config/file/file.go b/pkg/frontend/tf/config/file/file.go
index af77b9c9..c444cc7e 100644
--- a/pkg/frontend/tf/config/file/file.go
+++ b/pkg/frontend/tf/config/file/file.go
@@ -115,10 +115,9 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
Modifiers map[tfconfig.Id]modifierConfig `json:"modifiers"`
Batches []Batch `json:"batches"`
Configs []struct {
- Id tfconfig.Id `json:"id"`
- Name string `json:"name"`
- UseSubtree bool `json:"useSubtree"`
- Steps json.RawMessage `json:"steps"`
+ Id tfconfig.Id `json:"id"`
+ Name string `json:"name"`
+ Steps json.RawMessage `json:"steps"`
} `json:"configs"`
DefaultConfig tfconfig.Id `json:"defaultConfig"`
}
@@ -157,7 +156,8 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
priority: modifierConfig.Priority,
fn: func(config *tfconfig.Config) {
config.Id |= bitmask
- config.Name += fmt.Sprintf(" [%s]", displayName)
+ config.ModifierNames.Insert(displayName)
+ config.RecomputeName()
modifier.Modify(config)
},
})
@@ -183,10 +183,12 @@ func (p *FileProvider) loadJsonBytes(jsonBytes []byte) error {
}
config := &tfconfig.Config{
- Id: raw.Id,
- Name: raw.Name,
- UseSubtree: raw.UseSubtree,
- Steps: steps,
+ Id: raw.Id,
+ Name: raw.Name,
+ BaseName: raw.Name,
+ ModifierNames: sets.New[string](),
+ LinkSelector: tfconfig.ConstantLinkSelector(false),
+ Steps: steps,
}
p.register(registeredConfig{config: config, modifierClasses: sets.New[string]()})
diff --git a/pkg/frontend/tf/config/link_selector.go b/pkg/frontend/tf/config/link_selector.go
new file mode 100644
index 00000000..d61b0277
--- /dev/null
+++ b/pkg/frontend/tf/config/link_selector.go
@@ -0,0 +1,87 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tfconfig
+
+import utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+
+type LinkSelector interface {
+ // Whether to follow the given link.
+ //
+ // If link should be followed, return a non-nil LinkSelector.
+ // The returned object will be used to recursively follow links in the linked object.
+ Admit(parent utilobject.Key, child utilobject.Key, parentIsSource bool, linkClass string) LinkSelector
+}
+
+type ConstantLinkSelector bool
+
+func (selector ConstantLinkSelector) Admit(
+ parent utilobject.Key,
+ child utilobject.Key,
+ parentIsSource bool,
+ linkClass string,
+) LinkSelector {
+ if selector {
+ return selector
+ }
+
+ return nil
+}
+
+type IntersectLinkSelector []LinkSelector
+
+func (selector IntersectLinkSelector) Admit(
+ parentKey utilobject.Key,
+ childKey utilobject.Key,
+ parentIsSource bool,
+ linkClass string,
+) LinkSelector {
+ newChildren := make([]LinkSelector, len(selector))
+
+ for i, child := range selector {
+ newChildren[i] = child.Admit(parentKey, childKey, parentIsSource, linkClass)
+ if newChildren[i] == nil {
+ return nil
+ }
+ }
+
+ return IntersectLinkSelector(newChildren)
+}
+
+type UnionLinkSelector []LinkSelector
+
+func (selector UnionLinkSelector) Admit(
+ parentKey utilobject.Key,
+ childKey utilobject.Key,
+ parentIsSource bool,
+ linkClass string,
+) LinkSelector {
+ newChildren := make([]LinkSelector, len(selector))
+
+ ok := false
+ for i, child := range selector {
+ if child != nil {
+ newChildren[i] = child.Admit(parentKey, childKey, parentIsSource, linkClass)
+ if newChildren[i] != nil {
+ ok = true
+ }
+ }
+ }
+
+ if ok {
+ return UnionLinkSelector(newChildren)
+ }
+
+ return nil
+}
diff --git a/pkg/frontend/tf/defaults/modifier/exclusive.go b/pkg/frontend/tf/defaults/modifier/exclusive.go
deleted file mode 100644
index 7b141675..00000000
--- a/pkg/frontend/tf/defaults/modifier/exclusive.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2023 The Kelemetry Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tfmodifier
-
-import (
- "context"
-
- "github.com/spf13/pflag"
-
- tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
- "github.com/kubewharf/kelemetry/pkg/manager"
-)
-
-func init() {
- manager.Global.ProvideListImpl(
- "tf-modifier/exclusive",
- manager.Ptr(&ExclusiveModifierFactory{}),
- &manager.List[tfconfig.ModifierFactory]{},
- )
-}
-
-type ExclusiveModifierOptions struct {
- enable bool
-}
-
-func (options *ExclusiveModifierOptions) Setup(fs *pflag.FlagSet) {
- fs.BoolVar(&options.enable, "jaeger-tf-exclusive-modifier-enable", true, "enable exclusive modifier and list it in frontend")
-}
-
-func (options *ExclusiveModifierOptions) EnableFlag() *bool { return &options.enable }
-
-type ExclusiveModifierFactory struct {
- options ExclusiveModifierOptions
-}
-
-var _ manager.Component = &ExclusiveModifierFactory{}
-
-func (m *ExclusiveModifierFactory) Options() manager.Options { return &m.options }
-func (m *ExclusiveModifierFactory) Init() error { return nil }
-func (m *ExclusiveModifierFactory) Start(ctx context.Context) error { return nil }
-func (m *ExclusiveModifierFactory) Close(ctx context.Context) error { return nil }
-
-func (*ExclusiveModifierFactory) ListIndex() string { return "exclusive" }
-
-func (*ExclusiveModifierFactory) Build(jsonBuf []byte) (tfconfig.Modifier, error) {
- return &ExclusiveModifier{}, nil
-}
-
-type ExclusiveModifier struct{}
-
-func (*ExclusiveModifier) ModifierClass() string { return "kelemetry.kubewharf.io/exclusive" }
-
-func (*ExclusiveModifier) Modify(config *tfconfig.Config) {
- config.UseSubtree = true
-}
diff --git a/pkg/frontend/tf/defaults/modifier/link_selector.go b/pkg/frontend/tf/defaults/modifier/link_selector.go
new file mode 100644
index 00000000..5d49f590
--- /dev/null
+++ b/pkg/frontend/tf/defaults/modifier/link_selector.go
@@ -0,0 +1,200 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tfmodifier
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "github.com/spf13/pflag"
+
+ tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
+ "github.com/kubewharf/kelemetry/pkg/manager"
+ utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal"
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+func init() {
+ manager.Global.ProvideListImpl(
+ "tf-modifier/link-selector",
+ manager.Ptr(&LinkSelectorModifierFactory{}),
+ &manager.List[tfconfig.ModifierFactory]{},
+ )
+}
+
+type LinkSelectorModifierOptions struct {
+ enable bool
+}
+
+func (options *LinkSelectorModifierOptions) Setup(fs *pflag.FlagSet) {
+ fs.BoolVar(&options.enable, "jaeger-tf-link-selector-modifier-enable", true, "enable link selector modifiers and list it in frontend")
+}
+
+func (options *LinkSelectorModifierOptions) EnableFlag() *bool { return &options.enable }
+
+type LinkSelectorModifierFactory struct {
+ options LinkSelectorModifierOptions
+}
+
+var _ manager.Component = &LinkSelectorModifierFactory{}
+
+func (m *LinkSelectorModifierFactory) Options() manager.Options { return &m.options }
+func (m *LinkSelectorModifierFactory) Init() error { return nil }
+func (m *LinkSelectorModifierFactory) Start(ctx context.Context) error { return nil }
+func (m *LinkSelectorModifierFactory) Close(ctx context.Context) error { return nil }
+
+func (*LinkSelectorModifierFactory) ListIndex() string { return "link-selector" }
+
+func (*LinkSelectorModifierFactory) Build(jsonBuf []byte) (tfconfig.Modifier, error) {
+ modifier := &LinkSelectorModifier{}
+
+ if err := json.Unmarshal(jsonBuf, &modifier); err != nil {
+ return nil, fmt.Errorf("parse link selector modifier config: %w", err)
+ }
+
+ return modifier, nil
+}
+
+type LinkSelectorModifier struct {
+ Class string `json:"modifierClass"`
+ IncludeSiblings bool `json:"includeSiblings"`
+ PatternFilters []LinkPattern `json:"ifAll"`
+ UpwardDistance utilmarshal.Optional[uint32] `json:"upwardDistance"`
+ DownwardDistance utilmarshal.Optional[uint32] `json:"downwardDistance"`
+}
+
+type LinkPattern struct {
+ Parent utilmarshal.ObjectFilter `json:"parent"`
+ Child utilmarshal.ObjectFilter `json:"child"`
+ IncludeFromParent utilmarshal.Optional[bool] `json:"fromParent"`
+ IncludeFromChild utilmarshal.Optional[bool] `json:"fromChild"`
+ LinkClass utilmarshal.Optional[utilmarshal.StringFilter] `json:"linkClass"`
+}
+
+func (pattern *LinkPattern) Matches(parent utilobject.Key, child utilobject.Key, isFromParent bool, linkClass string) bool {
+ if !pattern.Parent.Matches(parent) {
+ return false
+ }
+
+ if !pattern.Child.Matches(child) {
+ return false
+ }
+
+ if !pattern.IncludeFromParent.GetOr(true) && isFromParent {
+ return false
+ }
+
+ if !pattern.IncludeFromChild.GetOr(true) && !isFromParent {
+ return false
+ }
+
+ if pattern.LinkClass.IsSet && !pattern.LinkClass.Value.Matches(linkClass) {
+ return false
+ }
+
+ return true
+}
+
+func (modifier *LinkSelectorModifier) ModifierClass() string {
+ return fmt.Sprintf("kelemetry.kubewharf.io/link-selectors/%s", modifier.Class)
+}
+
+func (modifier *LinkSelectorModifier) Modify(config *tfconfig.Config) {
+ intersectSelector := tfconfig.IntersectLinkSelector{
+ patternLinkSelector{patterns: modifier.PatternFilters},
+ }
+ if !modifier.IncludeSiblings {
+ intersectSelector = append(intersectSelector, denySiblingsLinkSelector{})
+ }
+ if modifier.UpwardDistance.IsSet {
+ intersectSelector = append(
+ intersectSelector,
+ directedDistanceLinkSelector{distance: modifier.UpwardDistance.Value, direction: directionUpwards},
+ )
+ }
+ if modifier.DownwardDistance.IsSet {
+ intersectSelector = append(
+ intersectSelector,
+ directedDistanceLinkSelector{distance: modifier.DownwardDistance.Value, direction: directionDownwards},
+ )
+ }
+
+ config.LinkSelector = tfconfig.UnionLinkSelector{config.LinkSelector, intersectSelector}
+}
+
+type denySiblingsLinkSelector struct {
+ hasFirst bool
+ firstIsFromParent bool
+}
+
+func (s denySiblingsLinkSelector) Admit(
+ parent utilobject.Key,
+ child utilobject.Key,
+ isFromParent bool,
+ linkClass string,
+) tfconfig.LinkSelector {
+ if !s.hasFirst {
+ return denySiblingsLinkSelector{hasFirst: true, firstIsFromParent: isFromParent}
+ }
+ if s.firstIsFromParent != isFromParent {
+ return nil
+ }
+ return s
+}
+
+type patternLinkSelector struct {
+ patterns []LinkPattern
+}
+
+func (s patternLinkSelector) Admit(parent utilobject.Key, child utilobject.Key, isFromParent bool, linkClass string) tfconfig.LinkSelector {
+ for _, pattern := range s.patterns {
+ if !pattern.Matches(parent, child, isFromParent, linkClass) {
+ return nil
+ }
+ }
+
+ return s
+}
+
+type direction bool
+
+const (
+ directionUpwards direction = true
+ directionDownwards direction = false
+)
+
+type directedDistanceLinkSelector struct {
+ direction direction
+ distance uint32
+}
+
+func (d directedDistanceLinkSelector) Admit(
+ parent utilobject.Key,
+ child utilobject.Key,
+ isFromParent bool,
+ linkClass string,
+) tfconfig.LinkSelector {
+ if isFromParent != (d.direction == directionDownwards) {
+ return d
+ }
+ if d.distance == 0 {
+ return nil
+ }
+ return directedDistanceLinkSelector{
+ direction: d.direction,
+ distance: d.distance - 1,
+ }
+}
diff --git a/pkg/frontend/tf/defaults/step/collapse_nesting.go b/pkg/frontend/tf/defaults/step/collapse_nesting.go
index 9d6bfb46..e1725793 100644
--- a/pkg/frontend/tf/defaults/step/collapse_nesting.go
+++ b/pkg/frontend/tf/defaults/step/collapse_nesting.go
@@ -24,6 +24,7 @@ import (
tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/manager"
+ utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal"
"github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
@@ -35,13 +36,13 @@ func init() {
)
}
-// Deletes child spans with a traceSource and injects them as logs in the nesting span.
+// Deletes child spans with a non-pseudo trace source and injects them as logs in the nesting span.
//
// Multiple logs of the same span are aggregated into one log, flattening them into a field.
//
// Must be followed by PruneTagsVisitor in the last step.
type CollapseNestingVisitor struct {
- ShouldCollapse StringFilter `json:"shouldCollapse"` // tests traceSource
+ ShouldCollapse utilmarshal.StringFilter `json:"shouldCollapse"` // tests traceSource
TagMappings map[string][]TagMapping `json:"tagMappings"` // key = traceSource
AuditDiffClasses AuditDiffClassification `json:"auditDiffClasses"` // key = prefix
LogTypeMapping map[zconstants.LogType]string `json:"logTypeMapping"` // key = log type, value = log field
@@ -104,7 +105,7 @@ func (classes *AuditDiffClassification) Get(prefix string) *AuditDiffClass {
}
func (visitor CollapseNestingVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor {
- if _, hasTag := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel); !hasTag {
+ if _, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); !isPseudo {
return visitor
}
@@ -119,15 +120,16 @@ func (visitor CollapseNestingVisitor) Exit(tree *tftree.SpanTree, span *model.Sp
func (visitor CollapseNestingVisitor) processChild(tree *tftree.SpanTree, span *model.Span, childId model.SpanID) {
childSpan := tree.Span(childId)
- if _, childHasTag := model.KeyValues(childSpan.Tags).FindByKey(zconstants.NestLevel); childHasTag {
+ if _, childIsPseudo := model.KeyValues(childSpan.Tags).FindByKey(zconstants.PseudoType); childIsPseudo {
return
}
+
traceSourceKv, hasTraceSource := model.KeyValues(childSpan.Tags).FindByKey(zconstants.TraceSource)
if !hasTraceSource {
return
}
traceSource := traceSourceKv.VStr
- if !visitor.ShouldCollapse.Test(traceSource) {
+ if !visitor.ShouldCollapse.Matches(traceSource) {
return
}
diff --git a/pkg/frontend/tf/defaults/step/compact_duration.go b/pkg/frontend/tf/defaults/step/compact_duration.go
index 129cd7ed..4aa6b800 100644
--- a/pkg/frontend/tf/defaults/step/compact_duration.go
+++ b/pkg/frontend/tf/defaults/step/compact_duration.go
@@ -45,7 +45,7 @@ func (visitor CompactDurationVisitor) Enter(tree *tftree.SpanTree, span *model.S
func (visitor CompactDurationVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {
// use exit hook to use compact results of children
- if _, hasNestLevel := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel); !hasNestLevel {
+ if _, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); !isPseudo {
return
}
diff --git a/pkg/frontend/tf/defaults/step/extract_nesting.go b/pkg/frontend/tf/defaults/step/extract_nesting.go
index 533e9297..0d4e3b69 100644
--- a/pkg/frontend/tf/defaults/step/extract_nesting.go
+++ b/pkg/frontend/tf/defaults/step/extract_nesting.go
@@ -20,6 +20,7 @@ import (
tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/manager"
+ utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal"
"github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
@@ -31,11 +32,10 @@ func init() {
)
}
-// Deletes spans matching MatchesNestLevel and brings their children one level up.
+// Deletes spans matching MatchesPseudoType and brings their children one level up.
type ExtractNestingVisitor struct {
- // NestLevels returns true if the span should be deleted.
- // It is only called on spans with the tag zconstants.Nesting
- MatchesNestLevel StringFilter `json:"matchesNestLevel"`
+ // Filters the trace sources to delete.
+ MatchesPseudoType utilmarshal.StringFilter `json:"matchesPseudoType"`
}
func (ExtractNestingVisitor) Kind() string { return "ExtractNestingVisitor" }
@@ -46,8 +46,8 @@ func (visitor ExtractNestingVisitor) Enter(tree *tftree.SpanTree, span *model.Sp
return visitor
}
- if nestLevel, ok := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel); ok {
- if visitor.MatchesNestLevel.Test(nestLevel.AsString()) {
+ if pseudoType, ok := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); ok {
+ if visitor.MatchesPseudoType.Matches(pseudoType.AsString()) {
childrenMap := tree.Children(span.SpanID)
childrenCopy := make([]model.SpanID, 0, len(childrenMap))
for childId := range childrenMap {
diff --git a/pkg/frontend/tf/defaults/step/group_by_trace_source.go b/pkg/frontend/tf/defaults/step/group_by_trace_source.go
index b376495e..b341d3a8 100644
--- a/pkg/frontend/tf/defaults/step/group_by_trace_source.go
+++ b/pkg/frontend/tf/defaults/step/group_by_trace_source.go
@@ -22,6 +22,7 @@ import (
tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
tftree "github.com/kubewharf/kelemetry/pkg/frontend/tf/tree"
"github.com/kubewharf/kelemetry/pkg/manager"
+ utilmarshal "github.com/kubewharf/kelemetry/pkg/util/marshal"
"github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
@@ -33,18 +34,18 @@ func init() {
)
}
-const pseudoSpanNestLevel = "groupByTraceSource"
+const myPseudoType = "groupByTraceSource"
// Splits span logs into pseudospans grouped by traceSource.
type GroupByTraceSourceVisitor struct {
- ShouldBeGrouped StringFilter `json:"shouldBeGrouped"`
+ ShouldBeGrouped utilmarshal.StringFilter `json:"shouldBeGrouped"`
}
func (GroupByTraceSourceVisitor) Kind() string { return "GroupByTraceSourceVisitor" }
func (visitor GroupByTraceSourceVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor {
- nestLevel, hasNestLevel := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel)
- if hasNestLevel && nestLevel.AsString() == pseudoSpanNestLevel {
+ pseudoType, hasPseudoType := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType)
+ if hasPseudoType && pseudoType.AsString() == myPseudoType {
// already grouped, don't recurse
return visitor
}
@@ -54,7 +55,7 @@ func (visitor GroupByTraceSourceVisitor) Enter(tree *tftree.SpanTree, span *mode
index := map[string][]model.Log{}
for _, log := range span.Logs {
traceSource, hasTraceSource := model.KeyValues(log.Fields).FindByKey(zconstants.TraceSource)
- if hasTraceSource && visitor.ShouldBeGrouped.Test(traceSource.AsString()) {
+ if hasTraceSource && visitor.ShouldBeGrouped.Matches(traceSource.AsString()) {
index[traceSource.AsString()] = append(index[traceSource.AsString()], log)
} else {
remainingLogs = append(remainingLogs, log)
@@ -74,9 +75,9 @@ func (visitor GroupByTraceSourceVisitor) Enter(tree *tftree.SpanTree, span *mode
Duration: span.Duration,
Tags: []model.KeyValue{
{
- Key: zconstants.NestLevel,
+ Key: zconstants.PseudoType,
VType: model.StringType,
- VStr: pseudoSpanNestLevel,
+ VStr: myPseudoType,
},
},
Logs: logs,
diff --git a/pkg/frontend/tf/defaults/step/object_tags.go b/pkg/frontend/tf/defaults/step/object_tags.go
index 46694609..134a6bb1 100644
--- a/pkg/frontend/tf/defaults/step/object_tags.go
+++ b/pkg/frontend/tf/defaults/step/object_tags.go
@@ -31,6 +31,7 @@ func init() {
)
}
+// Copy tags from child spans to the object.
type ObjectTagsVisitor struct {
ResourceTags []string `json:"resourceTags"`
}
@@ -38,7 +39,8 @@ type ObjectTagsVisitor struct {
func (ObjectTagsVisitor) Kind() string { return "ObjectTagsVisitor" }
func (visitor ObjectTagsVisitor) Enter(tree *tftree.SpanTree, span *model.Span) tftree.TreeVisitor {
- if tagKv, hasTag := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel); !hasTag || tagKv.VStr != zconstants.NestLevelObject {
+ if tagKv, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); !isPseudo ||
+ tagKv.VStr != string(zconstants.PseudoTypeObject) {
return visitor
}
if _, hasTag := model.KeyValues(span.Tags).FindByKey("resource"); !hasTag {
@@ -61,10 +63,14 @@ func (visitor ObjectTagsVisitor) findTagRecursively(tree *tftree.SpanTree, span
for childId := range tree.Children(span.SpanID) {
childSpan := tree.Span(childId)
- if tagKv, hasTag := model.KeyValues(childSpan.Tags).FindByKey(zconstants.NestLevel); hasTag &&
- tagKv.VStr == zconstants.NestLevelObject {
- continue
+ {
+ tagKv, isPseudo := model.KeyValues(childSpan.Tags).FindByKey(zconstants.PseudoType)
+ if isPseudo && tagKv.VStr == string(zconstants.PseudoTypeObject) {
+ // do not copy from another object
+ continue
+ }
}
+
kv := visitor.findTagRecursively(tree, childSpan, tagKey)
if len(kv.Key) > 0 {
span.Tags = append(span.Tags, kv)
diff --git a/pkg/frontend/tf/defaults/step/prune_childless.go b/pkg/frontend/tf/defaults/step/prune_childless.go
index ed15d006..26742542 100644
--- a/pkg/frontend/tf/defaults/step/prune_childless.go
+++ b/pkg/frontend/tf/defaults/step/prune_childless.go
@@ -41,7 +41,7 @@ func (visitor PruneChildlessVisitor) Enter(tree *tftree.SpanTree, span *model.Sp
// Prune in postorder traversal to recursively remove higher pseudospans without leaves.
func (visitor PruneChildlessVisitor) Exit(tree *tftree.SpanTree, span *model.Span) {
- if _, hasTag := model.KeyValues(span.Tags).FindByKey(zconstants.NestLevel); hasTag {
+ if _, isPseudo := model.KeyValues(span.Tags).FindByKey(zconstants.PseudoType); isPseudo {
if len(tree.Children(span.SpanID)) == 0 && span.SpanID != tree.Root.SpanID {
tree.Delete(span.SpanID)
}
diff --git a/pkg/frontend/tf/defaults/step/util.go b/pkg/frontend/tf/defaults/step/util.go
deleted file mode 100644
index a3864870..00000000
--- a/pkg/frontend/tf/defaults/step/util.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2023 The Kelemetry Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tfstep
-
-import (
- "encoding/json"
- "strings"
-
- "k8s.io/apimachinery/pkg/util/sets"
-)
-
-type StringFilter struct {
- OneOf []string `json:"oneOf"`
- Negate bool `json:"negate"`
- IgnoreCase bool `json:"ignoreCase"`
-}
-
-// Test whether a string matches the filter.
-func (f *StringFilter) Test(s string) bool {
- for _, choice := range f.OneOf {
- if (f.IgnoreCase && strings.EqualFold(s, choice)) || s == choice {
- return !f.Negate
- }
- }
-
- return f.Negate
-}
-
-type JsonStringSet struct {
- set sets.Set[string]
-}
-
-func (set *JsonStringSet) UnmarshalJSON(buf []byte) error {
- strings := []string{}
- if err := json.Unmarshal(buf, &strings); err != nil {
- return err
- }
- set.set = sets.New[string](strings...)
- return nil
-}
diff --git a/pkg/frontend/tf/extension.go b/pkg/frontend/tf/extension.go
index f3a0e252..24875a0b 100644
--- a/pkg/frontend/tf/extension.go
+++ b/pkg/frontend/tf/extension.go
@@ -96,7 +96,7 @@ func (x *FetchExtensionsAndStoreCache) ProcessExtensions(
span := span
tags := model.KeyValues(span.Tags)
- if tag, exists := tags.FindByKey(zconstants.NestLevel); exists && tag.VStr == zconstants.NestLevelObject {
+ if tag, isPseudo := tags.FindByKey(zconstants.PseudoType); isPseudo && tag.VStr == string(zconstants.PseudoTypeObject) {
for extId, ext := range extensions {
ext := ext
diff --git a/pkg/frontend/tf/transform.go b/pkg/frontend/tf/transform.go
index 8472a0f8..4b1849ae 100644
--- a/pkg/frontend/tf/transform.go
+++ b/pkg/frontend/tf/transform.go
@@ -16,14 +16,12 @@ package transform
import (
"context"
- "errors"
"fmt"
"time"
"github.com/jaegertracing/jaeger/model"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
- "k8s.io/apimachinery/pkg/util/sets"
"github.com/kubewharf/kelemetry/pkg/frontend/extension"
tfconfig "github.com/kubewharf/kelemetry/pkg/frontend/tf/config"
@@ -38,8 +36,7 @@ func init() {
type TransformerOptions struct{}
-func (options *TransformerOptions) Setup(fs *pflag.FlagSet) {
-}
+func (options *TransformerOptions) Setup(fs *pflag.FlagSet) {}
func (options *TransformerOptions) EnableFlag() *bool { return nil }
@@ -74,34 +71,6 @@ func (transformer *Transformer) Transform(
tree := tftree.NewSpanTree(trace.Spans)
- transformer.groupDuplicates(tree)
-
- if config.UseSubtree && rootObject != nil {
- var rootSpan model.SpanID
- hasRootSpan := false
-
- for _, span := range tree.GetSpans() {
- if key, hasKey := utilobject.FromSpan(span); hasKey && key == *rootObject {
- rootSpan = span.SpanID
- hasRootSpan = true
- }
- }
-
- if hasRootSpan {
- if err := tree.SetRoot(rootSpan); err != nil {
- if errors.Is(err, tftree.ErrRootDoesNotExist) {
- return fmt.Errorf(
- "trace data does not contain desired root span %v as indicated by the exclusive flag (%w)",
- rootSpan,
- err,
- )
- }
-
- return fmt.Errorf("cannot set root: %w", err)
- }
- }
- }
-
newSpans, err := extensionProcessor.ProcessExtensions(ctx, transformer, config.Extensions, trace.Spans, start, end)
if err != nil {
return fmt.Errorf("cannot prepare extension trace: %w", err)
@@ -118,48 +87,3 @@ func (transformer *Transformer) Transform(
return nil
}
-
-// merge spans of the same object from multiple traces
-func (transformer *Transformer) groupDuplicates(tree *tftree.SpanTree) {
- commonSpans := map[utilobject.Key][]model.SpanID{}
-
- for _, span := range tree.GetSpans() {
- if key, hasKey := utilobject.FromSpan(span); hasKey {
- commonSpans[key] = append(commonSpans[key], span.SpanID)
- }
- }
-
- for _, spans := range commonSpans {
- if len(spans) > 1 {
- // only retain the first span
-
- desiredParent := spans[0]
-
- originalTags := tree.Span(desiredParent).Tags
- originalTagKeys := make(sets.Set[string], len(originalTags))
- for _, tag := range originalTags {
- originalTagKeys.Insert(tag.Key)
- }
-
- for _, obsoleteParent := range spans[1:] {
- for _, tag := range tree.Span(obsoleteParent).Tags {
- if !originalTagKeys.Has(tag.Key) {
- originalTags = append(originalTags, tag)
- originalTagKeys.Insert(tag.Key)
- }
- }
-
- children := tree.Children(obsoleteParent)
- for child := range children {
- tree.Move(child, desiredParent)
- }
-
- if tree.Root.SpanID == obsoleteParent {
- tree.Root = tree.Span(desiredParent)
- }
-
- tree.Delete(obsoleteParent)
- }
- }
- }
-}
diff --git a/pkg/frontend/tf/tree/tree.go b/pkg/frontend/tf/tree/tree.go
index c8327f42..dfd2ce61 100644
--- a/pkg/frontend/tf/tree/tree.go
+++ b/pkg/frontend/tf/tree/tree.go
@@ -15,6 +15,7 @@
package tftree
import (
+ "encoding/json"
"fmt"
"github.com/jaegertracing/jaeger/model"
@@ -68,6 +69,34 @@ func NewSpanTree(spans []*model.Span) *SpanTree {
return tree
}
+func (tree *SpanTree) Clone() (*SpanTree, error) {
+ copiedSpans := make([]*model.Span, 0, len(tree.spanMap))
+ for _, span := range tree.spanMap {
+ spanCopy, err := CopySpan(span)
+ if err != nil {
+ return nil, err
+ }
+
+ copiedSpans = append(copiedSpans, spanCopy)
+ }
+
+ return NewSpanTree(copiedSpans), nil
+}
+
+func CopySpan(span *model.Span) (*model.Span, error) {
+ spanJson, err := json.Marshal(span)
+ if err != nil {
+ return nil, err
+ }
+
+ var spanCopy *model.Span
+ if err := json.Unmarshal(spanJson, &spanCopy); err != nil {
+ return nil, err
+ }
+
+ return spanCopy, nil
+}
+
func (tree *SpanTree) Span(id model.SpanID) *model.Span { return tree.spanMap[id] }
func (tree *SpanTree) Children(id model.SpanID) map[model.SpanID]struct{} {
return tree.childrenMap[id]
@@ -133,7 +162,10 @@ func (subtree spanNode) visit(visitor TreeVisitor) {
panic("cannot visit nonexistent node in tree")
}
- subvisitor := visitor.Enter(subtree.tree, subtree.node)
+ var subvisitor TreeVisitor
+ if visitor != nil {
+ subvisitor = visitor.Enter(subtree.tree, subtree.node)
+ }
// enter before visitorStack is populated to allow removal
if _, stillExists := subtree.tree.spanMap[subtree.node.SpanID]; !stillExists {
@@ -169,7 +201,10 @@ func (subtree spanNode) visit(visitor TreeVisitor) {
delete(subtree.tree.visitorStack, subtree.node.SpanID)
- visitor.Exit(subtree.tree, subtree.node)
+ if visitor != nil {
+ visitor.Exit(subtree.tree, subtree.node)
+ }
+
if _, stillExists := subtree.tree.spanMap[subtree.node.SpanID]; !stillExists {
// deleted during exit
return
@@ -282,6 +317,24 @@ func (tree *SpanTree) Delete(spanId model.SpanID) {
}
}
+// Adds all spans in a tree as a subtree in this span.
+//
+// TODO FIXME: when the two trees have overlapping span IDs, this does not work correctly.
+func (tree *SpanTree) AddTree(childTree *SpanTree, parentId model.SpanID) {
+ if tree == childTree {
+ panic("cannot add tree to itself")
+ }
+
+ tree.addSubtree(parentId, childTree, childTree.Root.SpanID)
+}
+
+func (tree *SpanTree) addSubtree(parentId model.SpanID, otherTree *SpanTree, subroot model.SpanID) {
+ tree.Add(otherTree.Span(subroot), parentId)
+ for child := range otherTree.Children(subroot) {
+ tree.addSubtree(subroot, otherTree, child)
+ }
+}
+
type TreeVisitor interface {
// Called before entering the descendents of the span.
//
diff --git a/pkg/imports.go b/pkg/imports.go
index f309a279..f7a779ff 100644
--- a/pkg/imports.go
+++ b/pkg/imports.go
@@ -18,6 +18,8 @@ package kelemetry_pkg
import (
_ "github.com/kubewharf/kelemetry/pkg/aggregator/aggregatorevent"
_ "github.com/kubewharf/kelemetry/pkg/aggregator/eventdecorator/eventtagger"
+ _ "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job/local"
+ _ "github.com/kubewharf/kelemetry/pkg/aggregator/linker/job/worker"
_ "github.com/kubewharf/kelemetry/pkg/aggregator/objectspandecorator/resourcetagger"
_ "github.com/kubewharf/kelemetry/pkg/aggregator/spancache/etcd"
_ "github.com/kubewharf/kelemetry/pkg/aggregator/spancache/local"
diff --git a/pkg/ownerlinker/linker.go b/pkg/ownerlinker/linker.go
index eaf0bc7b..9f055379 100644
--- a/pkg/ownerlinker/linker.go
+++ b/pkg/ownerlinker/linker.go
@@ -16,6 +16,7 @@ package ownerlinker
import (
"context"
+ "fmt"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
@@ -26,7 +27,9 @@ import (
"github.com/kubewharf/kelemetry/pkg/k8s/discovery"
"github.com/kubewharf/kelemetry/pkg/k8s/objectcache"
"github.com/kubewharf/kelemetry/pkg/manager"
+ "github.com/kubewharf/kelemetry/pkg/metrics"
utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+ "github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
func init() {
@@ -58,7 +61,8 @@ func (ctrl *Controller) Init() error { return nil }
func (ctrl *Controller) Start(ctx context.Context) error { return nil }
func (ctrl *Controller) Close(ctx context.Context) error { return nil }
-func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *utilobject.Rich {
+func (ctrl *Controller) LinkerName() string { return "owner-linker" }
+func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) ([]linker.LinkerResult, error) {
raw := object.Raw
logger := ctrl.Logger.WithFields(object.AsFields("object"))
@@ -70,16 +74,17 @@ func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
raw, err = ctrl.ObjectCache.Get(ctx, object.VersionedKey)
if err != nil {
- logger.WithError(err).Error("cannot fetch object value")
- return nil
+ return nil, metrics.LabelError(fmt.Errorf("cannot fetch object value from cache: %w", err), "FetchCache")
}
if raw == nil {
logger.Debug("object no longer exists")
- return nil
+ return nil, nil
}
}
+ var results []linker.LinkerResult
+
for _, owner := range raw.GetOwnerReferences() {
if owner.Controller != nil && *owner.Controller {
groupVersion, err := schema.ParseGroupVersion(owner.APIVersion)
@@ -101,12 +106,12 @@ func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
continue
}
- ret := &utilobject.Rich{
+ parentRef := utilobject.Rich{
VersionedKey: utilobject.VersionedKey{
Key: utilobject.Key{
Cluster: object.Cluster,
Group: gvr.Group,
- Resource: gvr.Group,
+ Resource: gvr.Resource,
Namespace: object.Namespace,
Name: owner.Name,
},
@@ -114,11 +119,16 @@ func (ctrl *Controller) Lookup(ctx context.Context, object utilobject.Rich) *uti
},
Uid: owner.UID,
}
- logger.WithField("owner", ret).Debug("Resolved owner")
-
- return ret
+ logger.WithField("owner", parentRef).Debug("Resolved owner")
+
+ results = append(results, linker.LinkerResult{
+ Object: parentRef,
+ Role: zconstants.LinkRoleParent,
+ Class: "children",
+ DedupId: "ownerReference",
+ })
}
}
- return nil
+ return results, nil
}
diff --git a/pkg/util/marshal/marshal.go b/pkg/util/marshal/marshal.go
new file mode 100644
index 00000000..3066a426
--- /dev/null
+++ b/pkg/util/marshal/marshal.go
@@ -0,0 +1,179 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Utilities for unmarshalling in config files
+package utilmarshal
+
+import (
+ "encoding/json"
+ "fmt"
+ "regexp"
+ "strings"
+
+ "k8s.io/apimachinery/pkg/util/sets"
+
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+type Optional[T any] struct {
+ IsSet bool
+ Value T
+}
+
+func (opt Optional[T]) GetOr(defaultValue T) T {
+ if opt.IsSet {
+ return opt.Value
+ } else {
+ return defaultValue
+ }
+}
+
+func IsSetTo[T comparable](opt Optional[T], t T) bool {
+ return opt.IsSet && opt.Value == t
+}
+
+func (v *Optional[T]) UnmarshalJSON(buf []byte) error {
+ if err := json.Unmarshal(buf, &v.Value); err != nil {
+ return err
+ }
+
+ v.IsSet = true
+ return nil
+}
+
+type ObjectFilter struct {
+ Cluster Optional[StringFilter] `json:"cluster"`
+ Group Optional[StringFilter] `json:"group"`
+ Resource Optional[StringFilter] `json:"resource"`
+ Namespace Optional[StringFilter] `json:"namespace"`
+ Name Optional[StringFilter] `json:"name"`
+}
+
+func (filter *ObjectFilter) Matches(key utilobject.Key) bool {
+ if filter.Cluster.IsSet {
+ if !filter.Cluster.Value.Matches(key.Cluster) {
+ return false
+ }
+ }
+
+ if filter.Group.IsSet {
+ if !filter.Group.Value.Matches(key.Group) {
+ return false
+ }
+ }
+
+ if filter.Resource.IsSet {
+ if !filter.Resource.Value.Matches(key.Resource) {
+ return false
+ }
+ }
+
+ if filter.Namespace.IsSet {
+ if !filter.Namespace.Value.Matches(key.Namespace) {
+ return false
+ }
+ }
+
+ if filter.Name.IsSet {
+ if !filter.Name.Value.Matches(key.Name) {
+ return false
+ }
+ }
+
+ return true
+}
+
+type stringPredicate = func(string) bool
+
+type StringFilter struct {
+ fn stringPredicate
+}
+
+type fields struct {
+ Exact Optional[string] `json:"exact"`
+ OneOf Optional[[]string] `json:"oneOf"`
+ CaseFold Optional[string] `json:"caseInsensitive"`
+ Regex Optional[string] `json:"regex"`
+ Then Optional[bool] `json:"then"`
+}
+
+func (value fields) getBasePredicate() (stringPredicate, error) {
+ isSet := 0
+ for _, b := range []bool{value.Exact.IsSet, value.OneOf.IsSet, value.CaseFold.IsSet, value.Regex.IsSet} {
+ if b {
+ isSet += 1
+ }
+ }
+
+ if isSet > 1 {
+ return nil, fmt.Errorf("string filter must set exactly one of `exact`, `oneOf`, `caseInsensitive` or `regex`")
+ }
+
+ if value.Exact.IsSet {
+ return func(s string) bool { return s == value.Exact.Value }, nil
+ } else if value.OneOf.IsSet {
+ options := sets.New[string](value.OneOf.Value...)
+ return options.Has, nil
+ } else if value.CaseFold.IsSet {
+ return func(s string) bool { return strings.EqualFold(value.CaseFold.Value, value.CaseFold.Value) }, nil
+ } else if value.Regex.IsSet {
+ regex, err := regexp.Compile(value.Regex.Value)
+ if err != nil {
+ return nil, fmt.Errorf("pattern contains invalid regex: %w", err)
+ }
+
+ return regex.MatchString, nil
+ } else {
+ return func(string) bool { return true }, nil // caller will change value to `then`
+ }
+}
+
+func (f *StringFilter) UnmarshalJSON(buf []byte) error {
+ var pattern string
+ if err := json.Unmarshal(buf, &pattern); err == nil {
+ f.fn = func(s string) bool { return s == pattern }
+ return nil
+ }
+
+ var value fields
+ if err := json.Unmarshal(buf, &value); err != nil {
+ return err
+ }
+
+ predicate, err := value.getBasePredicate()
+ if err != nil {
+ return err
+ }
+
+ then := value.Then.GetOr(true)
+ f.fn = func(s string) bool {
+ base := predicate(s)
+ if base {
+ return then
+ } else {
+ return !then
+ }
+ }
+
+ return nil
+}
+
+func (f *StringFilter) Matches(subject string) bool {
+ if f.fn == nil {
+ // string filter not set, take default then = true
+ return true
+ }
+
+ return f.fn(subject)
+}
diff --git a/pkg/util/object/key.go b/pkg/util/object/key.go
index 1208c076..bb208a6c 100644
--- a/pkg/util/object/key.go
+++ b/pkg/util/object/key.go
@@ -18,13 +18,9 @@ import (
"fmt"
"strings"
- "github.com/jaegertracing/jaeger/model"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
- "k8s.io/apimachinery/pkg/util/sets"
-
- "github.com/kubewharf/kelemetry/pkg/util/zconstants"
)
type Key struct {
@@ -80,39 +76,6 @@ func FromMap(tags map[string]string) (key Key, ok bool) {
return key, true
}
-func FromSpan(span *model.Span) (Key, bool) {
- tags := model.KeyValues(span.Tags)
- traceSource, hasTraceSource := tags.FindByKey(zconstants.TraceSource)
- if !hasTraceSource || traceSource.VStr != zconstants.TraceSourceObject {
- return Key{}, false
- }
-
- cluster, _ := tags.FindByKey("cluster")
- group, _ := tags.FindByKey("group")
- resource, _ := tags.FindByKey("resource")
- namespace, _ := tags.FindByKey("namespace")
- name, _ := tags.FindByKey("name")
- key := Key{
- Cluster: cluster.VStr,
- Group: group.VStr,
- Resource: resource.VStr,
- Namespace: namespace.VStr,
- Name: name.VStr,
- }
- return key, true
-}
-
-func FromSpans(spans []*model.Span) sets.Set[Key] {
- keys := sets.New[Key]()
-
- for _, span := range spans {
- if key, ok := FromSpan(span); ok {
- keys.Insert(key)
- }
- }
- return keys
-}
-
type VersionedKey struct {
Key
Version string `json:"version"`
diff --git a/pkg/util/reflect/reflect.go b/pkg/util/reflect/reflect.go
index 5e3f60a1..3ce6179d 100644
--- a/pkg/util/reflect/reflect.go
+++ b/pkg/util/reflect/reflect.go
@@ -23,3 +23,5 @@ func TypeOf[T any]() reflect.Type {
}
func ZeroOf[T any]() (_ T) { return }
+
+func Identity[T any](t T) T { return t }
diff --git a/pkg/util/semaphore/semaphore.go b/pkg/util/semaphore/semaphore.go
index c04070c5..da68928b 100644
--- a/pkg/util/semaphore/semaphore.go
+++ b/pkg/util/semaphore/semaphore.go
@@ -108,10 +108,23 @@ func (sem *Semaphore) Schedule(task Task) {
}
} else {
if publish != nil {
+ sem.doneWg.Add(1)
+ wrappedPublish := func() error {
+ defer sem.doneWg.Done()
+ return publish()
+ }
+
select {
- case sem.publishCh <- publish:
+ case sem.publishCh <- wrappedPublish:
+ // publishCh has zero capacity, so this case blocks until the main goroutine selects the publishCh case,
+ // so we can ensure that publishCh is received before calling sem.doneWg.Done().
+ // However we need to call doneWg again to ensure that
+ // schedules during publish are called before this function brings doneWg to zero.
+
+ // the main goroutine will call `sem.doneWg.Done()` for us.
case <-sem.errNotifyCh:
// no need to publish if the caller received error
+ sem.doneWg.Done()
}
}
}
diff --git a/pkg/util/zconstants/link.go b/pkg/util/zconstants/link.go
new file mode 100644
index 00000000..dd17ffa5
--- /dev/null
+++ b/pkg/util/zconstants/link.go
@@ -0,0 +1,130 @@
+// Copyright 2023 The Kelemetry Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zconstants
+
+import (
+ "github.com/jaegertracing/jaeger/model"
+
+ utilobject "github.com/kubewharf/kelemetry/pkg/util/object"
+)
+
+type LinkRef struct {
+ Key utilobject.Key
+ Role LinkRoleValue
+ Class string
+}
+
+// Tags for TraceSourceLink spans that indicate the linked object.
+const (
+ LinkedObjectCluster = "linkedCluster"
+ LinkedObjectGroup = "linkedGroup"
+ LinkedObjectResource = "linkedResource"
+ LinkedObjectNamespace = "linkedNamespace"
+ LinkedObjectName = "linkedName"
+
+ // Indicates how the linked trace interacts with the current trace.
+ LinkRole = "linkRole"
+
+ // If this tag is nonempty, a virtual span is inserted between the linked objects with the tag value as the name.
+ LinkClass = "linkClass"
+)
+
+func TagLinkedObject(tags map[string]string, ln LinkRef) {
+ tags[LinkedObjectCluster] = ln.Key.Cluster
+ tags[LinkedObjectGroup] = ln.Key.Group
+ tags[LinkedObjectResource] = ln.Key.Resource
+ tags[LinkedObjectNamespace] = ln.Key.Namespace
+ tags[LinkedObjectName] = ln.Key.Name
+ tags[LinkRole] = string(ln.Role)
+ tags[LinkClass] = ln.Class
+}
+
+func ObjectKeyFromSpan(span *model.Span) utilobject.Key {
+ tags := model.KeyValues(span.Tags)
+
+ cluster, _ := tags.FindByKey("cluster")
+ group, _ := tags.FindByKey("group")
+ resource, _ := tags.FindByKey("resource")
+ namespace, _ := tags.FindByKey("namespace")
+ name, _ := tags.FindByKey("name")
+ key := utilobject.Key{
+ Cluster: cluster.VStr,
+ Group: group.VStr,
+ Resource: resource.VStr,
+ Namespace: namespace.VStr,
+ Name: name.VStr,
+ }
+ return key
+}
+
+func LinkedKeyFromSpan(span *model.Span) (utilobject.Key, bool) {
+ tags := model.KeyValues(span.Tags)
+ pseudoType, isPseudo := tags.FindByKey(PseudoType)
+ if !isPseudo || pseudoType.VStr != string(PseudoTypeLink) {
+ return utilobject.Key{}, false
+ }
+
+ cluster, _ := tags.FindByKey(LinkedObjectCluster)
+ group, _ := tags.FindByKey(LinkedObjectGroup)
+ resource, _ := tags.FindByKey(LinkedObjectResource)
+ namespace, _ := tags.FindByKey(LinkedObjectNamespace)
+ name, _ := tags.FindByKey(LinkedObjectName)
+ key := utilobject.Key{
+ Cluster: cluster.VStr,
+ Group: group.VStr,
+ Resource: resource.VStr,
+ Namespace: namespace.VStr,
+ Name: name.VStr,
+ }
+ return key, true
+}
+
+func KeyToSpanTags(key utilobject.Key) map[string]string {
+ return map[string]string{
+ "cluster": key.Cluster,
+ "group": key.Group,
+ "resource": key.Resource,
+ "namespace": key.Namespace,
+ "name": key.Name,
+ }
+}
+
+func VersionedKeyToSpanTags(key utilobject.VersionedKey) map[string]string {
+ m := KeyToSpanTags(key.Key)
+ m["version"] = key.Version
+ return m
+}
+
+type LinkRoleValue string
+
+const (
+ // The current trace is a child trace under the linked trace
+ LinkRoleParent LinkRoleValue = "parent"
+
+ // The linked trace is a child trace under the current trace.
+ LinkRoleChild LinkRoleValue = "child"
+)
+
+// Determines the role of the reverse link.
+func ReverseLinkRole(role LinkRoleValue) LinkRoleValue {
+ switch role {
+ case LinkRoleParent:
+ return LinkRoleChild
+ case LinkRoleChild:
+ return LinkRoleParent
+ default:
+ return role
+ }
+}
diff --git a/pkg/util/zconstants/zconstants.go b/pkg/util/zconstants/zconstants.go
index bfac7185..02faeb5a 100644
--- a/pkg/util/zconstants/zconstants.go
+++ b/pkg/util/zconstants/zconstants.go
@@ -16,7 +16,9 @@
// for span transformation in the frontend storage plugin.
package zconstants
-import "time"
+import (
+ "time"
+)
// All tags with this prefix are not rendered.
const Prefix = "zzz-"
@@ -26,10 +28,23 @@ const SpanName = Prefix + "kelemetryName"
// Indicates that the current span is a pseudospan that can be folded or flattened.
// The value is the folding type.
-const NestLevel = Prefix + "nestingLevel"
+const PseudoType = Prefix + "pseudoType"
+
+// Indicates that the current span is not pseudo.
+// Used to optimize trace listing.
+//
+// This constant is used as both tag key and value.
+const NotPseudo = Prefix + "notPseudo"
+
+type PseudoTypeValue string
const (
- NestLevelObject = "object"
+ // Root span in an object trace.
+ PseudoTypeObject PseudoTypeValue = "object"
+ // Indicate that another trace shall be included.
+ PseudoTypeLink PseudoTypeValue = "link"
+ // A virtual span synthesized in the frontend when link class is nonempty.
+ PseudoTypeLinkClass PseudoTypeValue = "linkClass"
)
// Identifies that the span represents an actual event (rather than as a pseudospan).
@@ -37,11 +52,14 @@ const TraceSource = Prefix + "traceSource"
const (
TraceSourceObject = "object"
- TraceSourceAudit = "audit"
- TraceSourceEvent = "event"
+
+ TraceSourceAudit = "audit"
+ TraceSourceEvent = "event"
)
func KnownTraceSources(withPseudo bool) []string {
+ numPseudoTraceSources := 1
+
traceSources := []string{
// pseudo
TraceSourceObject,
@@ -52,7 +70,7 @@ func KnownTraceSources(withPseudo bool) []string {
}
if !withPseudo {
- traceSources = traceSources[1:]
+ traceSources = traceSources[numPseudoTraceSources:]
}
return traceSources
diff --git a/quickstart.docker-compose.yaml b/quickstart.docker-compose.yaml
index dbaf8a37..8272a2fd 100644
--- a/quickstart.docker-compose.yaml
+++ b/quickstart.docker-compose.yaml
@@ -70,6 +70,7 @@ services:
"--diff-cache-wrapper-enable",
"--diff-controller-leader-election-enable=false",
"--event-informer-leader-election-enable=false",
+ "--linker-worker-count=8",
"--span-cache=etcd",
"--span-cache-etcd-endpoints=etcd:2379",
"--tracer-otel-endpoint=jaeger-collector:4317",