From 4c88ec92ee937ea6ba7d24e3a1f7df190f621c98 Mon Sep 17 00:00:00 2001 From: Hisar Balik Date: Wed, 18 Dec 2024 11:46:02 +0100 Subject: [PATCH 1/3] update OTTL ADR with multi tranform/filter processor --- ...-filter-support-for-telemetry-pipelines.md | 62 +++++++++++++++---- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md index 4e4072279..20c06cc78 100644 --- a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md +++ b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md @@ -12,7 +12,7 @@ In the default setup of metric and trace pipelines, users currently cannot filte ## Decision -We will implement a consolidated solution in the OpenTelemetry Collector (OTel Collector) using a single Filter and Transform Processor. This processor will use the OpenTelemetry Transformation and Transport Language (OTTL) to handle both filtering and transformation tasks. Users will be able to configure the processor as needed; only a subset of OTTL functions will be supported, focusing on the most common and impactful use cases. +We will implement a consolidated solution in the OpenTelemetry Collector (OTel Collector) using Filter and Transform Processor. These processors will use the OpenTelemetry Transformation and Transport Language (OTTL) to handle both filtering and transformation tasks. Users will be able to configure the processor as needed; only a subset of OTTL functions will be supported, focusing on the most common and impactful use cases. MetricPipeline example configuration: @@ -23,15 +23,24 @@ metadata: name: metricpipeline-sample spec: transform: - conditions: + - name: set-description + conditions: - type == METRIC_DATA_TYPE_SUM - statements: + - IsMatch(attributes["service.name"], "unknown") + statements: - set(description, "Sum") + - name: convert-to-gauge + conditions: + - type == METRIC_DATA_TYPE_SUM + - IsMatch(attributes["service.name"], "unknown") + statements: + - convert_sum_to_gauge() where name == "system.processes.count" and (type == METRIC_DATA_TYPE_SUM or IsMatch(attributes["service.name"], "unknown") filter: - metric: - - type == METRIC_DATA_TYPE_NONE - datapoint: - - metric.name == "k8s.pod.phase" and value_int == 4 + - name: filter-by-metric + metric: + - type == METRIC_DATA_TYPE_NONE + datapoint: + - metric.name == "k8s.pod.phase" and value_int == 4 input: istio: enabled: true @@ -52,11 +61,16 @@ metadata: name: tracepipeline-sample spec: transform: - statements: - - set(status.code, 1) where attributes["http.path"] == "/health" + - name: set-status-code + statements: + - set(status.code, 1) where attributes["http.path"] == "/health" filter: - span: + - name: filter-by-span + span: - IsMatch(resource.attributes["k8s.pod.name"], "my-pod-name.*") + - name: filter-by-attribute + spanevent: + - 'attributes["grpc"] == true' output: otlp: endpoint: @@ -79,12 +93,33 @@ The recommended error mode is `ignore`, and this will be used as the default con The OTTL context will be embedded in the OTTL statements (in progress with [issue #29017](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017)) and will be available in the upcoming beta version. The solution will not implement the context as a configuration parameter. -To ensure data consistency and sampling efficiency, the custom OTTL transformation and filtering processor will be near the end of the pipeline chain, before the exporters. +To ensure data consistency and sampling efficiency, the custom OTTL transformation and filtering processors will be near the end of the pipeline chain, before the exporters. See the OTel configuration: ```yaml service: + processors: + transform/custom-set-description: + error_mode: ignore + metric_statements: + - conditions: + - type == METRIC_DATA_TYPE_SUM + - IsMatch(attributes["service.name"], "unknown") + statements: + - set(description, "Sum") + transform/custom-convert-to-gauge: + error_mode: ignore + metric_statements: + - statements: + - convert_sum_to_gauge() where name == "system.processes.count" + filter/custom-filter-by-metric: + error_mode: ignore + metrics: + metric: + - type == METRIC_DATA_TYPE_NONE + datapoint: + - metric.name == "k8s.pod.phase" and value_int == 4 pipelines: metrics/test-output: receivers: @@ -97,8 +132,9 @@ service: - transform/set-instrumentation-scope-kyma - resource/insert-cluster-name - resource/delete-skip-enrichment-attribute - - transform/custom-transform-processor - - filter/custom-filter-processor + - transform/custom-set-description + - transform/custom-convert-to-gauge + - filter/custom-filter-by-metric - batch exporters: - otlp/test From cbfb0ddcad33df94021386e245b1ade624e4e08c Mon Sep 17 00:00:00 2001 From: Hisar Balik Date: Fri, 20 Dec 2024 10:41:00 +0100 Subject: [PATCH 2/3] udpate API proposal after discussion with team --- ...-filter-support-for-telemetry-pipelines.md | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md index 20c06cc78..6098afa9c 100644 --- a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md +++ b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md @@ -23,24 +23,19 @@ metadata: name: metricpipeline-sample spec: transform: - - name: set-description - conditions: + - conditions: - type == METRIC_DATA_TYPE_SUM - IsMatch(attributes["service.name"], "unknown") statements: - set(description, "Sum") - - name: convert-to-gauge - conditions: - - type == METRIC_DATA_TYPE_SUM - - IsMatch(attributes["service.name"], "unknown") + - conditions: + - type == METRIC_DATA_TYPE_NONE statements: - convert_sum_to_gauge() where name == "system.processes.count" and (type == METRIC_DATA_TYPE_SUM or IsMatch(attributes["service.name"], "unknown") filter: - - name: filter-by-metric - metric: - - type == METRIC_DATA_TYPE_NONE - datapoint: + conditions: - metric.name == "k8s.pod.phase" and value_int == 4 + - metric.type == METRIC_DATA_TYPE_NONE input: istio: enabled: true @@ -61,16 +56,14 @@ metadata: name: tracepipeline-sample spec: transform: - - name: set-status-code + - conditions: + - IsMatch(span.resource.attributes["k8s.pod.name"], "my-pod-name.*") statements: - set(status.code, 1) where attributes["http.path"] == "/health" filter: - - name: filter-by-span - span: - - IsMatch(resource.attributes["k8s.pod.name"], "my-pod-name.*") - - name: filter-by-attribute - spanevent: - - 'attributes["grpc"] == true' + conditions: + - attributes["grpc"] == true + - IsMatch(span.resource.attributes["k8s.pod.name"], "my-pod-name.*") output: otlp: endpoint: @@ -93,6 +86,9 @@ The recommended error mode is `ignore`, and this will be used as the default con The OTTL context will be embedded in the OTTL statements (in progress with [issue #29017](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017)) and will be available in the upcoming beta version. The solution will not implement the context as a configuration parameter. +The proposed API uses no context configuration for filter processors, instead allowing to configure only condition expressions; the conditions will be translated to the `datapoint` context for metrics and `spanevent` context for traces. +That accessing higher-level context is still possible via OTTL expressions, for example accessing the `metric` context from `datapoint` context is possible via expression `metric.*` and accessing `span` context from `spanevent` context via expression `span.*`. + To ensure data consistency and sampling efficiency, the custom OTTL transformation and filtering processors will be near the end of the pipeline chain, before the exporters. See the OTel configuration: @@ -100,7 +96,7 @@ See the OTel configuration: ```yaml service: processors: - transform/custom-set-description: + transform/custom: error_mode: ignore metric_statements: - conditions: @@ -108,18 +104,16 @@ service: - IsMatch(attributes["service.name"], "unknown") statements: - set(description, "Sum") - transform/custom-convert-to-gauge: - error_mode: ignore - metric_statements: - - statements: + - conditions: + - type == METRIC_DATA_TYPE_NONE + statements: - convert_sum_to_gauge() where name == "system.processes.count" - filter/custom-filter-by-metric: + filter/custom: error_mode: ignore metrics: - metric: - - type == METRIC_DATA_TYPE_NONE datapoint: - metric.name == "k8s.pod.phase" and value_int == 4 + - metric.type == METRIC_DATA_TYPE_NONE pipelines: metrics/test-output: receivers: @@ -132,9 +126,8 @@ service: - transform/set-instrumentation-scope-kyma - resource/insert-cluster-name - resource/delete-skip-enrichment-attribute - - transform/custom-set-description - - transform/custom-convert-to-gauge - - filter/custom-filter-by-metric + - transform/custom + - filter/custom - batch exporters: - otlp/test From b0af1aa005288b6b5795e96fc0be762af14f05a9 Mon Sep 17 00:00:00 2001 From: Hisar Balik Date: Mon, 23 Dec 2024 10:05:12 +0100 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Natalia Sitko <80401180+nataliasitko@users.noreply.github.com> --- .../016-transform-filter-support-for-telemetry-pipelines.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md index 6098afa9c..0e82d9f61 100644 --- a/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md +++ b/docs/contributor/arch/016-transform-filter-support-for-telemetry-pipelines.md @@ -12,7 +12,7 @@ In the default setup of metric and trace pipelines, users currently cannot filte ## Decision -We will implement a consolidated solution in the OpenTelemetry Collector (OTel Collector) using Filter and Transform Processor. These processors will use the OpenTelemetry Transformation and Transport Language (OTTL) to handle both filtering and transformation tasks. Users will be able to configure the processor as needed; only a subset of OTTL functions will be supported, focusing on the most common and impactful use cases. +We will implement a consolidated solution in the OpenTelemetry Collector (OTel Collector) using Filter and Transform processors. These processors will use the OpenTelemetry Transformation and Transport Language (OTTL) to handle both filtering and transformation tasks. Users will be able to configure the processor as needed; only a subset of OTTL functions will be supported, focusing on the most common and impactful use cases. MetricPipeline example configuration: @@ -86,8 +86,8 @@ The recommended error mode is `ignore`, and this will be used as the default con The OTTL context will be embedded in the OTTL statements (in progress with [issue #29017](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/29017)) and will be available in the upcoming beta version. The solution will not implement the context as a configuration parameter. -The proposed API uses no context configuration for filter processors, instead allowing to configure only condition expressions; the conditions will be translated to the `datapoint` context for metrics and `spanevent` context for traces. -That accessing higher-level context is still possible via OTTL expressions, for example accessing the `metric` context from `datapoint` context is possible via expression `metric.*` and accessing `span` context from `spanevent` context via expression `span.*`. +The proposed API uses no context configuration for filter processors. Instead, it only allows the configuration of condition expressions. The conditions are translated to the `datapoint` context for metrics and the `spanevent` context for traces. +Accessing higher-level context is still possible via OTTL expressions. For example, accessing the `metric` context from the `datapoint` context is possible via the expression `metric.*`, and accessing the `span` context from the `spanevent` context via the expression `span.*`. To ensure data consistency and sampling efficiency, the custom OTTL transformation and filtering processors will be near the end of the pipeline chain, before the exporters.