[processor/awsentity] Scrape auto scaling group attributes from resou…

…rce metrics (#1521)
aws · Jan 29, 2025 · 8d8b261 · 8d8b261
1 parent 44f7d6f
commit 8d8b261
Show file tree

Hide file tree

Showing 23 changed files with 133 additions and 43 deletions.
diff --git a/plugins/processors/awsentity/processor.go b/plugins/processors/awsentity/processor.go
@@ -16,6 +16,7 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent/extension/entitystore"
 	"github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/entityattributes"
 	"github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsentity/internal/k8sattributescraper"
+	"github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger"
 	"github.com/aws/amazon-cloudwatch-agent/translator/config"
 )
 
@@ -131,6 +132,12 @@ func (p *awsEntityProcessor) processMetrics(_ context.Context, md pmetric.Metric
 		switch p.config.EntityType {
 		case entityattributes.Resource:
 			if p.config.Platform == config.ModeEC2 {
+				// ec2tagger processor may have picked up the ASG name from an ec2:DescribeTags call
+				if getAutoScalingGroupFromEntityStore() == EMPTY && p.config.ScrapeDatapointAttribute {
+					if autoScalingGroup := p.scrapeResourceEntityAttribute(rm.At(i).ScopeMetrics()); autoScalingGroup != EMPTY {
+						setAutoScalingGroup(autoScalingGroup)
+					}
+				}
 				ec2Info = getEC2InfoFromEntityStore()
 				if ec2Info.GetInstanceID() != EMPTY {
 					resourceAttrs.PutStr(entityattributes.AttributeEntityType, entityattributes.AttributeEntityAWSResource)
@@ -285,7 +292,8 @@ func (p *awsEntityProcessor) processMetrics(_ context.Context, md pmetric.Metric
 
 // scrapeServiceAttribute expands the datapoint attributes and search for
 // service name and environment attributes. This is only used for components
-// that only emit attributes on datapoint level.
+// that only emit attributes on datapoint level. This code block contains a lot
+// of repeated code because OTEL metrics type do not have a common interface.
 func (p *awsEntityProcessor) scrapeServiceAttribute(scopeMetric pmetric.ScopeMetricsSlice) (string, string, string) {
 	entityServiceName := EMPTY
 	entityServiceNameSource := EMPTY
@@ -408,6 +416,64 @@ func (p *awsEntityProcessor) scrapeServiceAttribute(scopeMetric pmetric.ScopeMet
 	return entityServiceName, entityEnvironmentName, entityServiceNameSource
 }
 
+// scrapeResourceEntityAttribute expands the datapoint attributes and search for
+// resource entity related attributes. This is only used for components
+// that only emit attributes on datapoint level. This code block contains a lot
+// of repeated code because OTEL metrics type do not have a common interface.
+func (p *awsEntityProcessor) scrapeResourceEntityAttribute(scopeMetric pmetric.ScopeMetricsSlice) string {
+	autoScalingGroup := EMPTY
+	for j := 0; j < scopeMetric.Len(); j++ {
+		metric := scopeMetric.At(j).Metrics()
+		for k := 0; k < metric.Len(); k++ {
+			if autoScalingGroup != EMPTY {
+				return autoScalingGroup
+			}
+			m := metric.At(k)
+			switch m.Type() {
+			case pmetric.MetricTypeGauge:
+				dps := m.Gauge().DataPoints()
+				for l := 0; l < dps.Len(); l++ {
+					if dpAutoScalingGroup, ok := dps.At(l).Attributes().Get(ec2tagger.CWDimensionASG); ok {
+						autoScalingGroup = dpAutoScalingGroup.Str()
+					}
+				}
+			case pmetric.MetricTypeSum:
+				dps := m.Sum().DataPoints()
+				for l := 0; l < dps.Len(); l++ {
+					if dpAutoScalingGroup, ok := dps.At(l).Attributes().Get(ec2tagger.CWDimensionASG); ok {
+						autoScalingGroup = dpAutoScalingGroup.Str()
+					}
+				}
+			case pmetric.MetricTypeHistogram:
+				dps := m.Histogram().DataPoints()
+				for l := 0; l < dps.Len(); l++ {
+					if dpAutoScalingGroup, ok := dps.At(l).Attributes().Get(ec2tagger.CWDimensionASG); ok {
+						autoScalingGroup = dpAutoScalingGroup.Str()
+					}
+				}
+			case pmetric.MetricTypeExponentialHistogram:
+				dps := m.ExponentialHistogram().DataPoints()
+				for l := 0; l < dps.Len(); l++ {
+					if dpAutoScalingGroup, ok := dps.At(l).Attributes().Get(ec2tagger.CWDimensionASG); ok {
+						autoScalingGroup = dpAutoScalingGroup.Str()
+					}
+				}
+			case pmetric.MetricTypeSummary:
+				dps := m.Sum().DataPoints()
+				for l := 0; l < dps.Len(); l++ {
+					if dpAutoScalingGroup, ok := dps.At(l).Attributes().Get(ec2tagger.CWDimensionASG); ok {
+						autoScalingGroup = dpAutoScalingGroup.Str()
+					}
+				}
+			default:
+				p.logger.Debug("Ignore unknown metric type", zap.String("type", m.Type().String()))
+			}
+
+		}
+	}
+	return autoScalingGroup
+}
+
 // getServiceAttributes prioritize service name retrieval based on
 // following attribute priority
 // 1. service.name

diff --git a/plugins/processors/ec2tagger/constants.go b/plugins/processors/ec2tagger/constants.go
@@ -61,7 +61,7 @@ const sampleConfig = `
 
 const (
 	Ec2InstanceTagKeyASG = "aws:autoscaling:groupName"
-	cwDimensionASG       = "AutoScalingGroupName"
+	CWDimensionASG       = "AutoScalingGroupName"
 	mdKeyInstanceId      = "InstanceId"
 	mdKeyImageId         = "ImageId"
 	mdKeyInstanceType    = "InstanceType"

diff --git a/plugins/processors/ec2tagger/ec2tagger.go b/plugins/processors/ec2tagger/ec2tagger.go
@@ -176,7 +176,7 @@ func (t *Tagger) updateTags() error {
 			key := *tag.Key
 			if Ec2InstanceTagKeyASG == key {
 				// rename to match CW dimension as applied by AutoScaling service, not the EC2 tag
-				key = cwDimensionASG
+				key = CWDimensionASG
 			}
 			tags[key] = *tag.Value
 		}
@@ -249,7 +249,7 @@ func (t *Tagger) ec2TagsRetrieved() bool {
 	if t.ec2TagCache != nil {
 		for _, key := range t.EC2InstanceTagKeys {
 			if key == Ec2InstanceTagKeyASG {
-				key = cwDimensionASG
+				key = CWDimensionASG
 			}
 			if key == "*" {
 				continue
@@ -303,7 +303,7 @@ func (t *Tagger) Start(ctx context.Context, host component.Host) error {
 	if !useAllTags && len(t.EC2InstanceTagKeys) > 0 {
 		// if the customer said 'AutoScalingGroupName' (the CW dimension), do what they mean not what they said
 		for i, key := range t.EC2InstanceTagKeys {
-			if cwDimensionASG == key {
+			if CWDimensionASG == key {
 				t.EC2InstanceTagKeys[i] = Ec2InstanceTagKeyASG
 			}
 		}

diff --git a/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml b/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     cumulativetodelta/hostDeltaMetrics:
         exclude:
             match_type: strict
@@ -86,8 +87,8 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_netstat
                 - telegraf_swap
@@ -98,9 +99,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_diskio
     telemetry:

diff --git a/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml b/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     cumulativetodelta/hostDeltaMetrics:
         exclude:
             match_type: strict
@@ -94,8 +95,8 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_ethtool
                 - telegraf_nvidia_smi
@@ -108,9 +109,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_diskio
     telemetry:

diff --git a/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml b/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     ec2tagger:
         ec2_instance_tag_keys:
             - AutoScalingGroupName
@@ -87,8 +88,8 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_win_perf_counters/2073218482
                 - telegraf_win_perf_counters/2039663244

diff --git a/translator/tocwconfig/sampleConfig/amp_config_linux.yaml b/translator/tocwconfig/sampleConfig/amp_config_linux.yaml
@@ -89,6 +89,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     batch/host/amp:
         metadata_cardinality_limit: 1000
         send_batch_max_size: 0
@@ -152,9 +153,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_cpu
     telemetry:

diff --git a/translator/tocwconfig/sampleConfig/basic_config_linux.yaml b/translator/tocwconfig/sampleConfig/basic_config_linux.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     ec2tagger:
         ec2_instance_tag_keys:
             - AutoScalingGroupName
@@ -60,8 +61,8 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_mem
                 - telegraf_disk

diff --git a/translator/tocwconfig/sampleConfig/basic_config_windows.yaml b/translator/tocwconfig/sampleConfig/basic_config_windows.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     ec2tagger:
         ec2_instance_tag_keys:
             - AutoScalingGroupName
@@ -62,8 +63,8 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_win_perf_counters/1492679118
                 - telegraf_win_perf_counters/4283769065

diff --git a/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml b/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml
@@ -105,6 +105,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     awsentity/service/telegraf:
         entity_type: Service
         platform: ec2
@@ -280,9 +281,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_disk
                 - telegraf_swap
@@ -305,10 +306,10 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_diskio
                 - telegraf_net

diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml
@@ -111,6 +111,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     awsentity/service/telegraf:
         entity_type: Service
         platform: ec2
@@ -387,9 +388,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_swap
                 - telegraf_cpu
@@ -412,10 +413,10 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics/cloudwatch
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_net
                 - telegraf_diskio

diff --git a/translator/tocwconfig/sampleConfig/complete_windows_config.yaml b/translator/tocwconfig/sampleConfig/complete_windows_config.yaml
@@ -105,6 +105,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     awsentity/service/telegraf:
         entity_type: Service
         platform: ec2
@@ -267,9 +268,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_win_perf_counters/1063858558
                 - telegraf_nvidia_smi

diff --git a/translator/tocwconfig/sampleConfig/delta_config_linux.yaml b/translator/tocwconfig/sampleConfig/delta_config_linux.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     cumulativetodelta/hostDeltaMetrics:
         exclude:
             match_type: strict
@@ -80,10 +81,10 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics
                 - ec2tagger
                 - transform
+                - awsentity/resource
             receivers:
                 - telegraf_diskio
     telemetry:

diff --git a/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml b/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml
@@ -31,6 +31,7 @@ processors:
     awsentity/resource:
         entity_type: Resource
         platform: ec2
+        scrape_datapoint_attribute: true
     cumulativetodelta/hostDeltaMetrics:
         exclude:
             match_type: ""
@@ -63,9 +64,9 @@ service:
             exporters:
                 - awscloudwatch
             processors:
-                - awsentity/resource
                 - cumulativetodelta/hostDeltaMetrics
                 - ec2tagger
+                - awsentity/resource
             receivers:
                 - telegraf_net
     telemetry: