out_cloudwatch: add account ID support for CloudWatch entity (#4)

* Add account ID support for CloudWatch entity * Send entity only when account ID is defined * Enable account and instance ID for entity without customer interaction * Add comment to explain entity drops without account ID
amazon-contributing · Nov 1, 2024 · 8949182 · 8949182
1 parent dca1459
commit 8949182
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 11 deletions.
diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c
@@ -467,6 +467,32 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx)
         ctx->new_keys++;
     }
 
+    if (ctx->enable_entity) {
+        if (!ctx->account_id) {
+            ret = get_metadata_by_key(ctx, FLB_FILTER_AWS_IMDS_ACCOUNT_ID_PATH,
+                                  &ctx->account_id, &ctx->account_id_len,
+                                  "accountId");
+            if (ret < 0) {
+                return -1;
+            }
+            ctx->new_keys++;
+        } else {
+            ctx->new_keys++;
+        }
+
+        if (!ctx->instance_id) {
+            ret = get_metadata(ctx, FLB_FILTER_AWS_IMDS_INSTANCE_ID_PATH,
+                   &ctx->instance_id, &ctx->instance_id_len);
+
+            if (ret < 0) {
+                return -1;
+            }
+            ctx->new_keys++;
+        } else {
+            ctx->new_keys++;
+        }
+    }
+
     ctx->metadata_retrieved = FLB_TRUE;
     return 0;
 }
@@ -558,22 +584,14 @@ static int cb_aws_filter(const void *data, size_t bytes,
                                   ctx->availability_zone_len);
         }
 
-        if (ctx->instance_id_include && !ctx->enable_entity) {
+        if (ctx->instance_id_include) {
             msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN);
             msgpack_pack_str_body(&tmp_pck,
                                   FLB_FILTER_AWS_INSTANCE_ID_KEY,
                                   FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN);
             msgpack_pack_str(&tmp_pck, ctx->instance_id_len);
             msgpack_pack_str_body(&tmp_pck,
                                   ctx->instance_id, ctx->instance_id_len);
-        } else if (ctx->instance_id_include && ctx->enable_entity) {
-            msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN);
-            msgpack_pack_str_body(&tmp_pck,
-                                  FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY,
-                                  FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN);
-            msgpack_pack_str(&tmp_pck, ctx->instance_id_len);
-            msgpack_pack_str_body(&tmp_pck,
-                                  ctx->instance_id, ctx->instance_id_len);
         }
 
         if (ctx->instance_type_include) {
@@ -635,6 +653,26 @@ static int cb_aws_filter(const void *data, size_t bytes,
             msgpack_pack_str_body(&tmp_pck,
                                   ctx->hostname, ctx->hostname_len);
         }
+
+        if (ctx->enable_entity && ctx->instance_id != NULL && ctx->account_id != NULL) {
+            // Pack instance ID with entity prefix for further processing
+            msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN);
+            msgpack_pack_str_body(&tmp_pck,
+                                  FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY,
+                                  FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN);
+            msgpack_pack_str(&tmp_pck, ctx->instance_id_len);
+            msgpack_pack_str_body(&tmp_pck,
+                                  ctx->instance_id, ctx->instance_id_len);
+
+            // Pack account ID with entity prefix for further processing
+            msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN);
+            msgpack_pack_str_body(&tmp_pck,
+                                  FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY,
+                                  FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN);
+            msgpack_pack_str(&tmp_pck, ctx->account_id_len);
+            msgpack_pack_str_body(&tmp_pck,
+                                  ctx->account_id, ctx->account_id_len);
+        }
     }
     msgpack_unpacked_destroy(&result);
 

diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h
@@ -63,6 +63,8 @@
 #define FLB_FILTER_AWS_AMI_ID_KEY_LEN                     6
 #define FLB_FILTER_AWS_ACCOUNT_ID_KEY                     "account_id"
 #define FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN                 10
+#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY              "aws_entity_account_id"
+#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN          21
 #define FLB_FILTER_AWS_HOSTNAME_KEY                       "hostname"
 #define FLB_FILTER_AWS_HOSTNAME_KEY_LEN                   8
 

diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c
@@ -221,6 +221,14 @@ static int entity_add_key_attributes(struct flb_cloudwatch *ctx, struct cw_flush
             goto error;
         }
     }
+    if(stream->entity->key_attributes->account_id != NULL && strlen(stream->entity->key_attributes->account_id) != 0) {
+        if (!snprintf(ts,KEY_ATTRIBUTES_MAX_LEN, ",%s%s%s","\"AwsAccountId\":\"",stream->entity->key_attributes->account_id,"\"")) {
+            goto error;
+        }
+        if (!try_to_write(buf->out_buf, offset, buf->out_buf_size,ts,0)) {
+            goto error;
+        }
+    }
     if (!try_to_write(buf->out_buf, offset, buf->out_buf_size,
               "},", 2)) {
         goto error;
@@ -359,8 +367,9 @@ static int init_put_payload(struct flb_cloudwatch *ctx, struct cw_flush *buf,
         goto error;
     }
     // If we are missing the service name, the entity will get rejected by the frontend anyway
-    // so do not emit entity unless service name is filled
-    if(ctx->add_entity && stream->entity != NULL && stream->entity->key_attributes != NULL && stream->entity->key_attributes->name != NULL) {
+    // so do not emit entity unless service name is filled. If we are missing account ID
+    // it is considered not having sufficient information for entity therefore we should drop the entity.
+    if(ctx->add_entity && stream->entity != NULL && stream->entity->key_attributes != NULL && stream->entity->key_attributes->name != NULL && stream->entity->key_attributes->account_id != NULL) {
         if (!try_to_write(buf->out_buf, offset, buf->out_buf_size,
                       "\"entity\":{", 10)) {
             goto error;
@@ -1109,6 +1118,14 @@ void parse_entity(struct flb_cloudwatch *ctx, entity *entity, msgpack_object map
             }
             entity->attributes->instance_id = flb_strndup(val.via.str.ptr, val.via.str.size);
         }
+        if(strncmp(key.via.str.ptr, "aws_entity_account_id",key.via.str.size ) == 0 ) {
+            if(entity->key_attributes->account_id == NULL) {
+                entity->root_filter_count++;
+            } else {
+                flb_free(entity->key_attributes->account_id);
+            }
+            entity->key_attributes->account_id = flb_strndup(val.via.str.ptr, val.via.str.size);
+        }
     }
     if(entity->key_attributes->name == NULL && entity->attributes->name_source == NULL &&entity->attributes->workload != NULL) {
         entity->key_attributes->name = flb_strndup(entity->attributes->workload, strlen(entity->attributes->workload));

diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/plugins/out_cloudwatch_logs/cloudwatch_logs.c
@@ -515,6 +515,7 @@ void entity_destroy(entity *entity) {
         flb_free(entity->key_attributes->environment);
         flb_free(entity->key_attributes->name);
         flb_free(entity->key_attributes->type);
+        flb_free(entity->key_attributes->account_id);
         flb_free(entity->key_attributes);
     }
     flb_free(entity);

diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.h b/plugins/out_cloudwatch_logs/cloudwatch_logs.h
@@ -48,6 +48,7 @@ typedef struct entity_key_attributes {
     char *type;
     char *name;
     char *environment;
+    char *account_id;
 }entity_key_attributes;
 
 /* Attributes used for CloudWatch Entity object