out_stackdriver: add project_id_key override (fluent#8209)

Closes fluent#8195 Signed-off-by: ryanohnemus <[email protected]> Signed-off-by: ahspw <[email protected]>
shaerpour · Jan 16, 2024 · 5c5c059 · 5c5c059
1 parent 8acc497
commit 5c5c059
Show file tree

Hide file tree

Showing 4 changed files with 142 additions and 2 deletions.
diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c
@@ -1513,6 +1513,7 @@ static int pack_json_payload(int insert_id_extracted,
     {
         monitored_resource_key,
         local_resource_id_key,
+        ctx->project_id_key,
         ctx->labels_key,
         ctx->severity_key,
         ctx->trace_key,
@@ -1684,6 +1685,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
     flb_sds_t out_buf;
     struct flb_mp_map_header mh;
 
+    /* Parameters for project_id_key */
+    int project_id_extracted = FLB_FALSE;
+    flb_sds_t project_id_key;
+
     /* Parameters for severity */
     int severity_extracted = FLB_FALSE;
     severity_t severity;
@@ -2208,6 +2213,13 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
             entry_size += 1;
         }
 
+        /* Extract project id */
+        project_id_extracted = FLB_FALSE;
+        if (ctx->project_id_key
+            && get_string(&project_id_key, obj, ctx->project_id_key) == 0) {
+            project_id_extracted = FLB_TRUE;
+        }
+
         /* Extract log name */
         log_name_extracted = FLB_FALSE;
         if (ctx->log_name_key
@@ -2413,10 +2425,16 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
             new_log_name = log_name;
         }
 
-        /* logName */
-        len = snprintf(path, sizeof(path) - 1,
+        if (project_id_extracted == FLB_TRUE) {
+            len = snprintf(path, sizeof(path) - 1,
+                       "projects/%s/logs/%s", project_id_key, new_log_name);
+            flb_sds_destroy(project_id_key);
+        } else {
+            len = snprintf(path, sizeof(path) - 1,
                        "projects/%s/logs/%s", ctx->export_to_project_id, new_log_name);
+        }
 
+        /* logName */
         if (log_name_extracted == FLB_TRUE) {
             flb_sds_destroy(log_name);
         }
@@ -2978,6 +2996,11 @@ static struct flb_config_map config_map[] = {
       0, FLB_TRUE, offsetof(struct flb_stackdriver, export_to_project_id),
       "Export to project id"
     },
+    {
+      FLB_CONFIG_MAP_STR, "project_id_key", DEFAULT_PROJECT_ID_KEY,
+      0, FLB_TRUE, offsetof(struct flb_stackdriver, project_id_key),
+      "Set the gcp project id key"
+    },
     {
       FLB_CONFIG_MAP_STR, "resource", FLB_SDS_RESOURCE_TYPE,
       0, FLB_TRUE, offsetof(struct flb_stackdriver, resource),

diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h
@@ -49,6 +49,7 @@
 #define MONITORED_RESOURCE_KEY "logging.googleapis.com/monitored_resource"
 #define LOCAL_RESOURCE_ID_KEY "logging.googleapis.com/local_resource_id"
 #define DEFAULT_LABELS_KEY "logging.googleapis.com/labels"
+#define DEFAULT_PROJECT_ID_KEY "logging.googleapis.com/projectId"
 #define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity"
 #define DEFAULT_TRACE_KEY "logging.googleapis.com/trace"
 #define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId"
@@ -170,6 +171,7 @@ struct flb_stackdriver {
 
     /* other */
     flb_sds_t export_to_project_id;
+    flb_sds_t project_id_key;
     flb_sds_t resource;
     flb_sds_t severity_key;
     flb_sds_t trace_key;

diff --git a/tests/runtime/data/stackdriver/stackdriver_test_log_name.h b/tests/runtime/data/stackdriver/stackdriver_test_log_name.h
@@ -8,3 +8,15 @@
 	"1591111124," \
 	"{"	\
 	"}]"
+
+#define LOG_NAME_PROJECT_ID_OVERRIDE "[" \
+	"1591111124," \
+	"{"	\
+		"\"test_project_key\": \"fluent-bit-test-project-2\"" \
+	"}]"
+
+#define LOG_NAME_PROJECT_ID_NO_OVERRIDE	"[" \
+	"1591111124," \
+	"{"	\
+	    "\"logging.googleapis.com/projectId\": \"fluent-bit-test-project-2\"" \
+	"}]"
diff --git a/tests/runtime/out_stackdriver.c b/tests/runtime/out_stackdriver.c
@@ -841,6 +841,24 @@ static void cb_check_log_name_no_override(void *ctx, int ffd,
     flb_sds_destroy(res_data);
 }
 
+static void cb_check_project_key_override(void *ctx, int ffd,
+                                          int res_ret, void *res_data, size_t res_size,
+                                          void *data)
+{
+    int ret;
+
+    /* logName in the entries is created using the value under log_name_key */
+    ret = mp_kv_cmp(
+        res_data, res_size, "$entries[0]['logName']", "projects/fluent-bit-test-project-2/logs/test");
+    TEST_CHECK(ret == FLB_TRUE);
+
+    /* log_name_key has been removed from jsonPayload */
+    ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['test_project_key']");
+    TEST_CHECK(ret == FLB_FALSE);
+
+    flb_sds_destroy(res_data);
+}
+
 static void cb_check_k8s_node_resource(void *ctx, int ffd,
                                        int res_ret, void *res_data, size_t res_size,
                                        void *data)
@@ -2656,6 +2674,87 @@ void flb_test_set_metadata_server()
     flb_destroy(ctx);
 }
 
+void flb_test_project_id_override()
+{
+    int ret;
+    int size = sizeof(LOG_NAME_PROJECT_ID_OVERRIDE) - 1;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+
+    /* Create context, flush every second (some checks omitted here) */
+    ctx = flb_create();
+    flb_service_set(ctx, "flush", "1", "grace", "1", NULL);
+
+    /* Lib input mode */
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    /* Stackdriver output */
+    out_ffd = flb_output(ctx, (char *) "stackdriver", NULL);
+    flb_output_set(ctx, out_ffd,
+                   "match", "test",
+                   "resource", "gce_instance",                   
+                   "project_id_key", "test_project_key",
+                   NULL);
+
+    /* Enable test mode */
+    ret = flb_output_set_test(ctx, out_ffd, "formatter",
+                              cb_check_project_key_override,
+                              NULL, NULL);
+
+    /* Start */
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    /* Ingest data sample */
+    flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_OVERRIDE, size);
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
+void flb_test_project_id_no_override()
+{
+    int ret;
+    int size = sizeof(LOG_NAME_PROJECT_ID_NO_OVERRIDE) - 1;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+
+    /* Create context, flush every second (some checks omitted here) */
+    ctx = flb_create();
+    flb_service_set(ctx, "flush", "1", "grace", "1", NULL);
+
+    /* Lib input mode */
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    /* Stackdriver output */
+    out_ffd = flb_output(ctx, (char *) "stackdriver", NULL);
+    flb_output_set(ctx, out_ffd,
+                   "match", "test",
+                   "resource", "gce_instance",                   
+                   NULL);
+
+    /* Enable test mode */
+    ret = flb_output_set_test(ctx, out_ffd, "formatter",
+                              cb_check_project_key_override,
+                              NULL, NULL);
+
+    /* Start */
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    /* Ingest data sample */
+    flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_NO_OVERRIDE, size);
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
 void flb_test_log_name_override()
 {
     int ret;
@@ -6150,6 +6249,10 @@ TEST_LIST = {
     /* test metadata server */
     {"set_metadata_server", flb_test_set_metadata_server},
 
+    /* test project key */
+    {"project_key_override", flb_test_project_id_override},
+    {"project_key_no_override", flb_test_project_id_no_override},
+
     /* test log name */
     {"log_name_override", flb_test_log_name_override},
     {"log_name_no_override", flb_test_log_name_no_override},