custom_calyptia: switch to YAML for Fleet config (#9698)

* custom_calyptia: switch to YAML for Fleet config Signed-off-by: Patrick Stephens <[email protected]> * in_calyptia_fleet: code review comments Signed-off-by: Patrick Stephens <[email protected]> --------- Signed-off-by: Patrick Stephens <[email protected]>
fluent · Jan 10, 2025 · 417d129 · 417d129
1 parent 14ca011
commit 417d129
Show file tree

Hide file tree

Showing 12 changed files with 411 additions and 98 deletions.
diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
@@ -78,6 +78,7 @@ jobs:
           CC: ${{ matrix.compiler }}
           CXX: ${{ matrix.compiler }}
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-macos-unit-tests:
     # We chain this after Linux one as there are costs and restrictions associated
@@ -111,6 +112,7 @@ jobs:
           CC: gcc
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-aarch64-unit-tests:
     # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
@@ -167,6 +169,7 @@ jobs:
         env:
           CC: ${{ matrix.config.compiler }}
           CXX: ${{ matrix.config.compiler }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-qemu-ubuntu-unit-tests:
     # We chain this after Linux one as there are CPU time costs for QEMU emulation
@@ -217,7 +220,6 @@ jobs:
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
 
-
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()

diff --git a/include/fluent-bit/calyptia/calyptia_constants.h b/include/fluent-bit/calyptia/calyptia_constants.h
@@ -36,8 +36,10 @@
 #define CALYPTIA_ENDPOINT_TRACE   "/v1/traces/%s"
 
 #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets"
-#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml"
 #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files"
+#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true"
 
 /* Storage */
 #define CALYPTIA_SESSION_FILE     "session.CALYPTIA"

diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c
@@ -232,6 +232,7 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *
     flb_input_set_property(fleet, "host", ctx->cloud_host);
     flb_input_set_property(fleet, "port", ctx->cloud_port);
     flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir);
+    flb_input_set_property(fleet, "fleet_config_legacy_format", ctx->fleet_config_legacy_format == 1 ? "on" : "off");
 
     /* Set TLS properties */
     flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off");
@@ -617,7 +618,7 @@ static int cb_calyptia_init(struct flb_custom_instance *ins,
             flb_free(ctx);
             return -1;
         }
-        ctx->machine_id_auto_configured = 1;
+        ctx->machine_id_auto_configured = FLB_TRUE;
     }
 
     /* input collector */
@@ -774,6 +775,11 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct calyptia, register_retry_on_flush),
      "Retry agent registration on flush if failed on init."
     },
+    {
+     FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true",
+     0, FLB_TRUE, offsetof(struct calyptia, fleet_config_legacy_format),
+     "If set, use legacy (TOML) format for configuration files."
+    },
     /* EOF */
     {0}
 };

diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h
@@ -20,8 +20,6 @@
 #ifndef FLB_CALYPTIA_H
 #define FLB_CALYPTIA_H
 
-#include <stdbool.h>
-
 struct calyptia {
     /* config map options */
     flb_sds_t api_key;
@@ -55,7 +53,8 @@ struct calyptia {
     flb_sds_t fleet_max_http_buffer_size;
     flb_sds_t fleet_interval_sec;
     flb_sds_t fleet_interval_nsec;
-    bool register_retry_on_flush;   /* retry registration on flush if failed */
+    int register_retry_on_flush;          /* retry registration on flush if failed */
+    int fleet_config_legacy_format;       /* Fleet config format to use: INI (true) or YAML (false) */
 };
 
 int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet);