Backend chart filtering backward compatibility fix (netdata#11002)

backends/backends.c

@@ -26,6 +26,7 @@
 //
 
 const char *global_backend_prefix = "netdata";
+const char *global_backend_send_charts_matching = "*";
 int global_backend_update_every = 10;
 BACKEND_OPTIONS global_backend_options = BACKEND_SOURCE_DATA_AVERAGE | BACKEND_OPTION_SEND_NAMES;
 const char *global_backend_source = NULL;
@@ -517,7 +518,10 @@ void *backends_main(void *ptr) {
     else
         global_backend_options &= ~BACKEND_OPTION_SEND_NAMES;
 
-    charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
+    charts_pattern = simple_pattern_create(
+        global_backend_send_charts_matching = config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"),
+        NULL,
+        SIMPLE_PATTERN_EXACT);
     hosts_pattern  = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE

backends/backends.h

@@ -37,6 +37,7 @@ extern int global_backend_update_every;
 extern BACKEND_OPTIONS global_backend_options;
 extern const char *global_backend_source;
 extern const char *global_backend_prefix;
+extern const char *global_backend_send_charts_matching;
 
 extern void *backends_main(void *ptr);
 BACKEND_TYPE backend_select_type(const char *type);

database/rrd.h

@@ -454,8 +454,8 @@ typedef enum rrdset_flags {
                                               // (the master data set should be the one that has the same family and is not detail)
     RRDSET_FLAG_DEBUG               = 1 << 2, // enables or disables debugging for a chart
     RRDSET_FLAG_OBSOLETE            = 1 << 3, // this is marked by the collector/module as obsolete
-    RRDSET_FLAG_BACKEND_SEND        = 1 << 4, // if set, this chart should be sent to backends
-    RRDSET_FLAG_BACKEND_IGNORE      = 1 << 5, // if set, this chart should not be sent to backends
+    RRDSET_FLAG_EXPORTING_SEND      = 1 << 4, // if set, this chart should be sent to Prometheus web API
+    RRDSET_FLAG_EXPORTING_IGNORE    = 1 << 5, // if set, this chart should not be sent to Prometheus web API
     RRDSET_FLAG_UPSTREAM_SEND       = 1 << 6, // if set, this chart should be sent upstream (streaming)
     RRDSET_FLAG_UPSTREAM_IGNORE     = 1 << 7, // if set, this chart should not be sent upstream (streaming)
     RRDSET_FLAG_UPSTREAM_EXPOSED    = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming)
@@ -468,7 +468,9 @@ typedef enum rrdset_flags {
     // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
     // least rrdset_free_obsolete_time seconds ago.
     RRDSET_FLAG_ARCHIVED            = 1 << 15,
-    RRDSET_FLAG_ACLK                = 1 << 16
+    RRDSET_FLAG_ACLK                = 1 << 16,
+    RRDSET_FLAG_BACKEND_SEND        = 1 << 17, // if set, this chart should be sent to backends
+    RRDSET_FLAG_BACKEND_IGNORE      = 1 << 18 // if set, this chart should not be sent to backends
 } RRDSET_FLAGS;
 
 #ifdef HAVE_C___ATOMIC

database/rrdset.c

@@ -175,6 +175,8 @@ int rrdset_set_name(RRDSET *st, const char *name) {
     if(unlikely(rrdset_index_add_name(host, st) != st))
         error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name);
 
+    rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
+    rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
     rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);
     rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
@@ -858,6 +860,8 @@ RRDSET *rrdset_create_custom(
     rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
     rrdset_flag_clear(st, RRDSET_FLAG_DEBUG);
     rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+    rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
+    rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
     rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);
     rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);

exporting/check_filters.c

@@ -50,15 +50,15 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)
 
     RRDSET_FLAGS *flags = &st->exporting_flags[instance->index];
 
-    if(unlikely(*flags & RRDSET_FLAG_BACKEND_IGNORE))
+    if(unlikely(*flags & RRDSET_FLAG_EXPORTING_IGNORE))
         return 0;
 
-    if(unlikely(!(*flags & RRDSET_FLAG_BACKEND_SEND))) {
+    if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) {
         // we have not checked this chart
         if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name))
-            *flags |= RRDSET_FLAG_BACKEND_SEND;
+            *flags |= RRDSET_FLAG_EXPORTING_SEND;
         else {
-            *flags |= RRDSET_FLAG_BACKEND_IGNORE;
+            *flags |= RRDSET_FLAG_EXPORTING_IGNORE;
             debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname);
             return 0;
         }

exporting/exporting_engine.h

@@ -77,6 +77,8 @@ struct instance_config {
     SIMPLE_PATTERN *charts_pattern;
     SIMPLE_PATTERN *hosts_pattern;
 
+    int initialized;
+
     void *connector_specific_config;
 };
 

exporting/prometheus/prometheus.c

@@ -18,16 +18,16 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
 {
     RRDHOST *host = st->rrdhost;
 
-    if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))
+    if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_IGNORE)))
         return 0;
 
-    if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) {
+    if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_SEND))) {
         // we have not checked this chart
         if (simple_pattern_matches(instance->config.charts_pattern, st->id) ||
             simple_pattern_matches(instance->config.charts_pattern, st->name))
-            rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND);
+            rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_SEND);
         else {
-            rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);
+            rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_IGNORE);
             debug(
                 D_BACKEND,
                 "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.",
@@ -855,7 +855,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
     EXPORTING_OPTIONS exporting_options,
     PROMETHEUS_OUTPUT_OPTIONS output_options)
 {
-    if (unlikely(!prometheus_exporter_instance))
+    if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized))
         return;
 
     prometheus_exporter_instance->before = now_realtime_sec();
@@ -892,7 +892,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
     EXPORTING_OPTIONS exporting_options,
     PROMETHEUS_OUTPUT_OPTIONS output_options)
 {
-    if (unlikely(!prometheus_exporter_instance))
+    if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized))
         return;
 
     prometheus_exporter_instance->before = now_realtime_sec();

exporting/read_config.c

@@ -267,12 +267,16 @@ struct engine *read_exporting_config()
         else
             prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
-        prometheus_exporter_instance->config.charts_pattern =
-            simple_pattern_create(prometheus_config_get("send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
+        prometheus_exporter_instance->config.charts_pattern = simple_pattern_create(
+            prometheus_config_get("send charts matching", global_backend_send_charts_matching),
+            NULL,
+            SIMPLE_PATTERN_EXACT);
         prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create(
             prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
 
         prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_backend_prefix);
+
+        prometheus_exporter_instance->config.initialized = 1;
     }
 
     // TODO: change BACKEND to EXPORTING

exporting/tests/exporting_fixtures.c

@@ -146,6 +146,8 @@ int setup_prometheus(void **state)
     prometheus_exporter_instance->config.charts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
     prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT);
 
+    prometheus_exporter_instance->config.initialized = 1;
+
     return 0;
 }
 

exporting/tests/test_exporting_engine.c

@@ -17,6 +17,7 @@ char log_line[MAX_LOG_LINE + 1];
 BACKEND_OPTIONS global_backend_options = 0;
 const char *global_backend_source = "average";
 const char *global_backend_prefix = "netdata";
+const char *global_backend_send_charts_matching = "*";
 
 void init_connectors_in_tests(struct engine *engine)
 {
@@ -268,7 +269,7 @@ static void test_rrdset_is_exportable(void **state)
     assert_int_equal(__real_rrdset_is_exportable(instance, st), 1);
 
     assert_ptr_not_equal(st->exporting_flags, NULL);
-    assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_SEND);
+    assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_SEND);
 }
 
 static void test_false_rrdset_is_exportable(void **state)
@@ -285,7 +286,7 @@ static void test_false_rrdset_is_exportable(void **state)
     assert_int_equal(__real_rrdset_is_exportable(instance, st), 0);
 
     assert_ptr_not_equal(st->exporting_flags, NULL);
-    assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_IGNORE);
+    assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_IGNORE);
 }
 
 static void test_exporting_calculate_value_from_stored_data(void **state)
@@ -993,9 +994,9 @@ static void test_can_send_rrdset(void **state)
 
     assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 1);
 
-    rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE);
+    rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE);
     assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0);
-    rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE);
+    rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE);
 
     // TODO: test with a denying simple pattern