out_stackdriver: process spanId special field

This PR adds support for processing the logging.googleapis.com/spanId special field from the jsonPayload Note: This is a backport of PR #7345 Signed-off-by: Andre Silva <[email protected]>
fluent · May 16, 2023 · bf3da78 · bf3da78
1 parent 5eb920f
commit bf3da78
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 1 deletion.
diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c
@@ -1417,6 +1417,7 @@ static int pack_json_payload(int insert_id_extracted,
         ctx->labels_key,
         ctx->severity_key,
         ctx->trace_key,
+        ctx->span_id_key,
         ctx->log_name_key,
         stream
         /* more special fields are required to be added, but, if this grows with more
@@ -1594,6 +1595,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
     char stackdriver_trace[PATH_MAX];
     const char *new_trace;
 
+    /* Parameters for span id */
+    int span_id_extracted = FLB_FALSE;
+    flb_sds_t span_id;
+
     /* Parameters for log name */
     int log_name_extracted = FLB_FALSE;
     flb_sds_t log_name = NULL;
@@ -2034,6 +2039,7 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
          *  "logName": "...",
          *  "jsonPayload": {...},
          *  "timestamp": "...",
+         *  "spanId": "...",
          *  "trace": "..."
          * }
          */
@@ -2055,6 +2061,14 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
             entry_size += 1;
         }
 
+        /* Extract span id */
+        span_id_extracted = FLB_FALSE;
+        if (ctx->span_id_key
+            && get_string(&span_id, obj, ctx->span_id_key) == 0) {
+            span_id_extracted = FLB_TRUE;
+            entry_size += 1;
+        }
+
         /* Extract log name */
         log_name_extracted = FLB_FALSE;
         if (ctx->log_name_key
@@ -2170,6 +2184,14 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
             flb_sds_destroy(trace);
         }
 
+        /* Add spanId field into the log entry */
+        if (span_id_extracted == FLB_TRUE) {
+            msgpack_pack_str_with_body(&mp_pck, "spanId", 6);
+            len = flb_sds_len(span_id);
+            msgpack_pack_str_with_body(&mp_pck, span_id, len);
+            flb_sds_destroy(span_id);
+        }
+
         /* Add insertId field into the log entry */
         if (insert_id_extracted == FLB_TRUE) {
             msgpack_pack_str(&mp_pck, 8);
@@ -2555,13 +2577,18 @@ static struct flb_config_map config_map[] = {
     {
       FLB_CONFIG_MAP_BOOL, "autoformat_stackdriver_trace", "false",
       0, FLB_TRUE, offsetof(struct flb_stackdriver, autoformat_stackdriver_trace),
-      "Autoformat the stacrdriver trace"
+      "Autoformat the stackdriver trace"
     },
     {
       FLB_CONFIG_MAP_STR, "trace_key", DEFAULT_TRACE_KEY,
       0, FLB_TRUE, offsetof(struct flb_stackdriver, trace_key),
       "Set the trace key"
     },
+    {
+      FLB_CONFIG_MAP_STR, "span_id_key", DEFAULT_SPAN_ID_KEY,
+      0, FLB_TRUE, offsetof(struct flb_stackdriver, span_id_key),
+      "Set the span id key"
+    },
     {
       FLB_CONFIG_MAP_STR, "log_name_key", DEFAULT_LOG_NAME_KEY,
       0, FLB_TRUE, offsetof(struct flb_stackdriver, log_name_key),

diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h
@@ -51,6 +51,7 @@
 #define DEFAULT_LABELS_KEY "logging.googleapis.com/labels"
 #define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity"
 #define DEFAULT_TRACE_KEY "logging.googleapis.com/trace"
+#define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId"
 #define DEFAULT_LOG_NAME_KEY "logging.googleapis.com/logName"
 #define DEFAULT_INSERT_ID_KEY "logging.googleapis.com/insertId"
 #define SOURCELOCATION_FIELD_IN_JSON "logging.googleapis.com/sourceLocation"
@@ -163,6 +164,7 @@ struct flb_stackdriver {
     flb_sds_t resource;
     flb_sds_t severity_key;
     flb_sds_t trace_key;
+    flb_sds_t span_id_key;
     flb_sds_t log_name_key;
     flb_sds_t http_request_key;
     int http_request_key_size;

diff --git a/tests/runtime/data/stackdriver/stackdriver_test_span_id.h b/tests/runtime/data/stackdriver/stackdriver_test_span_id.h
@@ -0,0 +1,6 @@
+#define SPAN_ID_COMMON_CASE	"[" \
+	"1591111124," \
+	"{"	\
+      "\"logging.googleapis.com/spanId\": \"000000000000004a\"" \
+	"}]"
+
diff --git a/tests/runtime/out_stackdriver.c b/tests/runtime/out_stackdriver.c
@@ -35,6 +35,7 @@
 #include "data/stackdriver/stackdriver_test_k8s_resource.h"
 #include "data/stackdriver/stackdriver_test_labels.h"
 #include "data/stackdriver/stackdriver_test_trace.h"
+#include "data/stackdriver/stackdriver_test_span_id.h"
 #include "data/stackdriver/stackdriver_test_log_name.h"
 #include "data/stackdriver/stackdriver_test_resource_labels.h"
 #include "data/stackdriver/stackdriver_test_insert_id.h"
@@ -753,6 +754,24 @@ static void cb_check_trace_stackdriver_autoformat(void *ctx, int ffd,
     flb_sds_destroy(res_data);
 }
 
+static void cb_check_span_id(void *ctx, int ffd,
+                                         int res_ret, void *res_data, size_t res_size,
+                                         void *data)
+{
+    int ret;
+
+    /* span id in the entries */
+    ret = mp_kv_cmp(res_data, res_size, "$entries[0]['spanId']", "000000000000004a");
+    TEST_CHECK(ret == FLB_TRUE);
+
+    /* span id has been removed from jsonPayload */
+    ret = mp_kv_exists(res_data, res_size,
+                       "$entries[0]['jsonPayload']['logging.googleapis.com/spanId']");
+    TEST_CHECK(ret == FLB_FALSE);
+
+    flb_sds_destroy(res_data);
+}
+
 static void cb_check_log_name_override(void *ctx, int ffd,
                                        int res_ret, void *res_data, size_t res_size,
                                        void *data)
@@ -2439,6 +2458,47 @@ void flb_test_trace_stackdriver_autoformat()
     flb_destroy(ctx);
 }
 
+
+void flb_test_span_id()
+{
+    int ret;
+    int size = sizeof(SPAN_ID_COMMON_CASE) - 1;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+
+    /* Create context, flush every second (some checks omitted here) */
+    ctx = flb_create();
+    flb_service_set(ctx, "flush", "1", "grace", "1", NULL);
+
+    /* Lib input mode */
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    /* Stackdriver output */
+    out_ffd = flb_output(ctx, (char *) "stackdriver", NULL);
+    flb_output_set(ctx, out_ffd,
+                   "match", "test",
+                   "resource", "gce_instance",
+                   NULL);
+
+    /* Enable test mode */
+    ret = flb_output_set_test(ctx, out_ffd, "formatter",
+                              cb_check_span_id,
+                              NULL, NULL);
+
+    /* Start */
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    /* Ingest data sample */
+    flb_lib_push(ctx, in_ffd, (char *) SPAN_ID_COMMON_CASE, size);
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
 void flb_test_set_metadata_server()
 {
     int ret;
@@ -5964,6 +6024,9 @@ TEST_LIST = {
     {"trace_no_autoformat", flb_test_trace_no_autoformat},
     {"trace_stackdriver_autoformat", flb_test_trace_stackdriver_autoformat},
 
+    /* test span id */
+    {"span_id", flb_test_span_id},
+
     /* test metadata server */
     {"set_metadata_server", flb_test_set_metadata_server},