From bf3da780bd647b3870901d798be2af22fc5fc29a Mon Sep 17 00:00:00 2001 From: Andre Silva Date: Tue, 16 May 2023 04:50:03 -0300 Subject: [PATCH] out_stackdriver: process spanId special field This PR adds support for processing the logging.googleapis.com/spanId special field from the jsonPayload Note: This is a backport of PR #7345 Signed-off-by: Andre Silva --- plugins/out_stackdriver/stackdriver.c | 29 ++++++++- plugins/out_stackdriver/stackdriver.h | 2 + .../stackdriver/stackdriver_test_span_id.h | 6 ++ tests/runtime/out_stackdriver.c | 63 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/runtime/data/stackdriver/stackdriver_test_span_id.h diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index 338484440a8..67247694988 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -1417,6 +1417,7 @@ static int pack_json_payload(int insert_id_extracted, ctx->labels_key, ctx->severity_key, ctx->trace_key, + ctx->span_id_key, ctx->log_name_key, stream /* more special fields are required to be added, but, if this grows with more @@ -1594,6 +1595,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, char stackdriver_trace[PATH_MAX]; const char *new_trace; + /* Parameters for span id */ + int span_id_extracted = FLB_FALSE; + flb_sds_t span_id; + /* Parameters for log name */ int log_name_extracted = FLB_FALSE; flb_sds_t log_name = NULL; @@ -2034,6 +2039,7 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, * "logName": "...", * "jsonPayload": {...}, * "timestamp": "...", + * "spanId": "...", * "trace": "..." * } */ @@ -2055,6 +2061,14 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, entry_size += 1; } + /* Extract span id */ + span_id_extracted = FLB_FALSE; + if (ctx->span_id_key + && get_string(&span_id, obj, ctx->span_id_key) == 0) { + span_id_extracted = FLB_TRUE; + entry_size += 1; + } + /* Extract log name */ log_name_extracted = FLB_FALSE; if (ctx->log_name_key @@ -2170,6 +2184,14 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, flb_sds_destroy(trace); } + /* Add spanId field into the log entry */ + if (span_id_extracted == FLB_TRUE) { + msgpack_pack_str_with_body(&mp_pck, "spanId", 6); + len = flb_sds_len(span_id); + msgpack_pack_str_with_body(&mp_pck, span_id, len); + flb_sds_destroy(span_id); + } + /* Add insertId field into the log entry */ if (insert_id_extracted == FLB_TRUE) { msgpack_pack_str(&mp_pck, 8); @@ -2555,13 +2577,18 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "autoformat_stackdriver_trace", "false", 0, FLB_TRUE, offsetof(struct flb_stackdriver, autoformat_stackdriver_trace), - "Autoformat the stacrdriver trace" + "Autoformat the stackdriver trace" }, { FLB_CONFIG_MAP_STR, "trace_key", DEFAULT_TRACE_KEY, 0, FLB_TRUE, offsetof(struct flb_stackdriver, trace_key), "Set the trace key" }, + { + FLB_CONFIG_MAP_STR, "span_id_key", DEFAULT_SPAN_ID_KEY, + 0, FLB_TRUE, offsetof(struct flb_stackdriver, span_id_key), + "Set the span id key" + }, { FLB_CONFIG_MAP_STR, "log_name_key", DEFAULT_LOG_NAME_KEY, 0, FLB_TRUE, offsetof(struct flb_stackdriver, log_name_key), diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h index 8983544aa11..f450d093899 100644 --- a/plugins/out_stackdriver/stackdriver.h +++ b/plugins/out_stackdriver/stackdriver.h @@ -51,6 +51,7 @@ #define DEFAULT_LABELS_KEY "logging.googleapis.com/labels" #define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity" #define DEFAULT_TRACE_KEY "logging.googleapis.com/trace" +#define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId" #define DEFAULT_LOG_NAME_KEY "logging.googleapis.com/logName" #define DEFAULT_INSERT_ID_KEY "logging.googleapis.com/insertId" #define SOURCELOCATION_FIELD_IN_JSON "logging.googleapis.com/sourceLocation" @@ -163,6 +164,7 @@ struct flb_stackdriver { flb_sds_t resource; flb_sds_t severity_key; flb_sds_t trace_key; + flb_sds_t span_id_key; flb_sds_t log_name_key; flb_sds_t http_request_key; int http_request_key_size; diff --git a/tests/runtime/data/stackdriver/stackdriver_test_span_id.h b/tests/runtime/data/stackdriver/stackdriver_test_span_id.h new file mode 100644 index 00000000000..5b7b823e221 --- /dev/null +++ b/tests/runtime/data/stackdriver/stackdriver_test_span_id.h @@ -0,0 +1,6 @@ +#define SPAN_ID_COMMON_CASE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/spanId\": \"000000000000004a\"" \ + "}]" + \ No newline at end of file diff --git a/tests/runtime/out_stackdriver.c b/tests/runtime/out_stackdriver.c index 67992b403d2..e45e2a957ee 100644 --- a/tests/runtime/out_stackdriver.c +++ b/tests/runtime/out_stackdriver.c @@ -35,6 +35,7 @@ #include "data/stackdriver/stackdriver_test_k8s_resource.h" #include "data/stackdriver/stackdriver_test_labels.h" #include "data/stackdriver/stackdriver_test_trace.h" +#include "data/stackdriver/stackdriver_test_span_id.h" #include "data/stackdriver/stackdriver_test_log_name.h" #include "data/stackdriver/stackdriver_test_resource_labels.h" #include "data/stackdriver/stackdriver_test_insert_id.h" @@ -753,6 +754,24 @@ static void cb_check_trace_stackdriver_autoformat(void *ctx, int ffd, flb_sds_destroy(res_data); } +static void cb_check_span_id(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* span id in the entries */ + ret = mp_kv_cmp(res_data, res_size, "$entries[0]['spanId']", "000000000000004a"); + TEST_CHECK(ret == FLB_TRUE); + + /* span id has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, + "$entries[0]['jsonPayload']['logging.googleapis.com/spanId']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + static void cb_check_log_name_override(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, void *data) @@ -2439,6 +2458,47 @@ void flb_test_trace_stackdriver_autoformat() flb_destroy(ctx); } + +void flb_test_span_id() +{ + int ret; + int size = sizeof(SPAN_ID_COMMON_CASE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_span_id, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) SPAN_ID_COMMON_CASE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + void flb_test_set_metadata_server() { int ret; @@ -5964,6 +6024,9 @@ TEST_LIST = { {"trace_no_autoformat", flb_test_trace_no_autoformat}, {"trace_stackdriver_autoformat", flb_test_trace_stackdriver_autoformat}, + /* test span id */ + {"span_id", flb_test_span_id}, + /* test metadata server */ {"set_metadata_server", flb_test_set_metadata_server},