From 60975571314fa30a5f92a7946fac05e28cc8e0db Mon Sep 17 00:00:00 2001 From: Zhihong Lin Date: Wed, 30 Oct 2024 12:32:25 -0400 Subject: [PATCH 1/4] Add account ID support for CloudWatch entity --- plugins/filter_aws/aws.c | 10 +++++++++- plugins/filter_aws/aws.h | 2 ++ plugins/out_cloudwatch_logs/cloudwatch_api.c | 16 ++++++++++++++++ plugins/out_cloudwatch_logs/cloudwatch_logs.c | 1 + plugins/out_cloudwatch_logs/cloudwatch_logs.h | 1 + 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index 6a7c6b5108a..b99aac83d99 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -616,7 +616,7 @@ static int cb_aws_filter(const void *data, size_t bytes, ctx->ami_id, ctx->ami_id_len); } - if (ctx->account_id_include) { + if (ctx->account_id_include && !ctx->enable_entity) { msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN); msgpack_pack_str_body(&tmp_pck, FLB_FILTER_AWS_ACCOUNT_ID_KEY, @@ -624,6 +624,14 @@ static int cb_aws_filter(const void *data, size_t bytes, msgpack_pack_str(&tmp_pck, ctx->account_id_len); msgpack_pack_str_body(&tmp_pck, ctx->account_id, ctx->account_id_len); + } else if (ctx->account_id_include && ctx->enable_entity) { + msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); + msgpack_pack_str_body(&tmp_pck, + FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY, + FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); + msgpack_pack_str(&tmp_pck, ctx->account_id_len); + msgpack_pack_str_body(&tmp_pck, + ctx->account_id, ctx->account_id_len); } if (ctx->hostname_include) { diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index 65d8a19c9ac..332f0f3fc9a 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -63,6 +63,8 @@ #define FLB_FILTER_AWS_AMI_ID_KEY_LEN 6 #define FLB_FILTER_AWS_ACCOUNT_ID_KEY "account_id" #define FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN 10 +#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY "aws_entity_account_id" +#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN 21 #define FLB_FILTER_AWS_HOSTNAME_KEY "hostname" #define FLB_FILTER_AWS_HOSTNAME_KEY_LEN 8 diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index d351d0c64e3..8bf9a6f6469 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -221,6 +221,14 @@ static int entity_add_key_attributes(struct flb_cloudwatch *ctx, struct cw_flush goto error; } } + if(stream->entity->key_attributes->account_id != NULL && strlen(stream->entity->key_attributes->account_id) != 0) { + if (!snprintf(ts,KEY_ATTRIBUTES_MAX_LEN, ",%s%s%s","\"AwsAccountId\":\"",stream->entity->key_attributes->account_id,"\"")) { + goto error; + } + if (!try_to_write(buf->out_buf, offset, buf->out_buf_size,ts,0)) { + goto error; + } + } if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, "},", 2)) { goto error; @@ -1109,6 +1117,14 @@ void parse_entity(struct flb_cloudwatch *ctx, entity *entity, msgpack_object map } entity->attributes->instance_id = flb_strndup(val.via.str.ptr, val.via.str.size); } + if(strncmp(key.via.str.ptr, "aws_entity_account_id",key.via.str.size ) == 0 ) { + if(entity->key_attributes->account_id == NULL) { + entity->root_filter_count++; + } else { + flb_free(entity->key_attributes->account_id); + } + entity->key_attributes->account_id = flb_strndup(val.via.str.ptr, val.via.str.size); + } } if(entity->key_attributes->name == NULL && entity->attributes->name_source == NULL &&entity->attributes->workload != NULL) { entity->key_attributes->name = flb_strndup(entity->attributes->workload, strlen(entity->attributes->workload)); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/plugins/out_cloudwatch_logs/cloudwatch_logs.c index c2ca0720f7e..0b85be26e01 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_logs.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_logs.c @@ -515,6 +515,7 @@ void entity_destroy(entity *entity) { flb_free(entity->key_attributes->environment); flb_free(entity->key_attributes->name); flb_free(entity->key_attributes->type); + flb_free(entity->key_attributes->account_id); flb_free(entity->key_attributes); } flb_free(entity); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.h b/plugins/out_cloudwatch_logs/cloudwatch_logs.h index df912e7b67f..037b366b53e 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_logs.h +++ b/plugins/out_cloudwatch_logs/cloudwatch_logs.h @@ -48,6 +48,7 @@ typedef struct entity_key_attributes { char *type; char *name; char *environment; + char *account_id; }entity_key_attributes; /* Attributes used for CloudWatch Entity object From 99f66151baa7eb906b925681e0c26f268d4f9358 Mon Sep 17 00:00:00 2001 From: Zhihong Lin Date: Wed, 30 Oct 2024 12:38:28 -0400 Subject: [PATCH 2/4] Send entity only when account ID is defined --- plugins/out_cloudwatch_logs/cloudwatch_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index 8bf9a6f6469..1d4dbf21ff4 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -368,7 +368,7 @@ static int init_put_payload(struct flb_cloudwatch *ctx, struct cw_flush *buf, } // If we are missing the service name, the entity will get rejected by the frontend anyway // so do not emit entity unless service name is filled - if(ctx->add_entity && stream->entity != NULL && stream->entity->key_attributes != NULL && stream->entity->key_attributes->name != NULL) { + if(ctx->add_entity && stream->entity != NULL && stream->entity->key_attributes != NULL && stream->entity->key_attributes->name != NULL && stream->entity->key_attributes->account_id != NULL) { if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, "\"entity\":{", 10)) { goto error; From b59d1f3b9556a2526af4181fc2fa23a0d6138015 Mon Sep 17 00:00:00 2001 From: Zhihong Lin Date: Wed, 30 Oct 2024 16:33:51 -0400 Subject: [PATCH 3/4] Enable account and instance ID for entity without customer interaction --- plugins/filter_aws/aws.c | 66 +++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index b99aac83d99..521945048da 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -467,6 +467,32 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) ctx->new_keys++; } + if (ctx->enable_entity) { + if (!ctx->account_id) { + ret = get_metadata_by_key(ctx, FLB_FILTER_AWS_IMDS_ACCOUNT_ID_PATH, + &ctx->account_id, &ctx->account_id_len, + "accountId"); + if (ret < 0) { + return -1; + } + ctx->new_keys++; + } else { + ctx->new_keys++; + } + + if (!ctx->instance_id) { + ret = get_metadata(ctx, FLB_FILTER_AWS_IMDS_INSTANCE_ID_PATH, + &ctx->instance_id, &ctx->instance_id_len); + + if (ret < 0) { + return -1; + } + ctx->new_keys++; + } else { + ctx->new_keys++; + } + } + ctx->metadata_retrieved = FLB_TRUE; return 0; } @@ -558,7 +584,7 @@ static int cb_aws_filter(const void *data, size_t bytes, ctx->availability_zone_len); } - if (ctx->instance_id_include && !ctx->enable_entity) { + if (ctx->instance_id_include) { msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN); msgpack_pack_str_body(&tmp_pck, FLB_FILTER_AWS_INSTANCE_ID_KEY, @@ -566,14 +592,6 @@ static int cb_aws_filter(const void *data, size_t bytes, msgpack_pack_str(&tmp_pck, ctx->instance_id_len); msgpack_pack_str_body(&tmp_pck, ctx->instance_id, ctx->instance_id_len); - } else if (ctx->instance_id_include && ctx->enable_entity) { - msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN); - msgpack_pack_str_body(&tmp_pck, - FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY, - FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN); - msgpack_pack_str(&tmp_pck, ctx->instance_id_len); - msgpack_pack_str_body(&tmp_pck, - ctx->instance_id, ctx->instance_id_len); } if (ctx->instance_type_include) { @@ -616,7 +634,7 @@ static int cb_aws_filter(const void *data, size_t bytes, ctx->ami_id, ctx->ami_id_len); } - if (ctx->account_id_include && !ctx->enable_entity) { + if (ctx->account_id_include) { msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN); msgpack_pack_str_body(&tmp_pck, FLB_FILTER_AWS_ACCOUNT_ID_KEY, @@ -624,14 +642,6 @@ static int cb_aws_filter(const void *data, size_t bytes, msgpack_pack_str(&tmp_pck, ctx->account_id_len); msgpack_pack_str_body(&tmp_pck, ctx->account_id, ctx->account_id_len); - } else if (ctx->account_id_include && ctx->enable_entity) { - msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); - msgpack_pack_str_body(&tmp_pck, - FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY, - FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); - msgpack_pack_str(&tmp_pck, ctx->account_id_len); - msgpack_pack_str_body(&tmp_pck, - ctx->account_id, ctx->account_id_len); } if (ctx->hostname_include) { @@ -643,6 +653,26 @@ static int cb_aws_filter(const void *data, size_t bytes, msgpack_pack_str_body(&tmp_pck, ctx->hostname, ctx->hostname_len); } + + if (ctx->enable_entity && ctx->instance_id != NULL && ctx->account_id != NULL) { + // Pack instance ID with entity prefix for further processing + msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN); + msgpack_pack_str_body(&tmp_pck, + FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY, + FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN); + msgpack_pack_str(&tmp_pck, ctx->instance_id_len); + msgpack_pack_str_body(&tmp_pck, + ctx->instance_id, ctx->instance_id_len); + + // Pack account ID with entity prefix for further processing + msgpack_pack_str(&tmp_pck, FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); + msgpack_pack_str_body(&tmp_pck, + FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY, + FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN); + msgpack_pack_str(&tmp_pck, ctx->account_id_len); + msgpack_pack_str_body(&tmp_pck, + ctx->account_id, ctx->account_id_len); + } } msgpack_unpacked_destroy(&result); From 6677bd707b95651bbdfc0695f0e03213f0058953 Mon Sep 17 00:00:00 2001 From: Zhihong Lin Date: Wed, 30 Oct 2024 16:38:34 -0400 Subject: [PATCH 4/4] Add comment to explain entity drops without account ID --- plugins/out_cloudwatch_logs/cloudwatch_api.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index 1d4dbf21ff4..fb330726afb 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -367,7 +367,8 @@ static int init_put_payload(struct flb_cloudwatch *ctx, struct cw_flush *buf, goto error; } // If we are missing the service name, the entity will get rejected by the frontend anyway - // so do not emit entity unless service name is filled + // so do not emit entity unless service name is filled. If we are missing account ID + // it is considered not having sufficient information for entity therefore we should drop the entity. if(ctx->add_entity && stream->entity != NULL && stream->entity->key_attributes != NULL && stream->entity->key_attributes->name != NULL && stream->entity->key_attributes->account_id != NULL) { if (!try_to_write(buf->out_buf, offset, buf->out_buf_size, "\"entity\":{", 10)) {