From d811d6a7d996f0c71d7330d3e5f4f60548dd9cfe Mon Sep 17 00:00:00 2001 From: John Whitman Date: Fri, 3 Feb 2023 19:04:28 -0500 Subject: [PATCH 1/2] Extracts Availability Zone from ECS Task Metadata. --- init/fluent_bit_init_process.go | 17 ++-- init/fluent_bit_init_process_test.go | 86 ++++++++++--------- .../init-process-for-fluent-bit/README.md | 4 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/init/fluent_bit_init_process.go b/init/fluent_bit_init_process.go index 0415576fe..afff69863 100644 --- a/init/fluent_bit_init_process.go +++ b/init/fluent_bit_init_process.go @@ -51,14 +51,15 @@ type S3Downloader interface { // all values in the structure are empty strings by default type ECSTaskMetadata struct { - AWS_REGION string `json:"AWSRegion"` - ECS_CLUSTER string `json:"Cluster"` // Cluster name - ECS_TASK_ARN string `json:"TaskARN"` - ECS_TASK_ID string `json:"TaskID"` - ECS_FAMILY string `json:"Family"` - ECS_LAUNCH_TYPE string `json:"LaunchType"` // Task launch type will be an empty string if container agent is under version 1.45.0 - ECS_REVISION string `json:"Revision"` // Revision number - ECS_TASK_DEFINITION string `json:"TaskDefinition"` // TaskDefinition = "family:revision" + AWS_REGION string `json:"AWSRegion"` + AWS_AVAILABILITY_ZONE string `json:"AvailabilityZone"` + ECS_CLUSTER string `json:"Cluster"` // Cluster name + ECS_TASK_ARN string `json:"TaskARN"` + ECS_TASK_ID string `json:"TaskID"` + ECS_FAMILY string `json:"Family"` + ECS_LAUNCH_TYPE string `json:"LaunchType"` // Task launch type will be an empty string if container agent is under version 1.45.0 + ECS_REVISION string `json:"Revision"` // Revision number + ECS_TASK_DEFINITION string `json:"TaskDefinition"` // TaskDefinition = "family:revision" } // get ECS Task Metadata via endpoint V4 diff --git a/init/fluent_bit_init_process_test.go b/init/fluent_bit_init_process_test.go index 406d431b7..fa6becaf4 100644 --- a/init/fluent_bit_init_process_test.go +++ b/init/fluent_bit_init_process_test.go @@ -31,14 +31,15 @@ func TestGetECSTaskMetadata(t *testing.T) { actualOutput1 := getECSTaskMetadata(&client1) expectedOutput1 := ECSTaskMetadata{ - AWS_REGION: "us-west-2", - ECS_CLUSTER: "ecs-cluster", - ECS_TASK_ARN: "arn:aws:ecs:us-west-2:123456789123:task/ecs-cluster/4ca5a280e68947cd84a8357f0d008fb5", - ECS_TASK_ID: "4ca5a280e68947cd84a8357f0d008fb5", - ECS_FAMILY: "code_test_A", - ECS_LAUNCH_TYPE: "EC2", - ECS_REVISION: "35", - ECS_TASK_DEFINITION: "code_test_A:35", + AWS_REGION: "us-west-2", + AWS_AVAILABILITY_ZONE: "us-west-2a", + ECS_CLUSTER: "ecs-cluster", + ECS_TASK_ARN: "arn:aws:ecs:us-west-2:123456789123:task/ecs-cluster/4ca5a280e68947cd84a8357f0d008fb5", + ECS_TASK_ID: "4ca5a280e68947cd84a8357f0d008fb5", + ECS_FAMILY: "code_test_A", + ECS_LAUNCH_TYPE: "EC2", + ECS_REVISION: "35", + ECS_TASK_DEFINITION: "code_test_A:35", } assert.Equal(t, actualOutput1, expectedOutput1) @@ -52,14 +53,15 @@ func TestGetECSTaskMetadata(t *testing.T) { actualOutput2 := getECSTaskMetadata(&client2) expectedOutput2 := ECSTaskMetadata{ - AWS_REGION: "us-west-2", - ECS_CLUSTER: "ecs-cluster", - ECS_TASK_ARN: "arn:aws:ecs:us-west-2:123456789123:task/ecs-cluster/4ca5a280e68947cd84a8357f0d008fb5", - ECS_TASK_ID: "4ca5a280e68947cd84a8357f0d008fb5", - ECS_FAMILY: "code_test_A", - ECS_LAUNCH_TYPE: "", // empty lunch type - ECS_REVISION: "35", - ECS_TASK_DEFINITION: "code_test_A:35", + AWS_REGION: "us-west-2", + AWS_AVAILABILITY_ZONE: "us-west-2a", + ECS_CLUSTER: "ecs-cluster", + ECS_TASK_ARN: "arn:aws:ecs:us-west-2:123456789123:task/ecs-cluster/4ca5a280e68947cd84a8357f0d008fb5", + ECS_TASK_ID: "4ca5a280e68947cd84a8357f0d008fb5", + ECS_FAMILY: "code_test_A", + ECS_LAUNCH_TYPE: "", // empty lunch type + ECS_REVISION: "35", + ECS_TASK_DEFINITION: "code_test_A:35", } assert.Equal(t, actualOutput2, expectedOutput2) @@ -74,14 +76,15 @@ func TestGetECSTaskMetadata(t *testing.T) { actualOutput3 := getECSTaskMetadata(&client3) expectedOutput3 := ECSTaskMetadata{ - AWS_REGION: "", - ECS_CLUSTER: "", - ECS_TASK_ARN: "", - ECS_TASK_ID: "", - ECS_FAMILY: "", - ECS_LAUNCH_TYPE: "", - ECS_REVISION: "", - ECS_TASK_DEFINITION: "", + AWS_REGION: "", + AWS_AVAILABILITY_ZONE: "", + ECS_CLUSTER: "", + ECS_TASK_ARN: "", + ECS_TASK_ID: "", + ECS_FAMILY: "", + ECS_LAUNCH_TYPE: "", + ECS_REVISION: "", + ECS_TASK_DEFINITION: "", } assert.Equal(t, actualOutput3, expectedOutput3) @@ -94,18 +97,20 @@ func TestSetECSTaskMetadata(t *testing.T) { // Test case 1: full metadata metadataTest1 := ECSTaskMetadata{ - AWS_REGION: "us-west-2", - ECS_CLUSTER: "ecs-Test", - ECS_TASK_ARN: "arn:aws:ecs:us-west-2:111:task/ecs-local-cluster/37e8", - ECS_TASK_ID: "56461", - ECS_FAMILY: "esc-task-definition", - ECS_LAUNCH_TYPE: "EC2", - ECS_REVISION: "1", - ECS_TASK_DEFINITION: "esc-task-definition:1", + AWS_REGION: "us-west-2", + AWS_AVAILABILITY_ZONE: "us-west-2a", + ECS_CLUSTER: "ecs-Test", + ECS_TASK_ARN: "arn:aws:ecs:us-west-2:111:task/ecs-local-cluster/37e8", + ECS_TASK_ID: "56461", + ECS_FAMILY: "esc-task-definition", + ECS_LAUNCH_TYPE: "EC2", + ECS_REVISION: "1", + ECS_TASK_DEFINITION: "esc-task-definition:1", } expectedContent1 := "export FLB_AWS_USER_AGENT=init\n" + "export AWS_REGION=us-west-2\n" + + "export AWS_AVAILABILITY_ZONE=us-west-2a\n" + "export ECS_CLUSTER=ecs-Test\n" + "export ECS_TASK_ARN=arn:aws:ecs:us-west-2:111:task/ecs-local-cluster/37e8\n" + "export ECS_TASK_ID=56461\n" + @@ -119,14 +124,15 @@ func TestSetECSTaskMetadata(t *testing.T) { // Test case 2: some environment variables is empty metadataTest2 := ECSTaskMetadata{ - AWS_REGION: "us-west-1", - ECS_CLUSTER: "ecs-Test", - ECS_TASK_ARN: "", - ECS_TASK_ID: "", - ECS_FAMILY: "", - ECS_LAUNCH_TYPE: "", - ECS_REVISION: "", - ECS_TASK_DEFINITION: "", + AWS_REGION: "us-west-1", + AWS_AVAILABILITY_ZONE: "", + ECS_CLUSTER: "ecs-Test", + ECS_TASK_ARN: "", + ECS_TASK_ID: "", + ECS_FAMILY: "", + ECS_LAUNCH_TYPE: "", + ECS_REVISION: "", + ECS_TASK_DEFINITION: "", } expectedContent2 := "export FLB_AWS_USER_AGENT=init\n" + diff --git a/use_cases/init-process-for-fluent-bit/README.md b/use_cases/init-process-for-fluent-bit/README.md index 47155a42b..d563ef79d 100644 --- a/use_cases/init-process-for-fluent-bit/README.md +++ b/use_cases/init-process-for-fluent-bit/README.md @@ -123,8 +123,8 @@ The init process also injects ECS Task Metadata into the Fluent Bit container as The init process injects ECS Task Metadata into the Fluent Bit container as environment variables: ``` -AWS_REGION / ECS_LAUNCH_TYPE / ECS_CLUSTER / ECS_FAMILY -ECS_TASK_ARN / ECS_TASK_ID / ECS_REVISION / ECS_TASK_DEFINITION +AWS_REGION / AWS_AVAILABILITY_ZONE / ECS_LAUNCH_TYPE / ECS_CLUSTER +ECS_FAMILY / ECS_TASK_ARN / ECS_TASK_ID / ECS_REVISION / ECS_TASK_DEFINITION ``` You can use them as env vars directly in the Fluent Bit config. From 60c7735dba9ea6e75bafeb2919056f85c1cae575 Mon Sep 17 00:00:00 2001 From: John Whitman Date: Fri, 3 Feb 2023 19:11:44 -0500 Subject: [PATCH 2/2] Fixes tests. --- init/fluent_bit_init_process_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/fluent_bit_init_process_test.go b/init/fluent_bit_init_process_test.go index fa6becaf4..1b959140c 100644 --- a/init/fluent_bit_init_process_test.go +++ b/init/fluent_bit_init_process_test.go @@ -108,7 +108,7 @@ func TestSetECSTaskMetadata(t *testing.T) { ECS_TASK_DEFINITION: "esc-task-definition:1", } - expectedContent1 := "export FLB_AWS_USER_AGENT=init\n" + + expectedContent1 := "export FLB_AWS_USER_AGENT=ecs-init\n" + "export AWS_REGION=us-west-2\n" + "export AWS_AVAILABILITY_ZONE=us-west-2a\n" + "export ECS_CLUSTER=ecs-Test\n" + @@ -135,7 +135,7 @@ func TestSetECSTaskMetadata(t *testing.T) { ECS_TASK_DEFINITION: "", } - expectedContent2 := "export FLB_AWS_USER_AGENT=init\n" + + expectedContent2 := "export FLB_AWS_USER_AGENT=ecs-init\n" + "export AWS_REGION=us-west-1\n" + "export ECS_CLUSTER=ecs-Test\n"