From 5b48be71e72c95260dea0f394440043d34964f63 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Wed, 13 Oct 2021 14:32:38 -0700 Subject: [PATCH] aws: imds fallback to v1 if token request fails Signed-off-by: Matthew Fala --- include/fluent-bit/aws/flb_aws_imds.h | 1 + src/aws/flb_aws_credentials_ec2.c | 5 +- src/aws/flb_aws_imds.c | 34 ++++- tests/internal/aws_credentials_ec2.c | 186 +++++++++++++++++++++++--- 4 files changed, 207 insertions(+), 19 deletions(-) diff --git a/include/fluent-bit/aws/flb_aws_imds.h b/include/fluent-bit/aws/flb_aws_imds.h index 0eb80f95e71..2a290a78c13 100644 --- a/include/fluent-bit/aws/flb_aws_imds.h +++ b/include/fluent-bit/aws/flb_aws_imds.h @@ -24,6 +24,7 @@ #define FLB_AWS_IMDS_HOST "169.254.169.254" #define FLB_AWS_IMDS_HOST_LEN 15 #define FLB_AWS_IMDS_PORT 80 +#define FLB_AWS_IMDS_TIMEOUT 1 /* 1 second */ #define FLB_AWS_IMDS_VERSION_EVALUATE 0 #define FLB_AWS_IMDS_VERSION_1 1 diff --git a/src/aws/flb_aws_credentials_ec2.c b/src/aws/flb_aws_credentials_ec2.c index b52e0b88289..02deec991d8 100644 --- a/src/aws/flb_aws_credentials_ec2.c +++ b/src/aws/flb_aws_credentials_ec2.c @@ -248,7 +248,10 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, return NULL; } - upstream->net.connect_timeout = FLB_AWS_CREDENTIAL_NET_TIMEOUT; + /* IMDSv2 token request will timeout if hops = 1 and running within container */ + upstream->net.connect_timeout = FLB_AWS_IMDS_TIMEOUT; + upstream->net.io_timeout = FLB_AWS_IMDS_TIMEOUT; + upstream->net.keepalive = FLB_FALSE; /* On timeout, the connection is broken */ implementation->client = generator->create(); if (!implementation->client) { diff --git a/src/aws/flb_aws_imds.c b/src/aws/flb_aws_imds.c index fbba13b9909..6f8551e2b60 100644 --- a/src/aws/flb_aws_imds.c +++ b/src/aws/flb_aws_imds.c @@ -238,6 +238,7 @@ flb_sds_t flb_aws_imds_get_vpc_id(struct flb_aws_imds *ctx) /* Obtain the IMDS version */ static int get_imds_version(struct flb_aws_imds *ctx) { + int ret; struct flb_aws_client *client = ctx->ec2_imds_client; struct flb_aws_header invalid_token_header; struct flb_http_client *c = NULL; @@ -259,20 +260,49 @@ static int get_imds_version(struct flb_aws_imds *ctx) &invalid_token_header, 1); if (!c) { + flb_debug("[imds] imds endpoint unavailable"); return FLB_AWS_IMDS_VERSION_EVALUATE; } /* Unauthorized response means that IMDS version 2 is in use */ if (c->resp.status == 401) { ctx->imds_version = FLB_AWS_IMDS_VERSION_2; - refresh_imds_v2_token(ctx); + ret = refresh_imds_v2_token(ctx); + if (ret == -1) { + /* + * Token cannot be refreshed, test IMDSv1 + * If IMDSv1 cannot be used, response will be status 401 + */ + flb_http_client_destroy(c); + ctx->imds_version = FLB_AWS_IMDS_VERSION_EVALUATE; + c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, + NULL, 0, NULL, 0); + if (!c) { + flb_debug("[imds] imds v1 attempt, endpoint unavailable"); + return FLB_AWS_IMDS_VERSION_EVALUATE; + } + + if (c->resp.status == 200) { + flb_info("[imds] to use IMDSv2, set --http-put-response-hop-limit to 2"); + } + else { + /* IMDSv1 unavailable. IMDSv2 beyond network hop count */ + flb_warn("[imds] failed to retrieve IMDSv2 token and IMDSv1 unavailable. " + "This is likely due to instance-metadata-options " + "--http-put-response-hop-limit being set to 1 and --http-tokens " + "set to required. " + "To use IMDSv2, please set --http-put-response-hop-limit to 2 as " + "described https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/" + "configuring-instance-metadata-options.html"); + } + } } /* * Success means that IMDS version 1 is in use - * (Not Tested, TODO: Must test this on an instance without IMDSv2) */ if (c->resp.status == 200) { + flb_warn("[imds] falling back on IMDSv1"); ctx->imds_version = FLB_AWS_IMDS_VERSION_1; } diff --git a/tests/internal/aws_credentials_ec2.c b/tests/internal/aws_credentials_ec2.c index 549c76ca207..ff5f4339658 100644 --- a/tests/internal/aws_credentials_ec2.c +++ b/tests/internal/aws_credentials_ec2.c @@ -290,6 +290,153 @@ static void test_ec2_provider_v1() cleanup_test(); } +/* + * IMDSv1 -- IMDSv2 Timeout, Fallback Test Summary + * First call to get_credentials(): + * -> 1 requests is made to test for IMDSv2 (IMDSv2) + * -> 1 request made to get token (Timeout failure) + * -> 1 request made to check IMDSv1 fallback (Failure - IMDSv1 not allowed) + * + * Second call to get_credentials(): + * -> 1 requests is made to test for IMDSv2 (IMDSv2) + * -> 1 request made to get token (Timeout failure) + * -> 1 request made to check IMDSv1 fallback (Success) + * -> 2 requests are made to access credentials + * Second call to get_credentials() hits cache + * -> 0 requests are made + * refresh(): + * -> 2 requests are made to access credentials + */ +static void test_ec2_provider_v1_v2_timeout() +{ + setup_test(FLB_AWS_CLIENT_MOCK( + /* First call to get_credentials() */ + response( + expect(URI, "/"), + expect(HEADER, "X-aws-ec2-metadata-token", "INVALID"), + expect(HEADER_COUNT, 1), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 401) + ), + response( + expect(URI, "/latest/api/token"), + expect(HEADER, "X-aws-ec2-metadata-token-ttl-seconds", "21600"), /* 6 hours */ + expect(METHOD, FLB_HTTP_PUT), + config(REPLACE, (struct flb_http_client *) NULL) /* Replicate timeout failure */ + ), + response( + expect(URI, "/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 401) /* IMDSv1 not allowed */ + ), + + /* Second call to get_credentials() */ + response( + expect(URI, "/"), + expect(HEADER, "X-aws-ec2-metadata-token", "INVALID"), + expect(HEADER_COUNT, 1), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 401) + ), + response( + expect(URI, "/latest/api/token"), + expect(HEADER, "X-aws-ec2-metadata-token-ttl-seconds", "21600"), /* 6 hours */ + expect(METHOD, FLB_HTTP_PUT), + config(REPLACE, (struct flb_http_client *) NULL) /* Replicate timeout failure */ + ), + response( + expect(URI, "/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200) /* IMDSv1 is allowed */ + ), + response( + expect(URI, "/latest/meta-data/iam/security-credentials/"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER_COUNT, 0), + set(STATUS, 200), + set(PAYLOAD, "My_Instance_Name"), + set(PAYLOAD_SIZE, 16) + ), + response( + expect(URI, "/latest/meta-data/iam/security-credentials/My_Instance_Name"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER_COUNT, 0), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"XACCESSEC2XXX\",\n \"SecretAccessKey\"" + " : \"XSECRETEC2XXXXXXXXXXXXXX\",\n \"Token\" : \"XTOKENEC2XXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), /* Expires Year 3021 */ + set(PAYLOAD_SIZE, 257) + ), + + /* Second call to get_credentials() hits cache */ + + /* Refresh credentials (No token refesh) */ + response( + expect(URI, "/latest/meta-data/iam/security-credentials/"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER_COUNT, 0), + set(STATUS, 200), + set(PAYLOAD, "My_Instance_Name_New"), + set(PAYLOAD_SIZE, 20) + ), + response( + expect(URI, "/latest/meta-data/iam/security-credentials/My_Instance_Name_New"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER_COUNT, 0), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"YACCESSEC2XXX\",\n \"SecretAccessKey\"" + " : \"YSECRETEC2XXXXXXXXXXXXXX\",\n \"Token\" : \"YTOKENEC2XXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), // Expires Year 3021 + set(PAYLOAD_SIZE, 257) + ) + )); + + /* First call: IMDSv1 and IMDSv2 not accessible */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds == NULL); + + /* + * Second call: IMDSv2 timeout, IMDSv1 accessible + * Repeated calls to get credentials should return the same set + */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEC2XXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEC2XXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEC2XXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Retrieve from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEC2XXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEC2XXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEC2XXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret == 0); + + /* Retrieve refreshed credentials from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("YACCESSEC2XXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("YSECRETEC2XXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("YTOKENEC2XXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(); +} + /* * IMDS Version Detection Error -- Test Summary @@ -397,8 +544,7 @@ static void test_ec2_provider_version_detection_error() * First call to get_credentials(): * -> 1 request made to test for IMDSv2 (Success) * -> 1 request made to obtain IMDSv2 token (Fails) <-* Aquire token error - * -> 1 requests are made to access credential (Invalid token) - * -> 1 request made to obtain IMDSv2 token (Fails) <-* Aquire token error + * -> 1 request made to check IMDSv1 fallback (Unauthorized) * Second call to get_credentials(): * -> 1 request made to access instance name (Invalid token) * -> 1 request made to obtain IMDSv2 token (Success) @@ -428,8 +574,7 @@ static void test_ec2_provider_acquire_token_error() * First call to get_credentials(): * -> 1 request made to test for IMDSv2 (Success) * -> 1 request made to obtain IMDSv2 token (Fails) <-* Aquire token error - * -> 1 requests are made to access credential (Invalid token) - * -> 1 request made to obtain IMDSv2 token (Fails) <-* Aquire token error + * -> 1 request made to check IMDSv1 fallback (Unauthorized) */ response( expect(URI, "/"), @@ -445,32 +590,40 @@ static void test_ec2_provider_acquire_token_error() config(REPLACE, NULL) /* HTTP Client is null */ ), response( - expect(URI, "/latest/meta-data/iam/security-credentials/"), + expect(URI, "/"), expect(METHOD, FLB_HTTP_GET), - expect(HEADER, "X-aws-ec2-metadata-token", "INVALID_TOKEN"), /* Token failed to be set */ - set(STATUS, 401) /* Unauthorized, bad token */ - ), - response( - expect(URI, "/latest/api/token"), - expect(HEADER, "X-aws-ec2-metadata-token-ttl-seconds", "21600"), /* 6 hours */ - expect(METHOD, FLB_HTTP_PUT), - set(STATUS, 200), /* Unauthorized, bad token */ - set(PAYLOAD, ""), - set(PAYLOAD_SIZE, 0) /* Aquire token failure 2: no token in response */ + set(STATUS, 401) /* IMDSv1 not allowed */ ), /* * Second call to get_credentials(): + * -> 1 request made to test for IMDSv2 (Success) + * -> 1 request made to obtain IMDSv2 token (Success) <-* Bad token * -> 1 request made to access instance name (Invalid token) * -> 1 request made to obtain IMDSv2 token (Success) * -> 1 request made to access instance name (Success) * -> 1 request made to access credentials (Invalid token) * -> 1 request made to obtain IMDSv2 token (Fails) <-* Aquire token error */ + response( + expect(URI, "/"), + expect(HEADER, "X-aws-ec2-metadata-token", "INVALID"), /* Why is this not invalid_token? */ + expect(HEADER_COUNT, 1), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 401) /* IMDSv2 */ + ), + response( + expect(URI, "/latest/api/token"), + expect(HEADER, "X-aws-ec2-metadata-token-ttl-seconds", "21600"), /* 6 hours */ + expect(METHOD, FLB_HTTP_PUT), + set(STATUS, 200), + set(PAYLOAD, "BAD_ANjUxxxxxxXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_Q=="), + set(PAYLOAD_SIZE, 56) + ), response( expect(URI, "/latest/meta-data/iam/security-credentials/"), expect(METHOD, FLB_HTTP_GET), - expect(HEADER, "X-aws-ec2-metadata-token", "INVALID_TOKEN"), /* Token failed to be set */ + expect(HEADER, "X-aws-ec2-metadata-token", "BAD_ANjUxxxxxxXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_Q=="), /* Token failed to be set */ set(STATUS, 401) /* Unauthorized, bad token */ ), response( @@ -875,6 +1028,7 @@ static void test_ec2_imds_create_and_destroy() TEST_LIST = { { "test_ec2_provider_v2" , test_ec2_provider_v2}, { "test_ec2_provider_v1" , test_ec2_provider_v1}, + { "test_ec2_provider_v1_v2_timeout" , test_ec2_provider_v1_v2_timeout}, { "test_ec2_provider_version_detection_error" , test_ec2_provider_version_detection_error}, { "test_ec2_provider_acquire_token_error" , test_ec2_provider_acquire_token_error}, { "test_ec2_provider_metadata_request_error" , test_ec2_provider_metadata_request_error},