diff --git a/include/fluent-bit/flb_aws_credentials.h b/include/fluent-bit/flb_aws_credentials.h index 575269cec8f..fe89223ca8a 100644 --- a/include/fluent-bit/flb_aws_credentials.h +++ b/include/fluent-bit/flb_aws_credentials.h @@ -250,19 +250,21 @@ struct flb_aws_provider *flb_aws_env_provider_create(); * Calling flb_aws_provider_destroy on this provider frees the memory * used by host and path. */ -struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - flb_sds_t host, - flb_sds_t path, - struct - flb_aws_client_generator - *generator); +struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, + flb_sds_t host, + flb_sds_t path, + int port, + int insecure, + struct + flb_aws_client_generator + *generator); /* - * ECS Provider + * HTTP Provider for EKS and ECS * The ECS Provider is just a wrapper around the HTTP Provider * with the ECS credentials endpoint. */ -struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, +struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, struct flb_aws_client_generator *generator); @@ -343,6 +345,25 @@ int try_lock_provider(struct flb_aws_provider *provider); void unlock_provider(struct flb_aws_provider *provider); +/* + * HTTP Credentials Provider - retrieve credentials from a local http server + * Used to implement the ECS Credentials provider. + * Equivalent to: + * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds + */ + +struct flb_aws_provider_http { + struct flb_aws_credentials *creds; + time_t next_refresh; + + struct flb_aws_client *client; + + /* Host and Path to request credentials */ + flb_sds_t host; + flb_sds_t path; + + flb_sds_t auth_token; /* optional */ +}; #endif #endif /* FLB_HAVE_AWS */ diff --git a/include/fluent-bit/flb_aws_util.h b/include/fluent-bit/flb_aws_util.h index 08507dccfbb..f0d0996e4ca 100644 --- a/include/fluent-bit/flb_aws_util.h +++ b/include/fluent-bit/flb_aws_util.h @@ -104,6 +104,17 @@ struct flb_aws_client { int debug_only; }; +/* frees dynamic_headers */ +struct flb_http_client *flb_aws_client_request_basic_auth( + struct flb_aws_client *aws_client, + int method, const char *uri, + const char *body, size_t body_len, + struct flb_aws_header + *dynamic_headers, + size_t dynamic_headers_len, + char *header_name, + char* auth_token); + /* * Frees the aws_client, the internal flb_http_client, error_code, * and flb_upstream. @@ -151,6 +162,13 @@ flb_sds_t flb_aws_error(char *response, size_t response_len); void flb_aws_print_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins); +/* + * Error parsing for json APIs that respond with a + * Code and Message fields for error responses. + */ +void flb_aws_print_error_code(char *response, size_t response_len, + char *api); + /* Similar to 'flb_aws_print_error', but for APIs that return XML */ void flb_aws_print_xml_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins); diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index 7e42a107c0d..80d67a26124 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -69,5 +69,7 @@ int flb_utils_read_file(char *path, char **out_buf, size_t *out_size); char *flb_utils_get_os_name(); int flb_utils_uuid_v4_gen(char *buf); int flb_utils_get_machine_id(char **out_id, size_t *out_size); +int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, + flb_sds_t *out_host, flb_sds_t *out_port, flb_sds_t *out_uri); #endif diff --git a/src/aws/flb_aws_credentials.c b/src/aws/flb_aws_credentials.c index da764a30034..5c0cef05475 100644 --- a/src/aws/flb_aws_credentials.c +++ b/src/aws/flb_aws_credentials.c @@ -569,7 +569,7 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config } } - sub_provider = flb_ecs_provider_create(config, generator); + sub_provider = flb_http_provider_create(config, generator); if (sub_provider) { /* ECS Provider will fail creation if we are not running in ECS */ mk_list_add(&sub_provider->_head, &implementation->sub_providers); diff --git a/src/aws/flb_aws_credentials_http.c b/src/aws/flb_aws_credentials_http.c index 0cf0c0d97f1..d8cc6ce8784 100644 --- a/src/aws/flb_aws_credentials_http.c +++ b/src/aws/flb_aws_credentials_http.c @@ -22,6 +22,7 @@ #include #include #include +#include "fluent-bit/flb_utils.h" #include #include @@ -36,33 +37,41 @@ #define ECS_CREDENTIALS_HOST "169.254.170.2" #define ECS_CREDENTIALS_HOST_LEN 13 -#define ECS_CREDENTIALS_PATH_ENV_VAR "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" +#define EKS_CREDENTIALS_HOST "169.254.170.23" +#define EKS_CREDENTIALS_HOST_LEN 14 +#define AWS_CREDENTIALS_RELATIVE_URI "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" +#define AWS_CREDENTIALS_FULL_URI "AWS_CONTAINER_CREDENTIALS_FULL_URI" + +#define AUTH_TOKEN_ENV_VAR "AWS_CONTAINER_AUTHORIZATION_TOKEN" +#define AUTH_TOKEN_FILE_ENV_VAR "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE" /* Declarations */ -struct flb_aws_provider_http; static int http_credentials_request(struct flb_aws_provider_http *implementation); - /* - * HTTP Credentials Provider - retrieve credentials from a local http server - * Used to implement the ECS Credentials provider. - * Equivalent to: - * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds - */ - -struct flb_aws_provider_http { - struct flb_aws_credentials *creds; - time_t next_refresh; - - struct flb_aws_client *client; - - /* Host and Path to request credentials */ - flb_sds_t host; - flb_sds_t path; -}; - +If the resolved URI’s scheme is HTTPS, its hostname may be used in the request. +Otherwise, implementations MUST fail to resolve when the URI hostname +does not satisfy any of the following conditions: +is within the loopback CIDR (IPv4 127.0.0.0/8, IPv6 ::1/128) +is the ECS container host 169.254.170.2 +is the EKS container host (IPv4 169.254.170.23, IPv6 fd00:ec2::23)*/ +static int validate_http_credential_uri(flb_sds_t protocol, flb_sds_t host) +{ + if (strncmp(protocol, "https", 5) == 0) { + return 0; + } else if (strncmp(host, "127.", 4) == 0 || + strncmp(host, ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN) == 0 || + strncmp(host, EKS_CREDENTIALS_HOST, EKS_CREDENTIALS_HOST_LEN) == 0 || + strstr(host, "::1") != NULL || + strstr(host, "fd00:ec2::23") != NULL || + strstr(host, "fe80:") != NULL) { + return 0; + } + + return -1; +} struct flb_aws_credentials *get_credentials_fn_http(struct flb_aws_provider *provider) @@ -83,6 +92,8 @@ struct flb_aws_credentials *get_credentials_fn_http(struct flb_aws_provider if (try_lock_provider(provider)) { http_credentials_request(implementation); unlock_provider(provider); + } else { + flb_error("try_lock_provider failed"); } } @@ -229,16 +240,19 @@ static struct flb_aws_provider_vtable http_provider_vtable = { .upstream_set = upstream_set_fn_http, }; -struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - flb_sds_t host, - flb_sds_t path, - struct - flb_aws_client_generator - *generator) +struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, + flb_sds_t host, + flb_sds_t path, + int port, + int insecure, + struct + flb_aws_client_generator + *generator) { struct flb_aws_provider_http *implementation = NULL; struct flb_aws_provider *provider = NULL; struct flb_upstream *upstream = NULL; + int io_flags = insecure == FLB_TRUE ? FLB_IO_TCP : FLB_IO_TLS; flb_debug("[aws_credentials] Configuring HTTP provider with %s:80%s", host, path); @@ -266,7 +280,7 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, implementation->host = host; implementation->path = path; - upstream = flb_upstream_create(config, host, 80, FLB_IO_TCP, NULL); + upstream = flb_upstream_create(config, host, port, io_flags, NULL); if (!upstream) { flb_aws_provider_destroy(provider); @@ -289,7 +303,7 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, implementation->client->provider = NULL; implementation->client->region = NULL; implementation->client->service = NULL; - implementation->client->port = 80; + implementation->client->port = port; implementation->client->flags = 0; implementation->client->proxy = NULL; implementation->client->upstream = upstream; @@ -303,38 +317,88 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, * with the ECS credentials endpoint. */ - struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, - struct - flb_aws_client_generator - *generator) +struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, + struct flb_aws_client_generator *generator) { - flb_sds_t host = NULL; flb_sds_t path = NULL; - char *path_var = NULL; + flb_sds_t protocol = NULL; + flb_sds_t host = NULL; + flb_sds_t port_sds = NULL; + int port = 80; + int insecure = FLB_TRUE; + char *relative_uri = NULL; + char *full_uri = NULL; + int ret; - host = flb_sds_create_len(ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN); - if (!host) { - flb_errno(); - return NULL; - } + relative_uri = getenv(AWS_CREDENTIALS_RELATIVE_URI); + full_uri = getenv(AWS_CREDENTIALS_FULL_URI); - path_var = getenv(ECS_CREDENTIALS_PATH_ENV_VAR); - if (path_var && strlen(path_var) > 0) { - path = flb_sds_create(path_var); + if (relative_uri && strlen(relative_uri) > 0) { + host = flb_sds_create_len(ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN); + if (!host) { + flb_errno(); + return NULL; + } + path = flb_sds_create(relative_uri); if (!path) { flb_errno(); flb_free(host); return NULL; } + } + else if (full_uri && strlen(full_uri) > 0) { + ret = flb_utils_url_split_sds(full_uri, &protocol, &host, &port_sds, &path); + if (ret < 0) { + return NULL; + } - return flb_http_provider_create(config, host, path, generator); - } else { - flb_debug("[aws_credentials] Not initializing ECS Provider because" - " %s is not set", ECS_CREDENTIALS_PATH_ENV_VAR); - flb_sds_destroy(host); + insecure = strncmp(protocol, "http", 4) == 0 ? FLB_TRUE : FLB_FALSE; + ret = validate_http_credential_uri(protocol, host); + if (ret < 0) { + flb_error("[aws credentials] %s must be set to an https:// address or a link local IP address." + " Found protocol=%s, host=%s, port=%s, path=%s", + AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); + flb_sds_destroy(protocol); + flb_sds_destroy(host); + flb_sds_destroy(port_sds); + flb_sds_destroy(path); + return NULL; + } + } + else { + flb_debug("[aws_credentials] Not initializing ECS/EKS HTTP Provider because" + " %s and %s is not set", AWS_CREDENTIALS_RELATIVE_URI, AWS_CREDENTIALS_FULL_URI); return NULL; } + if (port_sds != NULL) { + port = atoi(port_sds); + if (port == 0) { + flb_error("[aws credentials] invalid port: %s must be set to an https:// address or a link local IP address." + " Found protocol=%s, host=%s, port=%s, path=%s", + AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); + flb_sds_destroy(protocol); + flb_sds_destroy(host); + flb_sds_destroy(port_sds); + flb_sds_destroy(path); + return NULL; + } + } + + flb_sds_destroy(port_sds); + flb_sds_destroy(protocol); + + return flb_endpoint_provider_create(config, host, path, port, insecure, generator); +} + +static void trim_newline(char *token) +{ + int i; + for (i = strlen(token) - 1; i > 0; i--) { + if (token[i] == '\r' || token[i] == '\n') { + token[i] = '\0'; + } + } } static int http_credentials_request(struct flb_aws_provider_http @@ -346,16 +410,65 @@ static int http_credentials_request(struct flb_aws_provider_http struct flb_aws_credentials *creds = NULL; struct flb_aws_client *client = implementation->client; struct flb_http_client *c = NULL; + int ret; + char *tmp; + char *auth_token = NULL; + size_t auth_token_size = 0; + char *auth_token_path = NULL; + + auth_token_path = getenv(AUTH_TOKEN_FILE_ENV_VAR); + tmp = getenv(AUTH_TOKEN_ENV_VAR); + if (tmp) { + auth_token = flb_malloc(strlen(tmp) + 1); + if (!auth_token) { + flb_errno(); + return -1; + } + strcpy(auth_token, tmp); + } - c = client->client_vtable->request(client, FLB_HTTP_GET, - implementation->path, NULL, 0, - NULL, 0); + if (auth_token_path != NULL && strlen(auth_token_path) > 0) { + flb_debug("[aws] reading authorization token from %s", auth_token_path); + + if (auth_token) { + flb_free(auth_token); + auth_token = NULL; + } + + ret = flb_read_file(auth_token_path, &auth_token, + &auth_token_size); + if (ret < 0) { + flb_error("[aws credentials] failed to read authorization token from %s", + auth_token_path); + return -1; + } + } + + if (auth_token != NULL && strlen(auth_token) > 0) { + trim_newline(auth_token); + c = flb_aws_client_request_basic_auth(client, FLB_HTTP_GET, implementation->path, + NULL, 0, NULL, 0, + "Authorization", + auth_token); + } else { + c = client->client_vtable->request(client, FLB_HTTP_GET, + implementation->path, NULL, 0, + NULL, 0); + } + + if (auth_token) { + flb_free(auth_token); + auth_token = NULL; + } if (!c || c->resp.status != 200) { flb_debug("[aws_credentials] http credentials request failed"); if (c) { flb_http_client_destroy(c); } + if (auth_token) { + flb_free(auth_token); + } return -1; } @@ -365,6 +478,9 @@ static int http_credentials_request(struct flb_aws_provider_http creds = flb_parse_http_credentials(response, response_len, &expiration); if (!creds) { flb_http_client_destroy(c); + if (auth_token) { + flb_free(auth_token); + } return -1; } @@ -375,6 +491,7 @@ static int http_credentials_request(struct flb_aws_provider_http implementation->creds = creds; implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; flb_http_client_destroy(c); + return 0; } diff --git a/src/aws/flb_aws_util.c b/src/aws/flb_aws_util.c index 26c67635c9b..d3e2f3af234 100644 --- a/src/aws/flb_aws_util.c +++ b/src/aws/flb_aws_util.c @@ -200,10 +200,8 @@ struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client if (flb_aws_is_auth_error(c->resp.payload, c->resp.payload_size) == FLB_TRUE) { flb_error("[aws_client] auth error, refreshing creds"); - aws_client->refresh_limit = time(NULL) - + FLB_AWS_CREDENTIAL_REFRESH_LIMIT; - aws_client->provider->provider_vtable-> - refresh(aws_client->provider); + aws_client->refresh_limit = time(NULL) + FLB_AWS_CREDENTIAL_REFRESH_LIMIT; + aws_client->provider->provider_vtable->refresh(aws_client->provider); } } } @@ -211,6 +209,50 @@ struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client return c; } +/* always frees dynamic_headers */ +struct flb_http_client *flb_aws_client_request_basic_auth( + struct flb_aws_client *aws_client, + int method, const char *uri, + const char *body, size_t body_len, + struct flb_aws_header *dynamic_headers, + size_t dynamic_headers_len, + char *header_name, + char* auth_token) +{ + struct flb_http_client *c = NULL; + struct flb_aws_header *auth_header = NULL; + struct flb_aws_header *headers = NULL; + + auth_header = flb_calloc(1, sizeof(struct flb_aws_header)); + if (!auth_header) { + flb_errno(); + return NULL; + } + + auth_header->key = header_name; + auth_header->key_len = strlen(header_name); + auth_header->val = auth_token; + auth_header->val_len = strlen(auth_token); + + if (dynamic_headers_len == 0) { + c = aws_client->client_vtable->request(aws_client, method, uri, body, body_len, + auth_header, 1); + } else { + headers = flb_realloc(dynamic_headers, (dynamic_headers_len + 1) * sizeof(struct flb_aws_header)); + if (!headers) { + flb_free(auth_header); + flb_errno(); + return NULL; + } + *(headers + dynamic_headers_len) = *auth_header; + c = aws_client->client_vtable->request(aws_client, method, uri, body, body_len, + headers, dynamic_headers_len + 1); + flb_free(headers); + } + flb_free(auth_header); + return c; +} + static struct flb_aws_client_vtable client_vtable = { .request = flb_aws_client_request, }; @@ -573,6 +615,10 @@ flb_sds_t flb_aws_xml_get_val(char *response, size_t response_len, char *tag, ch return val; } +/* + * Error parsing for json APIs that respond with an + * __type and message fields for error responses. + */ void flb_aws_print_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins) { @@ -600,6 +646,37 @@ void flb_aws_print_error(char *response, size_t response_len, flb_sds_destroy(error); } +/* + * Error parsing for json APIs that respond with a + * Code and Message fields for error responses. + */ +void flb_aws_print_error_code(char *response, size_t response_len, + char *api) +{ + flb_sds_t error; + flb_sds_t message; + + error = flb_json_get_val(response, response_len, "Code"); + if (!error) { + /* error can not be parsed, print raw response */ + flb_warn("%s: Raw response: %s", api, response); + return; + } + + message = flb_json_get_val(response, response_len, "Message"); + if (!message) { + /* just print the error */ + flb_error("%s API responded with code='%s'", api, error); + } + else { + flb_error("%s API responded with code='%s', message='%s'", + api, error, message); + flb_sds_destroy(message); + } + + flb_sds_destroy(error); +} + /* parses AWS JSON API error responses and returns the value of the __type field */ flb_sds_t flb_aws_error(char *response, size_t response_len) { diff --git a/src/flb_utils.c b/src/flb_utils.c index 6905ed97cab..00dca33af98 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -890,13 +890,27 @@ int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t static char *flb_copy_host(const char *string, int pos_init, int pos_end) { if (string[pos_init] == '[') { /* IPv6 */ - if (string[pos_end-1] != ']') + if (string[pos_end-1] != ']') { return NULL; - + } return mk_string_copy_substr(string, pos_init + 1, pos_end - 1); } - else + else { return mk_string_copy_substr(string, pos_init, pos_end); + } +} + +static char *flb_utils_copy_host_sds(const char *string, int pos_init, int pos_end) +{ + if (string[pos_init] == '[') { /* IPv6 */ + if (string[pos_end-1] != ']') { + return NULL; + } + return flb_sds_create_len(string + pos_init + 1, pos_end - 1); + } + else { + return flb_sds_create_len(string + pos_init, pos_end); + } } int flb_utils_url_split(const char *in_url, char **out_protocol, @@ -994,6 +1008,134 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, return -1; } +int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, + flb_sds_t *out_host, flb_sds_t *out_port, flb_sds_t *out_uri) +{ + int i; + flb_sds_t protocol = NULL; + flb_sds_t host = NULL; + flb_sds_t port = NULL; + flb_sds_t uri = NULL; + char *p = NULL; + char *tmp = NULL; + char *sep = NULL; + + /* Protocol */ + p = strstr(in_url, "://"); + if (!p) { + return -1; + } + if (p == in_url) { + return -1; + } + + protocol = flb_sds_create_len(in_url, p - in_url); + if (!protocol) { + flb_errno(); + return -1; + } + + /* Advance position after protocol */ + p += 3; + + /* Check for first '/' */ + sep = strchr(p, '/'); + tmp = strchr(p, ':'); + + /* Validate port separator is found before the first slash */ + if (sep && tmp) { + if (tmp > sep) { + tmp = NULL; + } + } + + if (tmp) { + host = flb_utils_copy_host_sds(p, 0, tmp - p); + if (!host) { + flb_errno(); + goto error; + } + p = tmp + 1; + + /* Look for an optional URI */ + tmp = strchr(p, '/'); + if (tmp) { + port = flb_sds_create_len(p, tmp - p); + uri = flb_sds_create(tmp); + } + else { + port = flb_sds_create_len(p, strlen(p)); + uri = flb_sds_create("/"); + } + } + else { + tmp = strchr(p, '/'); + if (tmp) { + host = flb_utils_copy_host_sds(p, 0, tmp - p); + uri = flb_sds_create(tmp); + } + else { + host = flb_utils_copy_host_sds(p, 0, strlen(p)); + uri = flb_sds_create("/"); + } + } + + if (!port) { + if (strcmp(protocol, "http") == 0) { + port = flb_sds_create("80"); + } + else if (strcmp(protocol, "https") == 0) { + port = flb_sds_create("443"); + } + } + + if (!host) { + flb_errno(); + goto error; + } + + if (!port) { + flb_errno(); + goto error; + } + else { + /* check that port is a number */ + for (i = 0; i < flb_sds_len(port); i++) { + if (!isdigit(port[i])) { + goto error; + } + } + + } + + if (!uri) { + flb_errno(); + goto error; + } + + *out_protocol = protocol; + *out_host = host; + *out_port = port; + *out_uri = uri; + + return 0; + + error: + if (protocol) { + flb_sds_destroy(protocol); + } + if (host) { + flb_sds_destroy(host); + } + if (port) { + flb_sds_destroy(port); + } + if (uri) { + flb_sds_destroy(uri); + } + + return -1; +} /* * flb_utils_proxy_url_split parses a proxy's information from a http_proxy URL.