Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

out_http: Add AWS SigV4 Authentication Option to out_http #5165

Merged
merged 2 commits into from
Apr 5, 2022
Merged

out_http: Add AWS SigV4 Authentication Option to out_http #5165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a00424
aws: extract initialization of aws_provider commonly used
matthewfala Mar 23, 2022
f51496c
out_http: add SigV4 authentication options
matthewfala Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions include/fluent-bit/flb_aws_credentials.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,13 @@ struct flb_aws_provider {

/* Standard credentials chain is a list of providers */
struct mk_list _head;

/* Provider managed dependencies; to delete on destroy */
struct flb_aws_provider *base_aws_provider;
struct flb_tls *cred_tls; /* tls instances can't be re-used; aws provider requires
a separate one */
struct flb_tls *sts_tls; /* one for the standard chain provider, one for sts
assume role */
Comment on lines +135 to +140
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm this is only used by the new flb_managed_chain_provider_create function to use? the other providers don't use these new fields I guess?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's right. It's just if we want aws_provider_destroy to handle clean up operations. If null (due to calloc) clean up will not occur

};

/*
Expand All @@ -156,6 +163,43 @@ struct flb_aws_provider *flb_standard_chain_provider_create(struct flb_config
flb_aws_client_generator
*generator);

/* Provide base configuration options for managed chain */
#define FLB_AWS_CREDENTIAL_BASE_CONFIG_MAP(prefix) \
{ \
FLB_CONFIG_MAP_STR, prefix "region", NULL, \
0, FLB_FALSE, 0, \
"AWS region of your service" \
}, \
{ \
FLB_CONFIG_MAP_STR, prefix "sts_endpoint", NULL, \
0, FLB_FALSE, 0, \
"Custom endpoint for the AWS STS API, used with the `" prefix "role_arn` option" \
}, \
{ \
FLB_CONFIG_MAP_STR, prefix "role_arn", NULL, \
0, FLB_FALSE, 0, \
"ARN of an IAM role to assume (ex. for cross account access)" \
}, \
{ \
FLB_CONFIG_MAP_STR, prefix "external_id", NULL, \
0, FLB_FALSE, 0, \
"Specify an external ID for the STS API, can be used with the `" prefix \
"role_arn` parameter if your role requires an external ID." \
}

/*
* Managed chain provider; Creates and manages removal of dependancies for an instance
*/
struct flb_aws_provider *flb_managed_chain_provider_create(struct flb_output_instance
*ins,
struct flb_config
*config,
char *config_key_prefix,
char *proxy,
struct
flb_aws_client_generator
*generator);

/*
* A provider that uses OIDC tokens provided by kubernetes to obtain
* AWS credentials.
Expand Down
48 changes: 48 additions & 0 deletions plugins/out_http/http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@
#include <fluent-bit/flb_gzip.h>
#include <msgpack.h>

#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
#include <fluent-bit/flb_aws_credentials.h>
#include <fluent-bit/flb_signv4.h>
#endif
#endif

#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
Expand Down Expand Up @@ -79,6 +86,7 @@ static int http_post(struct flb_out_http *ctx,
struct flb_config_map_val *mv;
struct flb_slist_entry *key = NULL;
struct flb_slist_entry *val = NULL;
flb_sds_t signature = NULL;

/* Get upstream context and connection */
u = ctx->u;
Expand Down Expand Up @@ -174,6 +182,30 @@ static int http_post(struct flb_out_http *ctx,
val->str, flb_sds_len(val->str));
}

#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
/* AWS SigV4 headers */
if (ctx->has_aws_auth == FLB_TRUE) {
flb_plg_debug(ctx->ins, "signing request with AWS Sigv4");
signature = flb_signv4_do(c,
FLB_TRUE, /* normalize URI ? */
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what are the question mark for?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a boolean

FLB_TRUE, /* add x-amz-date header ? */
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same comment as above

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Booleans. Actually copied this comment from other plugins

time(NULL),
(char *) ctx->aws_region,
(char *) ctx->aws_service,
0,
ctx->aws_provider);

if (!signature) {
flb_plg_error(ctx->ins, "could not sign request with sigv4");
out_ret = FLB_RETRY;
goto cleanup;
}
flb_sds_destroy(signature);
}
#endif
#endif

ret = flb_http_do(c, &b_sent);
if (ret == 0) {
/*
Expand Down Expand Up @@ -220,6 +252,7 @@ static int http_post(struct flb_out_http *ctx,
out_ret = FLB_RETRY;
}

cleanup:
/*
* If the payload buffer is different than incoming records in body, means
* we generated a different payload and must be freed.
Expand Down Expand Up @@ -377,6 +410,21 @@ static struct flb_config_map config_map[] = {
0, FLB_TRUE, offsetof(struct flb_out_http, http_passwd),
"Set HTTP auth password"
},
#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
{
FLB_CONFIG_MAP_BOOL, "aws_auth", "false",
0, FLB_TRUE, offsetof(struct flb_out_http, has_aws_auth),
"Enable AWS SigV4 authentication"
},
{
FLB_CONFIG_MAP_STR, "aws_service", NULL,
0, FLB_TRUE, offsetof(struct flb_out_http, aws_service),
"AWS destination service code, used by SigV4 authentication"
},
FLB_AWS_CREDENTIAL_BASE_CONFIG_MAP(FLB_HTTP_AWS_CREDENTIAL_PREFIX),
#endif
#endif
{
FLB_CONFIG_MAP_STR, "header_tag", NULL,
0, FLB_TRUE, offsetof(struct flb_out_http, header_tag),
Expand Down
16 changes: 16 additions & 0 deletions plugins/out_http/http.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,27 @@
#define FLB_HTTP_MIME_MSGPACK "application/msgpack"
#define FLB_HTTP_MIME_JSON "application/json"

#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
#define FLB_HTTP_AWS_CREDENTIAL_PREFIX "aws_"
#endif
#endif

struct flb_out_http {
/* HTTP Auth */
char *http_user;
char *http_passwd;

/* AWS Auth */
#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
int has_aws_auth;
struct flb_aws_provider *aws_provider;
const char *aws_region;
const char *aws_service;
#endif
#endif

/* Proxy */
const char *proxy;
char *proxy_host;
Expand Down
47 changes: 46 additions & 1 deletion plugins/out_http/http_conf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,11 @@
#include <fluent-bit/flb_pack.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_kv.h>

#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
#include <fluent-bit/flb_aws_credentials.h>
#endif
#endif
#include "http.h"
#include "http_conf.h"

Expand Down Expand Up @@ -81,6 +85,39 @@ struct flb_out_http *flb_http_conf_create(struct flb_output_instance *ins,
flb_output_net_default("127.0.0.1", 80, ins);
}

/* Check if AWS SigV4 authentication is enabled */
#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
if (ctx->has_aws_auth) {
ctx->aws_service = flb_output_get_property(FLB_HTTP_AWS_CREDENTIAL_PREFIX
"service", ctx->ins);
if (!ctx->aws_service) {
flb_plg_error(ins, "aws_auth option requires " FLB_HTTP_AWS_CREDENTIAL_PREFIX
"service to be set");
flb_free(ctx);
return NULL;
}

ctx->aws_provider = flb_managed_chain_provider_create(
ins,
config,
FLB_HTTP_AWS_CREDENTIAL_PREFIX,
NULL,
flb_aws_client_generator()
);
if (!ctx->aws_provider) {
flb_plg_error(ins, "failed to create aws credential provider for sigv4 auth");
flb_free(ctx);
return NULL;
}

/* If managed provider creation succeeds, then region key is present */
ctx->aws_region = flb_output_get_property(FLB_HTTP_AWS_CREDENTIAL_PREFIX
"region", ctx->ins);
}
#endif /* !FLB_HAVE_AWS */
#endif /* !FLB_HAVE_SIGNV4 */

/* Check if SSL/TLS is enabled */
#ifdef FLB_HAVE_TLS
if (ins->use_tls == FLB_TRUE) {
Expand Down Expand Up @@ -213,6 +250,14 @@ void flb_http_conf_destroy(struct flb_out_http *ctx)
flb_upstream_destroy(ctx->u);
}

#ifdef FLB_HAVE_SIGNV4
#ifdef FLB_HAVE_AWS
if (ctx->aws_provider) {
flb_aws_provider_destroy(ctx->aws_provider);
}
#endif
#endif

flb_free(ctx->proxy_host);
flb_free(ctx->uri);
flb_free(ctx);
Expand Down
Loading