From 1f484d481f21e8c34f7412b3ff4641480b42674d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 7 Jan 2021 10:38:13 +0100 Subject: [PATCH] Provide more ways to set AWS credentials in Functionbeat (#23344) This PR makes credential settings when deploying Lambdas to AWS more flexible. New options are introduced: 1. `access_key_id`, `secret_access_key` and/or `session_token` for tokens ```yaml functionbeat.provider.aws.access_key_id: '${AWS_ACCESS_KEY_ID:""}' functionbeat.provider.aws.secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' functionbeat.provider.aws.session_token: '${AWS_SESSION_TOKEN:""}' ``` 2. `role_arn` for assuming IAM roles ```yaml functionbeat.provider.aws.role_arn: arn:aws:iam::123456789012:role/test-fnb ``` 3. `credential_profile_name` and/or `shared_credential_file` for credential files ```yaml functionbeat.provider.aws.credential_profile_name: fnb-aws functionbeat.provider.aws.shared_credential_file: /etc/functionbeat/aws_credentials ``` Credential configuration becomes more flexible and follows the same pattern as in Filebeat and Metricbeat. Based on #17658 Closes #12464 Co-authored-by: Brandon Morelli (cherry picked from commit 5e6558b74bbc25175589f02782852e07c10372d5) --- CHANGELOG.next.asciidoc | 1 + .../_meta/config/beat.reference.yml.tmpl | 12 ++++++++++ .../docs/config-options-aws.asciidoc | 17 ++++++++----- .../functionbeat/functionbeat.reference.yml | 12 ++++++++++ .../functionbeat/manager/aws/cli_manager.go | 24 +++++++++++++------ .../manager/aws/template_builder.go | 2 +- .../functionbeat/provider/aws/aws/config.go | 20 ++++++++++++++-- .../docs/aws-credentials-config.asciidoc | 18 ++++++++++++-- 8 files changed, 88 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bee03414282..099c2b45884 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -646,6 +646,7 @@ port. {pull}19209[19209] *Functionbeat* +- Provide more ways to set AWS credentials. {issue}12464[12464] {pull}23344[23344] *Winlogbeat* diff --git a/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl index b0ec63db137..64d00a2fc1c 100644 --- a/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl +++ b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl @@ -16,6 +16,18 @@ functionbeat.provider.aws.endpoint: "s3.amazonaws.com" # Configure which S3 bucket we should upload the lambda artifact. functionbeat.provider.aws.deploy_bucket: "functionbeat-deploy" +# Configure credentials of Functionbeat while deploying to AWS. +# Available options: +# * access_key_id, secret_access_key and/or session_token +#functionbeat.provider.aws.access_key_id: '${AWS_ACCESS_KEY_ID:""}' +#functionbeat.provider.aws.secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' +#functionbeat.provider.aws.session_token: '${AWS_SESSION_TOKEN:""}' +# * role_arn +#functionbeat.provider.aws.role_arn: arn:aws:iam::123456789012:role/test-fnb +# * credential_profile_name and/or shared_credential_file +#functionbeat.provider.aws.credential_profile_name: fnb-aws +#functionbeat.provider.aws.shared_credential_file: /etc/functionbeat/aws_credentials + functionbeat.provider.aws.functions: # Define the list of function availables, each function required to have a unique name. # Create a function that accepts events coming from cloudwatchlogs. diff --git a/x-pack/functionbeat/docs/config-options-aws.asciidoc b/x-pack/functionbeat/docs/config-options-aws.asciidoc index dd52ef21ad1..7f0328127f0 100644 --- a/x-pack/functionbeat/docs/config-options-aws.asciidoc +++ b/x-pack/functionbeat/docs/config-options-aws.asciidoc @@ -1,5 +1,8 @@ [id="configuration-{beatname_lc}-options"] [role="xpack"] + +:libbeat-xpack-dir: ../../../x-pack/libbeat + == Configure AWS functions ++++ @@ -15,6 +18,11 @@ You configure the functions in the the +{beatname_lc}.yml+ configuration file. When you're done, you can <> to your serverless environment. +The `aws` functions require AWS credentials configuration in order to make AWS API calls. +Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or +`AWS_SESSION_TOKEN`, or use shared AWS credentials file. +Please see <> for more details. + The following example configures two functions: `cloudwatch` and `sqs`. The `cloudwatch` function collects events from CloudWatch Logs. The `sqs` function collects messages from Amazon Simple Queue Service (SQS). Both functions forward @@ -56,12 +64,6 @@ to deploy. TIP: If you change the configuration after deploying the function, use the <> to update your deployment. -[float] -[id="{beatname_lc}-endpoint"] -==== `provider.aws.endpoint` - -AWS endpoint to use in the URL template to load functions. - [float] [id="{beatname_lc}-deploy-bucket"] ==== `provider.aws.deploy_bucket` @@ -212,3 +214,6 @@ version and the event timestamp; for access to dynamic fields, use Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might expand to `"functionbeat-myindex-2019.12.13"`. + +[id="aws-credentials-config"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index f39ef09c08b..80642768b74 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -16,6 +16,18 @@ functionbeat.provider.aws.endpoint: "s3.amazonaws.com" # Configure which S3 bucket we should upload the lambda artifact. functionbeat.provider.aws.deploy_bucket: "functionbeat-deploy" +# Configure credentials of Functionbeat while deploying to AWS. +# Available options: +# * access_key_id, secret_access_key and/or session_token +#functionbeat.provider.aws.access_key_id: '${AWS_ACCESS_KEY_ID:""}' +#functionbeat.provider.aws.secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' +#functionbeat.provider.aws.session_token: '${AWS_SESSION_TOKEN:""}' +# * role_arn +#functionbeat.provider.aws.role_arn: arn:aws:iam::123456789012:role/test-fnb +# * credential_profile_name and/or shared_credential_file +#functionbeat.provider.aws.credential_profile_name: fnb-aws +#functionbeat.provider.aws.shared_credential_file: /etc/functionbeat/aws_credentials + functionbeat.provider.aws.functions: # Define the list of function availables, each function required to have a unique name. # Create a function that accepts events coming from cloudwatchlogs. diff --git a/x-pack/functionbeat/manager/aws/cli_manager.go b/x-pack/functionbeat/manager/aws/cli_manager.go index 07479fc18c5..a19cb22823d 100644 --- a/x-pack/functionbeat/manager/aws/cli_manager.go +++ b/x-pack/functionbeat/manager/aws/cli_manager.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/external" cf "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/iam" @@ -24,6 +23,7 @@ import ( "github.com/elastic/beats/v7/x-pack/functionbeat/manager/core" "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" fnaws "github.com/elastic/beats/v7/x-pack/functionbeat/provider/aws/aws" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) const ( @@ -67,6 +67,12 @@ func (c *CLIManager) deployTemplate(update bool, name string) error { } c.log.Debugf("Using cloudformation template:\n%s", templateData.json) + + _, err = c.awsCfg.Credentials.Retrieve() + if err != nil { + return fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %+v", err) + } + svcCF := cf.New(c.awsCfg) executer := executor.NewExecutor(c.log) @@ -144,6 +150,11 @@ func (c *CLIManager) Remove(name string) error { c.log.Debugf("Removing function: %s", name) defer c.log.Debugf("Removal of function '%s' complete", name) + _, err := c.awsCfg.Credentials.Retrieve() + if err != nil { + return fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %+v", err) + } + svc := cf.New(c.awsCfg) executer := executor.NewExecutor(c.log) executer.Add(newOpDeleteCloudFormation(c.log, svc, c.stackName(name))) @@ -199,15 +210,14 @@ func NewCLI( cfg *common.Config, provider provider.Provider, ) (provider.CLIManager, error) { - awsCfg, err := external.LoadDefaultAWSConfig() - if err != nil { - return nil, err - } - - config := &fnaws.Config{} + config := fnaws.DefaultConfig() if err := cfg.Unpack(config); err != nil { return nil, err } + awsCfg, err := awscommon.GetAWSCredentials(config.Credentials) + if err != nil { + return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %+v", err) + } builder, err := provider.TemplateBuilder() if err != nil { diff --git a/x-pack/functionbeat/manager/aws/template_builder.go b/x-pack/functionbeat/manager/aws/template_builder.go index d28a75b3794..ed3b8a75a61 100644 --- a/x-pack/functionbeat/manager/aws/template_builder.go +++ b/x-pack/functionbeat/manager/aws/template_builder.go @@ -66,7 +66,7 @@ func NewTemplateBuilder(log *logp.Logger, cfg *common.Config, p provider.Provide return &defaultTemplateBuilder{ provider: p, log: log, - endpoint: config.Endpoint, + endpoint: config.Credentials.Endpoint, bucket: string(config.DeployBucket), }, nil } diff --git a/x-pack/functionbeat/provider/aws/aws/config.go b/x-pack/functionbeat/provider/aws/aws/config.go index 932b8a1bc52..24053b37111 100644 --- a/x-pack/functionbeat/provider/aws/aws/config.go +++ b/x-pack/functionbeat/provider/aws/aws/config.go @@ -15,12 +15,28 @@ import ( "github.com/dustin/go-humanize" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) // Config expose the configuration option the AWS provider. type Config struct { - Endpoint string `config:"endpoint" validate:"nonzero,required"` - DeployBucket bucket `config:"deploy_bucket" validate:"nonzero,required"` + DeployBucket bucket `config:"deploy_bucket" validate:"nonzero,required"` + Credentials awscommon.ConfigAWS `config:",inline"` +} + +func DefaultConfig() *Config { + return &Config{ + Credentials: awscommon.ConfigAWS{ + Endpoint: "s3.amazonaws.com", + }, + } +} + +func (c *Config) Validate() error { + if c.Credentials.Endpoint == "" { + return fmt.Errorf("functionbeat.providers.aws.enpoint cannot be empty") + } + return nil } // maxMegabytes maximums memory that a lambda can use. diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index db661dcc627..a9cb4ab8e88 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -38,12 +38,26 @@ given, the default profile will be used. `shared_credential_file` is optional to specify the directory of your shared credentials file. If it's empty, the default directory will be used. In Windows, shared credentials file is at `C:\Users\\.aws\credentials`. -For Linux, macOS or Unix, the file is located at `~/.aws/credentials`. When running as a service, +For Linux, macOS or Unix, the file is located at `~/.aws/credentials`. When running as a service, the home path depends on the user that manages the service, so the `shared_credential_file` parameter can be used to avoid ambiguity. Please see https://docs.aws.amazon.com/ses/latest/DeveloperGuide/create-shared-credentials-file.html[Create Shared Credentials File] for more details. -include::../../../{beatname_lc}/docs/aws-credentials-examples.asciidoc[] +ifeval::["{beatname_lc}"=="filebeat"] +include::../../../filebeat/docs/aws-credentials-examples.asciidoc[] +endif::[] + +ifeval::["{beatname_lc}"=="heartbeat"] +include::../../../heartbeat/docs/aws-credentials-examples.asciidoc[] +endif::[] + +ifeval::["{beatname_lc}"=="metricbeat"] +include::../../../metricbeat/docs/aws-credentials-examples.asciidoc[] +endif::[] + +ifeval::["{beatname_lc}"=="functionbeat"] +include::../../../filebeat/docs/aws-credentials-examples.asciidoc[] +endif::[] [float] ==== AWS Credentials Types