-
Notifications
You must be signed in to change notification settings - Fork 2.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[receiver/awscloudwatch] New Alpha Logs Component `awscloudwatchrecei…
…ver` (#14730) Adds a new awscloudwatch receiver that polls events against the AWS cloudwatch API. It uses the AWS Profile based authentication.
- Loading branch information
Showing
29 changed files
with
2,193 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' | ||
| change_type: new_component | ||
|
|
||
| # The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) | ||
| component: awscloudwatchreceiver | ||
|
|
||
| # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). | ||
| note: Adds the `awscloudwatchreceiver` in an alpha state | ||
|
|
||
| # One or more tracking issues related to the change | ||
| issues: [14449] | ||
|
|
||
| # (Optional) One or more lines of additional information to render under the primary note. | ||
| # These lines will be padded with 2 spaces and then inserted directly into the document. | ||
| # Use pipe (|) for multiline entries. | ||
| subtext: |
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include ../../Makefile.Common |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| # Cloudwatch Receiver | ||
|
|
||
| | Status | | | ||
| | ------------------------ | --------- | | ||
| | Stability | [alpha] | | ||
| | Supported pipeline types | logs | | ||
| | Distributions | [contrib] | | ||
|
|
||
| Receives Cloudwatch events from [AWS Cloudwatch](https://aws.amazon.com/cloudwatch/) via the [AWS SDK for Cloudwatch Logs](https://docs.aws.amazon.com/sdk-for-go/api/service/cloudwatchlogs/) | ||
|
|
||
| ## Getting Started | ||
|
|
||
| This receiver uses the [AWS SDK Profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) as mode of authentication. | ||
|
|
||
| ## Configuration | ||
|
|
||
| ### Top Level Parameters | ||
|
|
||
| | Parameter | Notes | type | Description | | ||
| | --------------- | ---------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ||
| | `region` | *required* | string | The AWS recognized region string | | ||
| | `profile` | *optional* | string | The AWS profile used to authenticate, if none is specified the default is chosen from the list of profiles | | ||
| | `imds_endpoint` | *optional* | string | A way of specifying a custom URL to be used by the EC2 IMDS client to validate the session. If unset, and the environment variable `AWS_EC2_METADATA_SERVICE_ENDPOINT` has a value the client will use the value of the environment variable as the endpoint for operation calls. | | ||
| | `logs` | *optional* | `Logs` | Configuration for Logs ingestion of this receiver | | ||
|
|
||
| ### Logs Parameters | ||
|
|
||
| | Parameter | Notes | type | Description | | ||
| | ------------------------ | ------------ | ---------------------- | ------------------------------------------------------------------------------------------ | | ||
| | `poll_interval` | `default=1m` | duration | The duration waiting in between requests. | | ||
| | `max_events_per_request` | `default=50` | int | The maximum number of events to process per request to Cloudwatch | | ||
| | `groups` | *optional* | `See Group Parameters` | Configuration for Log Groups, by default all Log Groups and Log Streams will be collected. | | ||
|
|
||
| ### Group Parameters | ||
|
|
||
| `autodiscover` and `named` are ways to control and filter which log groups and log streams which are collected from. They are mutually exclusive and are incompatible to be configured at the same time. | ||
|
|
||
| - `autodiscover` | ||
| - `limit`: (optional; default = 50) Limits the number of discovered log groups. | ||
| - `prefix`: (optional) A prefix for log groups to limit the number of log groups discovered. | ||
| - if omitted, all log streams up to the limit are collected from | ||
| - `streams`: (optional) If `streams` is omitted, then all streams will be attempted to retrieve events from. | ||
| - `names`: A list of full log stream names to filter the discovered log groups to collect from. | ||
| - `prefixes`: A list of prefixes to filter the discovered log groups to collect from. | ||
| - `named` | ||
| - This is a map of log group name to stream filtering options | ||
| - `streams`: (optional) | ||
| - `names`: A list of full log stream names to filter the discovered log groups to collect from. | ||
| - `prefixes`: A list of prefixes to filter the discovered log groups to collect from. | ||
|
|
||
| #### Autodiscovery Example Configuration | ||
|
|
||
| ```yaml | ||
| awscloudwatch: | ||
| region: us-west-1 | ||
| logs: | ||
| poll_interval: 1m | ||
| groups: | ||
| autodiscover: | ||
| limit: 100 | ||
| prefix: /aws/eks/ | ||
| streams: | ||
| prefixes: [kube-api-controller] | ||
| ``` | ||
| #### Named Example | ||
| ```yaml | ||
| awscloudwatch: | ||
| region: us-west-1 | ||
| logs: | ||
| poll_interval: 5m | ||
| groups: | ||
| named: | ||
| /aws/eks/dev-0/cluster: | ||
| streams: | ||
| names: [kube-apiserver-ea9c831555adca1815ae04b87661klasdj] | ||
| ``` | ||
| ## Sample Configs | ||
| This receiver has a number of sample configs for reference. | ||
| 1. [Default](./testdata/sample-configs/default.yaml) | ||
| - Minimal configuration of the receiver | ||
| - Performs autodiscovery | ||
| - Collects all log groups and log streams | ||
| 2. [Autodiscover Filtering Log Groups](./testdata/sample-configs/autodiscover-filter-groups.yaml) | ||
| - Performs autodiscovery | ||
| - Only collects log groups matching a prefix | ||
| - Limits the number of discovered Log Groups | ||
| 3. [Autodiscover Filtering Log Streams](./testdata/sample-configs/autodiscover-filter-streams.yaml) | ||
| - Performs autodiscovery for all Log Groups | ||
| - Filters log streams | ||
| 4. [Named Groups](./testdata/sample-configs/named-prefix.yaml) | ||
| - Specifies and only collects from the desired Log Groups | ||
| - Does not attempt autodiscovery | ||
| 5. [Named Groups Filter Log Streams](./testdata/sample-configs/named-prefix-streams.yaml) | ||
| - Specifies the names of the log groups to collect | ||
| - Does not attempt autodiscovery | ||
| - Only collects from log streams matching a prefix |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package awscloudwatchreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver" | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "go.opentelemetry.io/collector/config" | ||
| "go.uber.org/multierr" | ||
| ) | ||
|
|
||
| var ( | ||
| defaultPollInterval = time.Minute | ||
| defaultEventLimit = 1000 | ||
| defaultLogGroupLimit = 50 | ||
| ) | ||
|
|
||
| // Config is the overall config structure for the awscloudwatchreceiver | ||
| type Config struct { | ||
| config.ReceiverSettings `mapstructure:",squash"` | ||
| Region string `mapstructure:"region"` | ||
| Profile string `mapstructure:"profile"` | ||
| IMDSEndpoint string `mapstructure:"imds_endpoint"` | ||
| Logs *LogsConfig `mapstructure:"logs"` | ||
| } | ||
|
|
||
| // LogsConfig is the configuration for the logs portion of this receiver | ||
| type LogsConfig struct { | ||
| PollInterval time.Duration `mapstructure:"poll_interval"` | ||
| MaxEventsPerRequest int `mapstructure:"max_events_per_request"` | ||
| Groups GroupConfig `mapstructure:"groups"` | ||
| } | ||
|
|
||
| // GroupConfig is the configuration for log group collection | ||
| type GroupConfig struct { | ||
| AutodiscoverConfig *AutodiscoverConfig `mapstructure:"autodiscover,omitempty"` | ||
| NamedConfigs map[string]StreamConfig `mapstructure:"named"` | ||
| } | ||
|
|
||
| // AutodiscoverConfig is the configuration for the autodiscovery functionality of log groups | ||
| type AutodiscoverConfig struct { | ||
| Prefix string `mapstructure:"prefix"` | ||
| Limit int `mapstructure:"limit"` | ||
| Streams StreamConfig `mapstructure:"streams"` | ||
| } | ||
|
|
||
| // StreamConfig represents the configuration for the log stream filtering | ||
| type StreamConfig struct { | ||
| Prefixes []*string `mapstructure:"prefixes"` | ||
| Names []*string `mapstructure:"names"` | ||
| } | ||
|
|
||
| var ( | ||
| errNoRegion = errors.New("no region was specified") | ||
| errNoLogsConfigured = errors.New("no logs configured") | ||
| errInvalidEventLimit = errors.New("event limit is improperly configured, value must be greater than 0") | ||
| errInvalidPollInterval = errors.New("poll interval is incorrect, it must be a duration greater than one second") | ||
| errInvalidAutodiscoverLimit = errors.New("the limit of autodiscovery of log groups is improperly configured, value must be greater than 0") | ||
| errAutodiscoverAndNamedConfigured = errors.New("both autodiscover and named configs are configured, Only one or the other is permitted") | ||
| ) | ||
|
|
||
| // Validate validates all portions of the relevant config | ||
| func (c *Config) Validate() error { | ||
| if c.Region == "" { | ||
| return errNoRegion | ||
| } | ||
|
|
||
| if c.IMDSEndpoint != "" { | ||
| _, err := url.ParseRequestURI(c.IMDSEndpoint) | ||
| if err != nil { | ||
| return fmt.Errorf("unable to parse URI for imds_endpoint: %w", err) | ||
| } | ||
| } | ||
|
|
||
| var errs error | ||
| errs = multierr.Append(errs, c.ReceiverSettings.Validate()) | ||
| errs = multierr.Append(errs, c.validateLogsConfig()) | ||
| return errs | ||
| } | ||
|
|
||
| func (c *Config) validateLogsConfig() error { | ||
| if c.Logs == nil { | ||
| return errNoLogsConfigured | ||
| } | ||
|
|
||
| if c.Logs.MaxEventsPerRequest <= 0 { | ||
| return errInvalidEventLimit | ||
| } | ||
| if c.Logs.PollInterval < time.Second { | ||
| return errInvalidPollInterval | ||
| } | ||
|
|
||
| return c.Logs.Groups.validate() | ||
| } | ||
|
|
||
| func (c *GroupConfig) validate() error { | ||
| if c.AutodiscoverConfig != nil && len(c.NamedConfigs) > 0 { | ||
| return errAutodiscoverAndNamedConfigured | ||
| } | ||
|
|
||
| if c.AutodiscoverConfig != nil { | ||
| return validateAutodiscover(*c.AutodiscoverConfig) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func validateAutodiscover(cfg AutodiscoverConfig) error { | ||
| if cfg.Limit <= 0 { | ||
| return errInvalidAutodiscoverLimit | ||
| } | ||
| return nil | ||
| } |
Oops, something went wrong.