From e9b1c9df3d003b3b9bff6064cfc0083c74119812 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 22 Sep 2021 13:48:59 +0200 Subject: [PATCH] Remove old httpjson config implementation --- CHANGELOG.next.asciidoc | 1 + .../docs/inputs/input-httpjson.asciidoc | 91 --- x-pack/filebeat/input/httpjson/config.go | 189 +----- .../httpjson/{internal/v2 => }/config_auth.go | 2 +- .../filebeat/input/httpjson/config_oauth.go | 225 ------ .../input/httpjson/config_oauth_test.go | 94 --- .../{internal/v2 => }/config_request.go | 2 +- .../{internal/v2 => }/config_response.go | 2 +- x-pack/filebeat/input/httpjson/config_test.go | 291 ++++---- .../httpjson/{internal/v2 => }/cursor.go | 2 +- .../httpjson/{internal/v2 => }/cursor_test.go | 2 +- x-pack/filebeat/input/httpjson/date_cursor.go | 108 --- .../httpjson/{internal/v2 => }/encoding.go | 2 +- .../{internal/v2 => }/encoding_test.go | 2 +- x-pack/filebeat/input/httpjson/input.go | 166 +++-- .../filebeat/input/httpjson/input_cursor.go | 23 +- .../filebeat/input/httpjson/input_manager.go | 58 +- .../input/httpjson/input_stateless.go | 10 +- x-pack/filebeat/input/httpjson/input_test.go | 338 +++++++-- .../input/httpjson/internal/v2/config.go | 64 -- .../input/httpjson/internal/v2/config_test.go | 394 ----------- .../input/httpjson/internal/v2/input.go | 215 ------ .../httpjson/internal/v2/input_cursor.go | 56 -- .../httpjson/internal/v2/input_manager.go | 66 -- .../httpjson/internal/v2/input_stateless.go | 52 -- .../input/httpjson/internal/v2/input_test.go | 641 ------------------ .../input/httpjson/internal/v2/pagination.go | 184 ----- .../httpjson/internal/v2/rate_limiter.go | 141 ---- .../httpjson/internal/v2/rate_limiter_test.go | 90 --- .../internal/v2/testdata/credentials.json | 7 - .../v2/testdata/invalid_credentials.json | 1 - x-pack/filebeat/input/httpjson/pagination.go | 225 +++--- .../input/httpjson/pagination_test.go | 77 --- .../filebeat/input/httpjson/rate_limiter.go | 55 +- .../input/httpjson/rate_limiter_test.go | 80 ++- .../httpjson/{internal/v2 => }/request.go | 2 +- .../{internal/v2 => }/request_test.go | 2 +- x-pack/filebeat/input/httpjson/requester.go | 299 -------- .../filebeat/input/httpjson/requester_test.go | 86 --- .../httpjson/{internal/v2 => }/response.go | 2 +- .../{internal/v2 => }/response_test.go | 2 +- .../input/httpjson/{internal/v2 => }/split.go | 2 +- .../httpjson/{internal/v2 => }/split_test.go | 2 +- .../httpjson/{internal/v2 => }/transform.go | 2 +- .../{internal/v2 => }/transform_append.go | 2 +- .../v2 => }/transform_append_test.go | 2 +- .../{internal/v2 => }/transform_delete.go | 2 +- .../v2 => }/transform_delete_test.go | 2 +- .../{internal/v2 => }/transform_registry.go | 2 +- .../{internal/v2 => }/transform_set.go | 2 +- .../{internal/v2 => }/transform_set_test.go | 2 +- .../{internal/v2 => }/transform_target.go | 2 +- .../v2 => }/transform_target_test.go | 2 +- .../{internal/v2 => }/transform_test.go | 2 +- .../httpjson/{internal/v2 => }/value_tpl.go | 2 +- .../{internal/v2 => }/value_tpl_test.go | 2 +- .../module/cisco/amp/config/config.yml | 1 - .../google_workspace/admin/config/config.yml | 1 - .../google_workspace/drive/config/config.yml | 1 - .../google_workspace/groups/config/config.yml | 1 - .../google_workspace/login/config/config.yml | 1 - .../google_workspace/saml/config/config.yml | 1 - .../user_accounts/config/config.yml | 1 - .../module/misp/threat/config/input.yml | 1 - .../module/okta/system/config/input.yml | 1 - .../module/snyk/audit/config/config.yml | 1 - .../snyk/vulnerabilities/config/config.yml | 1 - .../abusemalware/config/config.yml | 1 - .../threatintel/abuseurl/config/config.yml | 1 - .../threatintel/anomali/config/config.yml | 1 - .../malwarebazaar/config/config.yml | 1 - .../module/threatintel/misp/config/config.yml | 1 - .../module/threatintel/otx/config/config.yml | 1 - .../recordedfuture/config/config.yml | 1 - 74 files changed, 834 insertions(+), 3563 deletions(-) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/config_auth.go (99%) delete mode 100644 x-pack/filebeat/input/httpjson/config_oauth.go delete mode 100644 x-pack/filebeat/input/httpjson/config_oauth_test.go rename x-pack/filebeat/input/httpjson/{internal/v2 => }/config_request.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/config_response.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/cursor.go (98%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/cursor_test.go (99%) delete mode 100644 x-pack/filebeat/input/httpjson/date_cursor.go rename x-pack/filebeat/input/httpjson/{internal/v2 => }/encoding.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/encoding_test.go (99%) delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/config.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/config_test.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_manager.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_test.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/pagination.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json delete mode 100644 x-pack/filebeat/input/httpjson/pagination_test.go rename x-pack/filebeat/input/httpjson/{internal/v2 => }/request.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/request_test.go (99%) delete mode 100644 x-pack/filebeat/input/httpjson/requester.go delete mode 100644 x-pack/filebeat/input/httpjson/requester_test.go rename x-pack/filebeat/input/httpjson/{internal/v2 => }/response.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/response_test.go (98%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/split.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/split_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_append.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_append_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_delete.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_delete_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_registry.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_set.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_set_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_target.go (98%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_target_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/transform_test.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/value_tpl.go (99%) rename x-pack/filebeat/input/httpjson/{internal/v2 => }/value_tpl_test.go (99%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 812f5d15e21..1aa74ca99a3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -91,6 +91,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762] - Remove deprecated fields in Kafka module. {pull}27938[27938] - Remove deprecated fields in coredns module. {pull}28196[28196] +- Remove old `httpjson` config implementation. {pull}28054[28054] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 665ce3eb8ba..b0b2852c74b 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -31,7 +31,6 @@ Example configurations: filebeat.inputs: # Fetch your public IP every minute. - type: httpjson - config_version: 2 interval: 1m request.url: https://api.ipify.org/?format=json processors: @@ -44,7 +43,6 @@ filebeat.inputs: ---- filebeat.inputs: - type: httpjson - config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST response.split: @@ -69,7 +67,6 @@ Example configurations with authentication: ---- filebeat.inputs: - type: httpjson - config_version: 2 request.url: http://localhost request.transforms: - set: @@ -81,7 +78,6 @@ filebeat.inputs: ---- filebeat.inputs: - type: httpjson - config_version: 2 auth.oauth2: client.id: 12345678901234567890abcdef client.secret: abcdef12345678901234567890 @@ -217,13 +213,6 @@ In addition to the provided functions, any of the native functions for https://g The `httpjson` input supports the following configuration options plus the <<{beatname_lc}-input-{type}-common-options>> described later. -[float] -==== `config_version` - -Defines the configuration version. Current supported versions are: `1` and `2`. Default: `1`. - -NOTE: This setting defaults to `1` to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in future releases. Any new configuration should use `config_version: 2`. - [float] ==== `interval` @@ -296,7 +285,6 @@ Can be set for all providers except `google`. ["source","yaml",subs="attributes"] ---- - type: httpjson - config_version: 2 auth.oauth2: endpoint_params: Param1: @@ -374,7 +362,6 @@ Defaults to `null` (no HTTP body). ["source","yaml",subs="attributes"] ---- - type: httpjson - config_version: 2 request.method: POST request.body: query: @@ -406,7 +393,6 @@ This specifies proxy configuration in the form of `http[s]://:@< filebeat.inputs: # Fetch your public IP every minute. - type: httpjson - config_version: 2 interval: 1m request.url: https://api.ipify.org/?format=json request.proxy_url: http://proxy.example:8080 @@ -474,7 +460,6 @@ Can write state to: [`body.*`, `header.*`, `url.*`]. ---- filebeat.inputs: - type: httpjson - config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST request.transforms: @@ -504,7 +489,6 @@ Can write state to: [`body.*`]. ---- filebeat.inputs: - type: httpjson - config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST response.transforms: @@ -639,7 +623,6 @@ The config will look like: ---- filebeat.inputs: - type: httpjson - config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -735,7 +718,6 @@ The config will look like: ---- filebeat.inputs: - type: httpjson - config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -810,7 +792,6 @@ The config will look like: ---- filebeat.inputs: - type: httpjson - config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -862,7 +843,6 @@ The config will look like: ---- filebeat.inputs: - type: httpjson - config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -913,7 +893,6 @@ NOTE: Default templates do not have access to any state, only to functions. ---- filebeat.inputs: - type: httpjson - config_version: 2 interval: 1m request.url: https://api.ipify.org/?format=json response.transforms: @@ -930,76 +909,6 @@ filebeat.inputs: target: "json" ---- -[float] -==== `api_key` - -Deprecated, use `request.transforms`. - -[float] -==== `http_client_timeout` - -Deprecated, use `request.timeout`. - -[float] -==== `http_headers` - -Deprecated, use `request.transforms`. - -[float] -==== `http_method` - -Deprecated, use `request.method`. - -[float] -==== `http_request_body` - -Deprecated, use `request.body`. - -[float] -==== `json_objects_array` - -Deprecated, use `request.split`. - -[float] -==== `split_events_by` - -Deprecated, use `request.split`. - -[float] -==== `no_http_body` - -Deprecated. - -[float] -==== `pagination.*` - -Deprecated, use `response.pagination`. - -[float] -==== `rate_limit.*` - -Deprecated, use `request.rate_limit.*`. - -[float] -==== `retry.*` - -Deprecated, use `request.retry.*`. - -[float] -==== `ssl` - -Deprecated, use `request.ssl`. - -[float] -==== `url` - -Deprecated, use `request.url`. - -[float] -==== `oauth2.*` - -Deprecated, use `auth.oauth2.*`. - ==== Request life cycle image:images/input-httpjson-lifecycle.png[Request lifecycle] diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index a2965ace4ef..0964cc2d36e 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -6,174 +6,59 @@ package httpjson import ( "errors" - "fmt" - "net/url" - "regexp" - "strings" - "text/template" "time" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" ) -// config contains information about httpjson configuration type config struct { - OAuth2 *oauth2Config `config:"oauth2"` - APIKey string `config:"api_key"` - AuthenticationScheme string `config:"authentication_scheme"` - HTTPClientTimeout time.Duration `config:"http_client_timeout"` - HTTPHeaders common.MapStr `config:"http_headers"` - HTTPMethod string `config:"http_method" validate:"required"` - HTTPRequestBody common.MapStr `config:"http_request_body"` - Interval time.Duration `config:"interval"` - JSONObjects string `config:"json_objects_array"` - SplitEventsBy string `config:"split_events_by"` - NoHTTPBody bool `config:"no_http_body"` - Pagination *paginationConfig `config:"pagination"` - RateLimit *rateLimitConfig `config:"rate_limit"` - RetryMax int `config:"retry.max_attempts"` - RetryWaitMin time.Duration `config:"retry.wait_min"` - RetryWaitMax time.Duration `config:"retry.wait_max"` - URL *urlConfig `config:"url" validate:"required"` - DateCursor *dateCursorConfig `config:"date_cursor"` - - Transport httpcommon.HTTPTransportSettings `config:",inline"` -} - -// Pagination contains information about httpjson pagination settings -type paginationConfig struct { - Enabled *bool `config:"enabled"` - ExtraBodyContent common.MapStr `config:"extra_body_content"` - Header *headerConfig `config:"header"` - IDField string `config:"id_field"` - RequestField string `config:"req_field"` - URLField string `config:"url_field"` - URL string `config:"url"` -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (p *paginationConfig) isEnabled() bool { - return p != nil && (p.Enabled == nil || *p.Enabled) + Interval time.Duration `config:"interval" validate:"required"` + Auth *authConfig `config:"auth"` + Request *requestConfig `config:"request" validate:"required"` + Response *responseConfig `config:"response"` + Cursor cursorConfig `config:"cursor"` } -// HTTP Header information for pagination -type headerConfig struct { - FieldName string `config:"field_name" validate:"required"` - RegexPattern *regexp.Regexp `config:"regex_pattern" validate:"required"` -} - -// HTTP Header Rate Limit information -type rateLimitConfig struct { - Limit string `config:"limit"` - Reset string `config:"reset"` - Remaining string `config:"remaining"` -} +type cursorConfig map[string]cursorEntry -type dateCursorConfig struct { - Enabled *bool `config:"enabled"` - Field string `config:"field"` - URLField string `config:"url_field" validate:"required"` - ValueTemplate *templateConfig `config:"value_template"` - DateFormat string `config:"date_format"` - InitialInterval time.Duration `config:"initial_interval"` +type cursorEntry struct { + Value *valueTpl `config:"value"` + Default *valueTpl `config:"default"` + IgnoreEmptyValue *bool `config:"ignore_empty_value"` } -type templateConfig struct { - *template.Template +func (ce cursorEntry) mustIgnoreEmptyValue() bool { + return ce.IgnoreEmptyValue == nil || *ce.IgnoreEmptyValue } -func (t *templateConfig) Unpack(in string) error { - tpl, err := template.New("tpl").Parse(in) - if err != nil { - return err +func (c config) Validate() error { + if c.Interval <= 0 { + return errors.New("interval must be greater than 0") } - - *t = templateConfig{Template: tpl} - return nil } -type urlConfig struct { - *url.URL -} - -func (u *urlConfig) Unpack(in string) error { - parsed, err := url.Parse(in) - if err != nil { - return err +func defaultConfig() config { + maxAttempts := 5 + waitMin := time.Second + waitMax := time.Minute + transport := httpcommon.DefaultHTTPTransportSettings() + transport.Timeout = 30 * time.Second + + return config{ + Interval: time.Minute, + Auth: &authConfig{}, + Request: &requestConfig{ + Method: "GET", + Retry: retryConfig{ + MaxAttempts: &maxAttempts, + WaitMin: &waitMin, + WaitMax: &waitMax, + }, + RedirectForwardHeaders: false, + RedirectMaxRedirects: 10, + Transport: transport, + }, + Response: &responseConfig{}, } - - *u = urlConfig{URL: parsed} - - return nil -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (dc *dateCursorConfig) isEnabled() bool { - return dc != nil && (dc.Enabled == nil || *dc.Enabled) -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (dc *dateCursorConfig) getDateFormat() string { - if dc.DateFormat == "" { - return time.RFC3339 - } - return dc.DateFormat -} - -func (dc *dateCursorConfig) Validate() error { - if dc.DateFormat == "" { - return nil - } - - const knownTimestamp = 1602601228 // 2020-10-13T15:00:28+00:00 RFC3339 - knownDate := time.Unix(knownTimestamp, 0).UTC() - - dateStr := knownDate.Format(dc.DateFormat) - if _, err := time.Parse(dc.DateFormat, dateStr); err != nil { - return errors.New("invalid configuration: date_format is not a valid date layout") - } - - return nil -} - -func (c *config) Validate() error { - switch strings.ToUpper(c.HTTPMethod) { - case "GET", "POST": - break - default: - return fmt.Errorf("httpjson input: Invalid http_method, %s", c.HTTPMethod) - } - if c.NoHTTPBody { - if len(c.HTTPRequestBody) > 0 { - return errors.New("invalid configuration: both no_http_body and http_request_body cannot be set simultaneously") - } - if c.Pagination != nil && (len(c.Pagination.ExtraBodyContent) > 0 || c.Pagination.RequestField != "") { - return errors.New("invalid configuration: both no_http_body and pagination.extra_body_content or pagination.req_field cannot be set simultaneously") - } - } - if c.Pagination != nil { - if c.Pagination.Header != nil { - if c.Pagination.RequestField != "" || c.Pagination.IDField != "" || len(c.Pagination.ExtraBodyContent) > 0 { - return errors.New("invalid configuration: both pagination.header and pagination.req_field or pagination.id_field or pagination.extra_body_content cannot be set simultaneously") - } - } - } - if c.OAuth2.isEnabled() { - if c.APIKey != "" || c.AuthenticationScheme != "" { - return errors.New("invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously") - } - } - return nil -} - -func newDefaultConfig() config { - var c config - c.HTTPMethod = "GET" - c.HTTPClientTimeout = 60 * time.Second - c.RetryWaitMin = 1 * time.Second - c.RetryWaitMax = 60 * time.Second - c.RetryMax = 5 - return c } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/config_auth.go rename to x-pack/filebeat/input/httpjson/config_auth.go index 0d862ea181b..88eac44edc1 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "context" diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go deleted file mode 100644 index 6c31aa135b6..00000000000 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" - "golang.org/x/oauth2/endpoints" - "golang.org/x/oauth2/google" - - "github.com/elastic/beats/v7/libbeat/common" -) - -// An oauth2Provider represents a supported oauth provider. -type oauth2Provider string - -const ( - oauth2ProviderDefault oauth2Provider = "" // OAuth2ProviderDefault means no specific provider is set. - oauth2ProviderAzure oauth2Provider = "azure" // OAuth2ProviderAzure AzureAD. - oauth2ProviderGoogle oauth2Provider = "google" // OAuth2ProviderGoogle Google. -) - -func (p *oauth2Provider) Unpack(in string) error { - *p = oauth2Provider(in) - return nil -} - -func (p oauth2Provider) canonical() oauth2Provider { - return oauth2Provider(strings.ToLower(string(p))) -} - -// oauth2Config contains information about oauth2 authentication settings. -type oauth2Config struct { - // common oauth fields - ClientID string `config:"client.id"` - ClientSecret string `config:"client.secret"` - Enabled *bool `config:"enabled"` - EndpointParams map[string][]string `config:"endpoint_params"` - Provider oauth2Provider `config:"provider"` - Scopes []string `config:"scopes"` - TokenURL string `config:"token_url"` - - // google specific - GoogleCredentialsFile string `config:"google.credentials_file"` - GoogleCredentialsJSON common.JSONBlob `config:"google.credentials_json"` - GoogleJWTFile string `config:"google.jwt_file"` - GoogleDelegatedAccount string `config:"google.delegated_account"` - - // microsoft azure specific - AzureTenantID string `config:"azure.tenant_id"` - AzureResource string `config:"azure.resource"` -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (o *oauth2Config) isEnabled() bool { - return o != nil && (o.Enabled == nil || *o.Enabled) -} - -// Client wraps the given http.Client and returns a new one that will use the oauth authentication. -func (o *oauth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { - // only required to let oauth2 library to find our custom client in the context - ctx = context.WithValue(context.Background(), oauth2.HTTPClient, client) - - switch o.getProvider() { - case oauth2ProviderAzure, oauth2ProviderDefault: - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.getTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.getEndpointParams(), - } - return creds.Client(ctx), nil - case oauth2ProviderGoogle: - if o.GoogleJWTFile != "" { - cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error loading jwt credentials: %w", err) - } - cfg.Subject = o.GoogleDelegatedAccount - return cfg.Client(ctx), nil - } - - creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) - } - return oauth2.NewClient(ctx, creds.TokenSource), nil - default: - return nil, errors.New("oauth2 client: unknown provider") - } -} - -// GetTokenURL returns the TokenURL. -func (o *oauth2Config) getTokenURL() string { - switch o.getProvider() { - case oauth2ProviderAzure: - if o.TokenURL == "" { - return endpoints.AzureAD(o.AzureTenantID).TokenURL - } - } - - return o.TokenURL -} - -// GetProvider returns provider in its canonical form. -func (o oauth2Config) getProvider() oauth2Provider { - return o.Provider.canonical() -} - -// GetEndpointParams returns endpoint params with any provider ones combined. -func (o oauth2Config) getEndpointParams() map[string][]string { - switch o.getProvider() { - case oauth2ProviderAzure: - if o.AzureResource != "" { - if o.EndpointParams == nil { - o.EndpointParams = map[string][]string{} - } - o.EndpointParams["resource"] = []string{o.AzureResource} - } - } - - return o.EndpointParams -} - -// Validate checks if oauth2 config is valid. -func (o *oauth2Config) Validate() error { - switch o.getProvider() { - case oauth2ProviderAzure: - return o.validateAzureProvider() - case oauth2ProviderGoogle: - return o.validateGoogleProvider() - case oauth2ProviderDefault: - if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { - return errors.New("invalid configuration: both token_url and client credentials must be provided") - } - default: - return fmt.Errorf("invalid configuration: unknown provider %q", o.getProvider()) - } - return nil -} - -// findDefaultGoogleCredentials will default to google.FindDefaultCredentials and will only be changed for testing purposes -var findDefaultGoogleCredentials = google.FindDefaultCredentials - -func (o *oauth2Config) validateGoogleProvider() error { - if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || - o.AzureTenantID != "" || o.AzureResource != "" || len(o.EndpointParams) > 0 { - return errors.New("invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") - } - - // credentials_json - if len(o.GoogleCredentialsJSON) > 0 { - if o.GoogleDelegatedAccount != "" { - return errors.New("invalid configuration: google.delegated_account can only be provided with a jwt_file") - } - return nil - } - - // credentials_file - if o.GoogleCredentialsFile != "" { - if o.GoogleDelegatedAccount != "" { - return errors.New("invalid configuration: google.delegated_account can only be provided with a jwt_file") - } - return o.populateCredentialsJSONFromFile(o.GoogleCredentialsFile) - } - - // jwt_file - if o.GoogleJWTFile != "" { - return o.populateCredentialsJSONFromFile(o.GoogleJWTFile) - } - - // Application Default Credentials (ADC) - ctx := context.Background() - if creds, err := findDefaultGoogleCredentials(ctx, o.Scopes...); err == nil { - o.GoogleCredentialsJSON = creds.JSON - return nil - } - - return fmt.Errorf("invalid configuration: no authentication credentials were configured or detected (ADC)") -} - -func (o *oauth2Config) populateCredentialsJSONFromFile(file string) error { - if _, err := os.Stat(file); os.IsNotExist(err) { - return fmt.Errorf("invalid configuration: the file %q cannot be found", file) - } - - credBytes, err := ioutil.ReadFile(file) - if err != nil { - return fmt.Errorf("invalid configuration: the file %q cannot be read", file) - } - - if !json.Valid(credBytes) { - return fmt.Errorf("invalid configuration: the file %q does not contain valid JSON", file) - } - - o.GoogleCredentialsJSON = credBytes - - return nil -} - -func (o *oauth2Config) validateAzureProvider() error { - if o.TokenURL == "" && o.AzureTenantID == "" { - return errors.New("invalid configuration: at least one of token_url or tenant_id must be provided") - } - if o.TokenURL != "" && o.AzureTenantID != "" { - return errors.New("invalid configuration: only one of token_url and tenant_id can be used") - } - if o.ClientID == "" || o.ClientSecret == "" { - return errors.New("invalid configuration: client credentials must be provided") - } - - return nil -} diff --git a/x-pack/filebeat/input/httpjson/config_oauth_test.go b/x-pack/filebeat/input/httpjson/config_oauth_test.go deleted file mode 100644 index 67ec63b6650..00000000000 --- a/x-pack/filebeat/input/httpjson/config_oauth_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "reflect" - "testing" -) - -func TestProviderCanonical(t *testing.T) { - const ( - a oauth2Provider = "gOoGle" - b oauth2Provider = "google" - ) - - if a.canonical() != b.canonical() { - t.Fatal("Canonical provider should be equal") - } -} - -func TestGetProviderIsCanonical(t *testing.T) { - const expected oauth2Provider = "google" - - oauth2 := oauth2Config{Provider: "GOogle"} - if oauth2.getProvider() != expected { - t.Fatal("GetProvider should return canonical provider") - } -} - -func TestIsEnabled(t *testing.T) { - oauth2 := oauth2Config{} - if !oauth2.isEnabled() { - t.Fatal("OAuth2 should be enabled by default") - } - - var enabled = false - oauth2.Enabled = &enabled - - if oauth2.isEnabled() { - t.Fatal("OAuth2 should be disabled") - } - - enabled = true - if !oauth2.isEnabled() { - t.Fatal("OAuth2 should be enabled") - } -} - -func TestGetTokenURL(t *testing.T) { - const expected = "http://localhost" - oauth2 := oauth2Config{TokenURL: "http://localhost"} - if got := oauth2.getTokenURL(); got != expected { - t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) - } -} - -func TestGetTokenURLWithAzure(t *testing.T) { - const expectedWithoutTenantID = "http://localhost" - oauth2 := oauth2Config{TokenURL: "http://localhost", Provider: "azure"} - if got := oauth2.getTokenURL(); got != expectedWithoutTenantID { - t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) - } - - oauth2.TokenURL = "" - oauth2.AzureTenantID = "a_tenant_id" - const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" - if got := oauth2.getTokenURL(); got != expectedWithTenantID { - t.Fatalf("GetTokenURL should return the generated TokenURL but got %q", got) - } -} - -func TestGetEndpointParams(t *testing.T) { - var expected = map[string][]string{"foo": {"bar"}} - oauth2 := oauth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} - if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expected) { - t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) - } -} - -func TestGetEndpointParamsWithAzure(t *testing.T) { - var expectedWithoutResource = map[string][]string{"foo": {"bar"}} - oauth2 := oauth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} - if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expectedWithoutResource) { - t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) - } - - oauth2.AzureResource = "baz" - var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} - if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expectedWithResource) { - t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) - } -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/config_request.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/config_request.go rename to x-pack/filebeat/input/httpjson/config_request.go index a8bc7204a23..f5982e37d1a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/config_request.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "errors" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go b/x-pack/filebeat/input/httpjson/config_response.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/config_response.go rename to x-pack/filebeat/input/httpjson/config_response.go index 1bc3056ab17..669875a7265 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/config_response.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "fmt" diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 776c46bde0c..67662b40752 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -9,103 +9,89 @@ import ( "errors" "os" "testing" - "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/oauth2/google" "github.com/elastic/beats/v7/libbeat/common" ) -func TestConfigValidationCase1(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "http_request_body": map[string]interface{}{"test": "abc"}, - "no_http_body": true, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") - } +func TestProviderCanonical(t *testing.T) { + const ( + a oAuth2Provider = "gOoGle" + b oAuth2Provider = "google" + ) + + assert.Equal(t, a.canonical(), b.canonical()) } -func TestConfigValidationCase2(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") - } +func TestGetProviderIsCanonical(t *testing.T) { + const expected oAuth2Provider = "google" + + oauth2 := oAuth2Config{Provider: "GOogle"} + assert.Equal(t, expected, oauth2.getProvider()) } -func TestConfigValidationCase3(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") +func TestIsEnabled(t *testing.T) { + oauth2 := oAuth2Config{} + if !oauth2.isEnabled() { + t.Fatal("OAuth2 should be enabled by default") } + + var enabled = false + oauth2.Enabled = &enabled + + assert.False(t, oauth2.isEnabled()) + + enabled = true + + assert.True(t, oauth2.isEnabled()) } -func TestConfigValidationCase4(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") - } +func TestGetTokenURL(t *testing.T) { + const expected = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost"} + assert.Equal(t, expected, oauth2.getTokenURL()) } -func TestConfigValidationCase5(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "id_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") - } +func TestGetTokenURLWithAzure(t *testing.T) { + const expectedWithoutTenantID = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost", Provider: "azure"} + + assert.Equal(t, expectedWithoutTenantID, oauth2.getTokenURL()) + + oauth2.TokenURL = "" + oauth2.AzureTenantID = "a_tenant_id" + const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" + + assert.Equal(t, expectedWithTenantID, oauth2.getTokenURL()) } -func TestConfigValidationCase6(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") - } +func TestGetEndpointParams(t *testing.T) { + var expected = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} + assert.Equal(t, expected, oauth2.getEndpointParams()) +} + +func TestGetEndpointParamsWithAzure(t *testing.T) { + var expectedWithoutResource = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + + assert.Equal(t, expectedWithoutResource, oauth2.getEndpointParams()) + + oauth2.AzureResource = "baz" + var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} + + assert.Equal(t, expectedWithResource, oauth2.getEndpointParams()) } -func TestConfigValidationCase7(t *testing.T) { +func TestConfigFailsWithInvalidMethod(t *testing.T) { m := map[string]interface{}{ - "http_method": "DELETE", - "no_http_body": true, - "url": "localhost", + "request.method": "DELETE", } cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() + conf := defaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") } @@ -113,12 +99,12 @@ func TestConfigValidationCase7(t *testing.T) { func TestConfigMustFailWithInvalidURL(t *testing.T) { m := map[string]interface{}{ - "url": "::invalid::", + "request.url": "::invalid::", } cfg := common.MustNewConfigFrom(m) - conf := newDefaultConfig() + conf := defaultConfig() err := cfg.Unpack(&conf) - assert.EqualError(t, err, `parse "::invalid::": missing protocol scheme accessing 'url'`) + assert.EqualError(t, err, `parse "::invalid::": missing protocol scheme accessing 'request.url'`) } func TestConfigOauth2Validation(t *testing.T) { @@ -130,25 +116,26 @@ func TestConfigOauth2Validation(t *testing.T) { teardown func() }{ { - name: "can't set oauth2 and api_key together", - expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + name: "can't set oauth2 and basic auth together", + expectedErr: "only one kind of auth can be enabled accessing 'auth'", input: map[string]interface{}{ - "api_key": "an_api_key", - "oauth2": map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.oauth2": map[string]interface{}{ "token_url": "localhost", "client": map[string]interface{}{ "id": "a_client_id", "secret": "a_client_secret", }, }, - "url": "localhost", }, }, { - name: "can set oauth2 and api_key together if oauth2 is disabled", + name: "can set oauth2 and basic auth together if oauth2 is disabled", input: map[string]interface{}{ - "api_key": "an_api_key", - "oauth2": map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.oauth2": map[string]interface{}{ "enabled": false, "token_url": "localhost", "client": map[string]interface{}{ @@ -156,79 +143,58 @@ func TestConfigOauth2Validation(t *testing.T) { "secret": "a_client_secret", }, }, - "url": "localhost", - }, - }, - { - name: "can't set oauth2 and authentication_scheme", - expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", - input: map[string]interface{}{ - "authentication_scheme": "a_scheme", - "oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", - }, - }, - "url": "localhost", }, }, { name: "token_url and client credentials must be set", - expectedErr: "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'", + expectedErr: "both token_url and client credentials must be provided accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{}, - "url": "localhost", + "auth.oauth2": map[string]interface{}{}, }, }, { name: "must fail with an unknown provider", - expectedErr: "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'", + expectedErr: "unknown provider \"unknown\" accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "unknown", }, - "url": "localhost", }, }, { name: "azure must have either tenant_id or token_url", - expectedErr: "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'", + expectedErr: "at least one of token_url or tenant_id must be provided accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "azure", }, - "url": "localhost", }, }, { name: "azure must have only one of token_url and tenant_id", - expectedErr: "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'", + expectedErr: "only one of token_url and tenant_id can be used accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "azure", "azure.tenant_id": "a_tenant_id", "token_url": "localhost", }, - "url": "localhost", }, }, { name: "azure must have client credentials set", - expectedErr: "invalid configuration: client credentials must be provided accessing 'oauth2'", + expectedErr: "client credentials must be provided accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "azure", "azure.tenant_id": "a_tenant_id", }, - "url": "localhost", }, }, { name: "azure config is valid", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "azure", "azure": map[string]interface{}{ "tenant_id": "a_tenant_id", @@ -236,14 +202,13 @@ func TestConfigOauth2Validation(t *testing.T) { "client.id": "a_client_id", "client.secret": "a_client_secret", }, - "url": "localhost", }, }, { name: "google can't have token_url or client credentials set", - expectedErr: "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'", + expectedErr: "none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "azure": map[string]interface{}{ "tenant_id": "a_tenant_id", @@ -252,17 +217,15 @@ func TestConfigOauth2Validation(t *testing.T) { "client.secret": "a_client_secret", "token_url": "localhost", }, - "url": "localhost", }, }, { name: "google must fail if no ADC available", - expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", }, - "url": "localhost", }, setup: func() { // we change the default function to force a failure @@ -274,60 +237,55 @@ func TestConfigOauth2Validation(t *testing.T) { }, { name: "google must fail if credentials file not found", - expectedErr: "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'", + expectedErr: "the file \"./wrong\" cannot be found accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_file": "./wrong", }, - "url": "localhost", }, }, { name: "google must fail if ADC is wrongly set", - expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", }, - "url": "localhost", }, setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, }, { name: "google must work if ADC is set up", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", }, - "url": "localhost", }, setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, }, { name: "google must work if credentials_file is correct", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_file": "./testdata/credentials.json", }, - "url": "localhost", }, }, { name: "google must work if jwt_file is correct", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.jwt_file": "./testdata/credentials.json", }, - "url": "localhost", }, }, { name: "google must work if credentials_json is correct", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_json": `{ "type": "service_account", @@ -337,67 +295,47 @@ func TestConfigOauth2Validation(t *testing.T) { "client_id": "0" }`, }, - "url": "localhost", }, }, { name: "google must fail if credentials_json is not a valid JSON", - expectedErr: "the field can't be converted to valid JSON accessing 'oauth2.google.credentials_json'", + expectedErr: "the field can't be converted to valid JSON accessing 'auth.oauth2.google.credentials_json'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_json": `invalid`, }, - "url": "localhost", }, }, { name: "google must fail if the provided credentials file is not a valid JSON", - expectedErr: "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'", + expectedErr: "the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_file": "./testdata/invalid_credentials.json", }, - "url": "localhost", - }, - }, - { - name: "date_cursor.date_format will fail if invalid", - expectedErr: "invalid configuration: date_format is not a valid date layout accessing 'date_cursor'", - input: map[string]interface{}{ - "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": "1234"}, - "url": "localhost", - }, - }, - { - name: "date_cursor must work with a valid date_format", - input: map[string]interface{}{ - "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": time.RFC3339}, - "url": "localhost", }, }, { name: "google must fail if the delegated_account is set without jwt_file", - expectedErr: "invalid configuration: google.delegated_account can only be provided with a jwt_file accessing 'oauth2'", + expectedErr: "google.delegated_account can only be provided with a jwt_file accessing 'auth.oauth2'", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.credentials_file": "./testdata/credentials.json", "google.delegated_account": "delegated@account.com", }, - "url": "localhost", }, }, { name: "google must work with delegated_account and a valid jwt_file", input: map[string]interface{}{ - "oauth2": map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ "provider": "google", "google.jwt_file": "./testdata/credentials.json", "google.delegated_account": "delegated@account.com", }, - "url": "localhost", }, }, } @@ -413,8 +351,9 @@ func TestConfigOauth2Validation(t *testing.T) { defer c.teardown() } + c.input["request.url"] = "localhost" cfg := common.MustNewConfigFrom(c.input) - conf := newDefaultConfig() + conf := defaultConfig() err := cfg.Unpack(&conf) switch { @@ -431,3 +370,25 @@ func TestConfigOauth2Validation(t *testing.T) { }) } } + +func TestCursorEntryConfig(t *testing.T) { + in := map[string]interface{}{ + "entry1": map[string]interface{}{ + "ignore_empty_value": true, + }, + "entry2": map[string]interface{}{ + "ignore_empty_value": false, + }, + "entry3": map[string]interface{}{ + "ignore_empty_value": nil, + }, + "entry4": map[string]interface{}{}, + } + cfg := common.MustNewConfigFrom(in) + conf := cursorConfig{} + require.NoError(t, cfg.Unpack(&conf)) + assert.True(t, conf["entry1"].mustIgnoreEmptyValue()) + assert.False(t, conf["entry2"].mustIgnoreEmptyValue()) + assert.True(t, conf["entry3"].mustIgnoreEmptyValue()) + assert.True(t, conf["entry4"].mustIgnoreEmptyValue()) +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go b/x-pack/filebeat/input/httpjson/cursor.go similarity index 98% rename from x-pack/filebeat/input/httpjson/internal/v2/cursor.go rename to x-pack/filebeat/input/httpjson/cursor.go index 224b27f2311..f603a12b0d8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go +++ b/x-pack/filebeat/input/httpjson/cursor.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/cursor_test.go b/x-pack/filebeat/input/httpjson/cursor_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/cursor_test.go rename to x-pack/filebeat/input/httpjson/cursor_test.go index 7d05c3a6746..7f451abe3ce 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/cursor_test.go +++ b/x-pack/filebeat/input/httpjson/cursor_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "testing" diff --git a/x-pack/filebeat/input/httpjson/date_cursor.go b/x-pack/filebeat/input/httpjson/date_cursor.go deleted file mode 100644 index eb20573eff2..00000000000 --- a/x-pack/filebeat/input/httpjson/date_cursor.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "bytes" - "net/url" - "text/template" - "time" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -type dateCursor struct { - log *logp.Logger - enabled bool - field string - url url.URL - urlField string - initialInterval time.Duration - dateFormat string - - valueTpl *template.Template -} - -func newDateCursorFromConfig(config config, log *logp.Logger) *dateCursor { - c := &dateCursor{ - enabled: config.DateCursor.isEnabled(), - url: *config.URL.URL, - } - - if !c.enabled { - return c - } - - c.log = log - c.field = config.DateCursor.Field - c.urlField = config.DateCursor.URLField - c.initialInterval = config.DateCursor.InitialInterval - c.dateFormat = config.DateCursor.getDateFormat() - if config.DateCursor.ValueTemplate != nil { - c.valueTpl = config.DateCursor.ValueTemplate.Template - } - - return c -} - -func (c *dateCursor) getURL(prevValue string) string { - if !c.enabled { - return c.url.String() - } - - var dateStr string - if prevValue == "" { - t := timeNow().UTC().Add(-c.initialInterval) - dateStr = t.Format(c.dateFormat) - } else { - dateStr = prevValue - } - - q := c.url.Query() - - var value string - if c.valueTpl == nil { - value = dateStr - } else { - buf := new(bytes.Buffer) - if err := c.valueTpl.Execute(buf, dateStr); err != nil { - return c.url.String() - } - value = buf.String() - } - - q.Set(c.urlField, value) - - url := c.url - url.RawQuery = q.Encode() - - return url.String() -} - -func (c *dateCursor) getNextValue(m common.MapStr) string { - if c.field == "" { - return time.Now().UTC().Format(c.dateFormat) - } - - v, err := m.GetValue(c.field) - if err != nil { - c.log.Warnf("date_cursor field: %q", err) - return "" - } - - switch t := v.(type) { - case string: - _, err := time.Parse(c.dateFormat, t) - if err != nil { - c.log.Warn("date_cursor field does not have the expected layout") - return "" - } - return t - } - - c.log.Warn("date_cursor field must be a string, cursor will not advance") - return "" -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/encoding.go b/x-pack/filebeat/input/httpjson/encoding.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/encoding.go rename to x-pack/filebeat/input/httpjson/encoding.go index 30768c3e416..178b5918aba 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/encoding.go +++ b/x-pack/filebeat/input/httpjson/encoding.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "bytes" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go b/x-pack/filebeat/input/httpjson/encoding_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go rename to x-pack/filebeat/input/httpjson/encoding_test.go index 2cc50a59c7b..6c967f7cbb5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go +++ b/x-pack/filebeat/input/httpjson/encoding_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "encoding/json" diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 9c6a2b4e71b..17da6632650 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -6,25 +6,24 @@ package httpjson import ( "context" + "encoding/json" "fmt" "net" "net/http" "net/url" "time" - "github.com/hashicorp/go-retryablehttp" + retryablehttp "github.com/hashicorp/go-retryablehttp" "go.uber.org/zap" - inputv2 "github.com/elastic/beats/v7/filebeat/input/v2" - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/logp" - v2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/v2" "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/timed" ) @@ -44,9 +43,9 @@ type retryLogger struct { log *logp.Logger } -func newRetryLogger() *retryLogger { +func newRetryLogger(log *logp.Logger) *retryLogger { return &retryLogger{ - log: logp.NewLogger("httpjson.retryablehttp", zap.AddCallerSkip(1)), + log: log.Named("retryablehttp").WithOptions(zap.AddCallerSkip(1)), } } @@ -66,22 +65,12 @@ func (log *retryLogger) Warn(format string, args ...interface{}) { log.log.Warnf(format, args...) } -func Plugin(log *logp.Logger, store cursor.StateStore) inputv2.Plugin { - sim := stateless.NewInputManager(statelessConfigure) - return inputv2.Plugin{ +func Plugin(log *logp.Logger, store inputcursor.StateStore) v2.Plugin { + return v2.Plugin{ Name: inputName, Stability: feature.Stable, Deprecated: false, - Manager: inputManager{ - v2inputManager: v2.NewInputManager(log, store), - stateless: &sim, - cursor: &cursor.InputManager{ - Logger: log, - StateStore: store, - Type: inputName, - Configure: cursorConfigure, - }, - }, + Manager: NewInputManager(log, store), } } @@ -106,96 +95,131 @@ func test(url *url.URL) error { } func run( - ctx inputv2.Context, + ctx v2.Context, config config, - publisher cursor.Publisher, - cursor *cursor.Cursor, + publisher inputcursor.Publisher, + cursor *inputcursor.Cursor, ) error { - log := ctx.Logger.With("input_url", config.URL) + log := ctx.Logger.With("input_url", config.Request.URL) stdCtx := ctxtool.FromCanceller(ctx.Cancelation) - httpClient, err := newHTTPClient(stdCtx, config) + httpClient, err := newHTTPClient(stdCtx, config, log) if err != nil { return err } - dateCursor := newDateCursorFromConfig(config, log) + requestFactory := newRequestFactory(config.Request, config.Auth, log) + pagination := newPagination(config, httpClient, log) + responseProcessor := newResponseProcessor(config.Response, pagination, log) + requester := newRequester(httpClient, requestFactory, responseProcessor, log) - rateLimiter := newRateLimiterFromConfig(config, log) + trCtx := emptyTransformContext() + trCtx.cursor = newCursor(config.Cursor, log) + trCtx.cursor.load(cursor) - pagination := newPaginationFromConfig(config) + doFunc := func() error { + log.Info("Process another repeated request.") - requester := newRequester( - config, - rateLimiter, - dateCursor, - pagination, - httpClient, - log, - ) + if err := requester.doRequest(stdCtx, trCtx, publisher); err != nil { + log.Errorf("Error while processing http request: %v", err) + } - requester.loadCursor(cursor, log) + if stdCtx.Err() != nil { + return err + } - // TODO: disallow passing interval = 0 as a mean to run once. - if config.Interval == 0 { - return requester.processHTTPRequest(stdCtx, publisher) + return nil } - err = timed.Periodic(stdCtx, config.Interval, func() error { - log.Info("Process another repeated request.") - if err := requester.processHTTPRequest(stdCtx, publisher); err != nil { - log.Error(err) - } - return nil - }) + // we trigger the first call immediately, + // then we schedule it on the given interval using timed.Periodic + if err = doFunc(); err == nil { + err = timed.Periodic(stdCtx, config.Interval, doFunc) + } - log.Infof("Context done: %v", err) + log.Infof("Input stopped because context was cancelled with: %v", err) return nil } -func newHTTPClient(ctx context.Context, config config) (*http.Client, error) { - config.Transport.Timeout = config.HTTPClientTimeout - - httpClient, err := - config.Transport.Client( - httpcommon.WithAPMHTTPInstrumentation(), - httpcommon.WithKeepaliveSettings{Disable: true}, - httpcommon.WithHeaderRoundTripper(map[string]string{"User-Agent": userAgent}), - ) +func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpClient, error) { + // Make retryable HTTP client + netHTTPClient, err := config.Request.Transport.Client( + httpcommon.WithAPMHTTPInstrumentation(), + httpcommon.WithKeepaliveSettings{Disable: true}, + ) if err != nil { return nil, err } - // Make retryable HTTP client + netHTTPClient.CheckRedirect = checkRedirect(config.Request, log) + client := &retryablehttp.Client{ - HTTPClient: httpClient, - Logger: newRetryLogger(), - RetryWaitMin: config.RetryWaitMin, - RetryWaitMax: config.RetryWaitMax, - RetryMax: config.RetryMax, + HTTPClient: netHTTPClient, + Logger: newRetryLogger(log), + RetryWaitMin: config.Request.Retry.getWaitMin(), + RetryWaitMax: config.Request.Retry.getWaitMax(), + RetryMax: config.Request.Retry.getMaxAttempts(), CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, } - if config.OAuth2.isEnabled() { - return config.OAuth2.client(ctx, client.StandardClient()) + limiter := newRateLimiterFromConfig(config.Request.RateLimit, log) + + if config.Auth.OAuth2.isEnabled() { + authClient, err := config.Auth.OAuth2.client(ctx, client.StandardClient()) + if err != nil { + return nil, err + } + return &httpClient{client: authClient, limiter: limiter}, nil } - return client.StandardClient(), nil + return &httpClient{client: client.StandardClient(), limiter: limiter}, nil +} + +func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + log.Debug("http client: checking redirect") + if len(via) >= config.RedirectMaxRedirects { + log.Debug("http client: max redirects exceeded") + return fmt.Errorf("stopped after %d redirects", config.RedirectMaxRedirects) + } + + if !config.RedirectForwardHeaders || len(via) == 0 { + log.Debugf("http client: nothing to do while checking redirects - forward_headers: %v, via: %#v", config.RedirectForwardHeaders, via) + return nil + } + + prev := via[len(via)-1] // previous request to get headers from + + log.Debugf("http client: forwarding headers from previous request: %#v", prev.Header) + req.Header = prev.Header.Clone() + + for _, k := range config.RedirectHeadersBanList { + log.Debugf("http client: ban header %v", k) + req.Header.Del(k) + } + + return nil + } } -func makeEvent(body string) beat.Event { +func makeEvent(body common.MapStr) (beat.Event, error) { + bodyBytes, err := json.Marshal(body) + if err != nil { + return beat.Event{}, err + } + now := timeNow() fields := common.MapStr{ "event": common.MapStr{ - "created": time.Now().UTC(), + "created": now, }, - "message": body, + "message": string(bodyBytes), } return beat.Event{ - Timestamp: time.Now().UTC(), + Timestamp: now, Fields: fields, - } + }, nil } diff --git a/x-pack/filebeat/input/httpjson/input_cursor.go b/x-pack/filebeat/input/httpjson/input_cursor.go index 0ee0ac44e9a..248a60b6f34 100644 --- a/x-pack/filebeat/input/httpjson/input_cursor.go +++ b/x-pack/filebeat/input/httpjson/input_cursor.go @@ -6,7 +6,7 @@ package httpjson import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" ) @@ -21,36 +21,35 @@ type source struct { } func (src source) Name() string { - return src.config.URL.String() + return src.config.Request.URL.String() } -func cursorConfigure(cfg *common.Config) ([]cursor.Source, cursor.Input, error) { - conf := newDefaultConfig() +func cursorConfigure(cfg *common.Config) ([]inputcursor.Source, inputcursor.Input, error) { + conf := defaultConfig() if err := cfg.Unpack(&conf); err != nil { return nil, nil, err } - sources, inp := newCursorInput(conf) return sources, inp, nil } -func newCursorInput(config config) ([]cursor.Source, cursor.Input) { +func newCursorInput(config config) ([]inputcursor.Source, inputcursor.Input) { // we only allow one url per config, if we wanted to allow more than one // each source should hold only one url - return []cursor.Source{&source{config: config}}, &cursorInput{} + return []inputcursor.Source{&source{config: config}}, &cursorInput{} } -func (in *cursorInput) Test(src cursor.Source, _ v2.TestContext) error { - return test((src.(*source)).config.URL.URL) +func (in *cursorInput) Test(src inputcursor.Source, _ v2.TestContext) error { + return test((src.(*source)).config.Request.URL.URL) } // Run starts the input and blocks until it ends the execution. // It will return on context cancellation, any other error will be retried. func (in *cursorInput) Run( ctx v2.Context, - src cursor.Source, - cursor cursor.Cursor, - publisher cursor.Publisher, + src inputcursor.Source, + cursor inputcursor.Cursor, + publisher inputcursor.Publisher, ) error { s := src.(*source) return run(ctx, s.config, publisher, &cursor) diff --git a/x-pack/filebeat/input/httpjson/input_manager.go b/x-pack/filebeat/input/httpjson/input_manager.go index 68a929fb8fb..a0cf4a45efc 100644 --- a/x-pack/filebeat/input/httpjson/input_manager.go +++ b/x-pack/filebeat/input/httpjson/input_manager.go @@ -9,52 +9,58 @@ import ( "github.com/elastic/go-concert/unison" - inputv2 "github.com/elastic/beats/v7/filebeat/input/v2" - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - v2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/v2" + "github.com/elastic/beats/v7/libbeat/logp" ) // inputManager wraps one stateless input manager // and one cursor input manager. It will create one or the other // based on the config that is passed. -type inputManager struct { +type InputManager struct { stateless *stateless.InputManager - cursor *cursor.InputManager - - v2inputManager v2.InputManager + cursor *inputcursor.InputManager } -var _ inputv2.InputManager = inputManager{} +var _ v2.InputManager = InputManager{} + +func NewInputManager(log *logp.Logger, store inputcursor.StateStore) InputManager { + sim := stateless.NewInputManager(statelessConfigure) + return InputManager{ + stateless: &sim, + cursor: &inputcursor.InputManager{ + Logger: log, + StateStore: store, + Type: inputName, + Configure: cursorConfigure, + }, + } +} // Init initializes both wrapped input managers. -func (m inputManager) Init(grp unison.Group, mode inputv2.Mode) error { +func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { + registerRequestTransforms() + registerResponseTransforms() + registerPaginationTransforms() + registerEncoders() + registerDecoders() return multierr.Append( - multierr.Append( - m.stateless.Init(grp, mode), - m.cursor.Init(grp, mode), - ), - m.v2inputManager.Init(grp, mode), + m.stateless.Init(grp, mode), + m.cursor.Init(grp, mode), ) } // Create creates a cursor input manager if the config has a date cursor set up, // otherwise it creates a stateless input manager. -func (m inputManager) Create(cfg *common.Config) (inputv2.Input, error) { - if v, _ := cfg.String("config_version", -1); v == "2" { - return m.v2inputManager.Create(cfg) - } - cfgwarn.Deprecate("7.12", "you are using a deprecated version of httpjson config") - config := newDefaultConfig() +func (m InputManager) Create(cfg *common.Config) (v2.Input, error) { + config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err } - - if config.DateCursor != nil { - return m.cursor.Create(cfg) + if len(config.Cursor) == 0 { + return m.stateless.Create(cfg) } - - return m.stateless.Create(cfg) + return m.cursor.Create(cfg) } diff --git a/x-pack/filebeat/input/httpjson/input_stateless.go b/x-pack/filebeat/input/httpjson/input_stateless.go index fbf28b2d20c..86fd50767c5 100644 --- a/x-pack/filebeat/input/httpjson/input_stateless.go +++ b/x-pack/filebeat/input/httpjson/input_stateless.go @@ -20,19 +20,19 @@ func (statelessInput) Name() string { } func statelessConfigure(cfg *common.Config) (stateless.Input, error) { - conf := newDefaultConfig() + conf := defaultConfig() if err := cfg.Unpack(&conf); err != nil { return nil, err } - return newStatelessInput(conf), nil + return newStatelessInput(conf) } -func newStatelessInput(config config) *statelessInput { - return &statelessInput{config: config} +func newStatelessInput(config config) (*statelessInput, error) { + return &statelessInput{config: config}, nil } func (in *statelessInput) Test(v2.TestContext) error { - return test(in.config.URL.URL) + return test(in.config.Request.URL.URL) } type statelessPublisher struct { diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 9630d0c81fb..92f97a6a507 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -23,7 +23,7 @@ import ( beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" ) -func TestStatelessHTTPJSONInput(t *testing.T) { +func TestInput(t *testing.T) { testCases := []struct { name string setupServer func(*testing.T, http.HandlerFunc, map[string]interface{}) @@ -35,8 +35,8 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test simple GET request", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, + "interval": 1, + "request.method": "GET", }, handler: defaultHandler("GET", ""), expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, @@ -45,9 +45,9 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test simple HTTPS GET request", setupServer: newTestServer(httptest.NewTLSServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "ssl.verification_mode": "none", + "interval": 1, + "request.method": "GET", + "request.ssl.verification_mode": "none", }, handler: defaultHandler("GET", ""), expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, @@ -56,11 +56,11 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test request honors rate limit", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "rate_limit.limit": "X-Rate-Limit-Limit", - "rate_limit.remaining": "X-Rate-Limit-Remaining", - "rate_limit.reset": "X-Rate-Limit-Reset", + "interval": 1, + "http_method": "GET", + "request.rate_limit.limit": `[[.last_response.header.Get "X-Rate-Limit-Limit"]]`, + "request.rate_limit.remaining": `[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`, + "request.rate_limit.reset": `[[.last_response.header.Get "X-Rate-Limit-Reset"]]`, }, handler: rateLimitHandler(), expected: []string{`{"hello":"world"}`}, @@ -69,8 +69,8 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test request retries when failed", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, + "interval": 1, + "request.method": "GET", }, handler: retryHandler(), expected: []string{`{"hello":"world"}`}, @@ -79,9 +79,9 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test POST request with body", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "POST", - "interval": 0, - "http_request_body": map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "request.body": map[string]interface{}{ "test": "abc", }, }, @@ -92,8 +92,8 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test repeated POST requests", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "POST", - "interval": "100ms", + "interval": "100ms", + "request.method": "POST", }, handler: defaultHandler("POST", ""), expected: []string{ @@ -102,23 +102,28 @@ func TestStatelessHTTPJSONInput(t *testing.T) { }, }, { - name: "Test json objects array", + name: "Test split by json objects array", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "json_objects_array": "hello", + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + }, }, handler: defaultHandler("GET", ""), expected: []string{`{"world":"moon"}`, `{"space":[{"cake":"pumpkin"}]}`}, }, { - name: "Test split events by", + name: "Test split by json objects array with keep parent", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "split_events_by": "hello", + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + "keep_parent": true, + }, }, handler: defaultHandler("GET", ""), expected: []string{ @@ -127,13 +132,18 @@ func TestStatelessHTTPJSONInput(t *testing.T) { }, }, { - name: "Test split events by with array", + name: "Test nested split", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "split_events_by": "space", - "json_objects_array": "hello", + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + "split": map[string]interface{}{ + "target": "body.space", + "keep_parent": true, + }, + }, }, handler: defaultHandler("GET", ""), expected: []string{ @@ -145,16 +155,20 @@ func TestStatelessHTTPJSONInput(t *testing.T) { name: "Test split events by not found", setupServer: newTestServer(httptest.NewServer), baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "split_events_by": "unknwown", + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.unknown", + }, }, handler: defaultHandler("GET", ""), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + expected: []string{}, }, { name: "Test date cursor", setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) // mock timeNow func to return a fixed value timeNow = func() time.Time { t, _ := time.Parse(time.RFC3339, "2002-10-02T15:00:00Z") @@ -162,17 +176,27 @@ func TestStatelessHTTPJSONInput(t *testing.T) { } server := httptest.NewServer(h) - config["url"] = server.URL + config["request.url"] = server.URL t.Cleanup(server.Close) + t.Cleanup(func() { timeNow = time.Now }) }, baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": "100ms", - "date_cursor.field": "@timestamp", - "date_cursor.url_field": "$filter", - "date_cursor.value_template": "alertCreationTime ge {{.}}", - "date_cursor.initial_interval": "10m", - "date_cursor.date_format": "2006-01-02T15:04:05Z", + "interval": 1, + "request.method": "GET", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.$filter", + "value": "alertCreationTime ge [[.cursor.timestamp]]", + "default": `alertCreationTime ge [[formatDate (now (parseDuration "-10m")) "2006-01-02T15:04:05Z"]]`, + }, + }, + }, + "cursor": map[string]interface{}{ + "timestamp": map[string]interface{}{ + "value": `[[index .last_response.body "@timestamp"]]`, + }, + }, }, handler: dateCursorHandler(), expected: []string{ @@ -182,39 +206,189 @@ func TestStatelessHTTPJSONInput(t *testing.T) { }, }, { - name: "Test pagination", - setupServer: newTestServer(httptest.NewServer), + name: "Test pagination", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": time.Second, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.items", + }, + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.page", + "value": "[[.last_response.body.nextPageToken]]", + }, + }, + }, + }, + handler: paginationHandler(), + expected: []string{`{"foo":"a"}`, `{"foo":"b"}`}, + }, + { + name: "Test first event", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + registerResponseTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, baseConfig: map[string]interface{}{ - "http_method": "GET", - "interval": 0, - "pagination.id_field": "nextPageToken", - "pagination.url_field": "page", - "json_objects_array": "items", + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.items", + "transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.first", + "value": "[[.cursor.first]]", + "default": "none", + }, + }, + }, + }, + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.page", + "value": "[[.last_response.body.nextPageToken]]", + "fail_on_template_error": true, + }, + }, + }, + "cursor": map[string]interface{}{ + "first": map[string]interface{}{ + "value": "[[.first_event.foo]]", + }, + }, }, handler: paginationHandler(), - expected: []string{`{"foo":"bar"}`, `{"foo":"bar"}`}, + expected: []string{`{"first":"none", "foo":"a"}`, `{"first":"a", "foo":"b"}`, `{"first":"a", "foo":"c"}`, `{"first":"c", "foo":"d"}`}, + }, + { + name: "Test pagination with array response", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.page", + "value": `[[index (index .last_response.body 0) "nextPageToken"]]`, + }, + }, + }, + }, + handler: paginationArrayHandler(), + expected: []string{`{"nextPageToken":"bar","foo":"bar"}`, `{"foo":"bar"}`, `{"foo":"bar"}`}, }, { name: "Test oauth2", setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { server := httptest.NewServer(h) - config["url"] = server.URL - config["oauth2.token_url"] = server.URL + "/token" + config["request.url"] = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" t.Cleanup(server.Close) }, baseConfig: map[string]interface{}{ - "http_method": "POST", - "interval": "0", - "oauth2.client.id": "a_client_id", - "oauth2.client.secret": "a_client_secret", - "oauth2.endpoint_params": map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "auth.oauth2.client.id": "a_client_id", + "auth.oauth2.client.secret": "a_client_secret", + "auth.oauth2.endpoint_params": map[string]interface{}{ "param1": "v1", }, - "oauth2.scopes": []string{"scope1", "scope2"}, + "auth.oauth2.scopes": []string{"scope1", "scope2"}, }, handler: oauth2Handler, expected: []string{`{"hello": "world"}`}, }, + { + name: "Test request transforms can access state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + "/test-path" + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.url.path", + "value": `[[.url.Path]]`, + }, + }, + }, + }, + handler: defaultHandler("POST", `{"bar":"foo","url":{"path":"/test-path"}}`), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test response transforms can't access request state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + registerResponseTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 10, + "request.method": "GET", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + }, + "response.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, } for _, testCase := range testCases { @@ -224,10 +398,12 @@ func TestStatelessHTTPJSONInput(t *testing.T) { cfg := common.MustNewConfigFrom(tc.baseConfig) - conf := newDefaultConfig() + conf := defaultConfig() assert.NoError(t, cfg.Unpack(&conf)) - input := newStatelessInput(conf) + input, err := newStatelessInput(conf) + + assert.NoError(t, err) assert.Equal(t, "httpjson-stateless", input.Name()) assert.NoError(t, input.Test(v2.TestContext{})) @@ -238,17 +414,26 @@ func TestStatelessHTTPJSONInput(t *testing.T) { t.Cleanup(cancel) var g errgroup.Group - g.Go(func() error { return input.Run(ctx, chanClient) }) + g.Go(func() error { + return input.Run(ctx, chanClient) + }) timeout := time.NewTimer(5 * time.Second) t.Cleanup(func() { _ = timeout.Stop() }) + if len(tc.expected) == 0 { + cancel() + assert.NoError(t, g.Wait()) + return + } + var receivedCount int wait: for { select { case <-timeout.C: t.Errorf("timed out waiting for %d events", len(tc.expected)) + cancel() return case got := <-chanClient.Channel: val, err := got.Fields.GetValue("message") @@ -271,7 +456,7 @@ func newTestServer( ) func(*testing.T, http.HandlerFunc, map[string]interface{}) { return func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { server := newServer(h) - config["url"] = server.URL + config["request.url"] = server.URL t.Cleanup(server.Close) } } @@ -419,14 +604,37 @@ func paginationHandler() http.HandlerFunc { w.Header().Set("content-type", "application/json") switch count { case 0: - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","nextPageToken":"bar","items":[{"foo":"bar"}]}`)) + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","nextPageToken":"bar","items":[{"foo":"a"}]}`)) + case 1: + if r.URL.Query().Get("page") != "bar" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) + return + } + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","items":[{"foo":"b"}]}`)) + case 2: + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:02Z","items":[{"foo":"c"}]}`)) + case 3: + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:03Z","items":[{"foo":"d"}]}`)) + } + count += 1 + } +} + +func paginationArrayHandler() http.HandlerFunc { + var count int + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + switch count { + case 0: + _, _ = w.Write([]byte(`[{"nextPageToken":"bar","foo":"bar"},{"foo":"bar"}]`)) case 1: if r.URL.Query().Get("page") != "bar" { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) return } - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","items":[{"foo":"bar"}]}`)) + _, _ = w.Write([]byte(`[{"foo":"bar"}]`)) } count += 1 } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config.go b/x-pack/filebeat/input/httpjson/internal/v2/config.go deleted file mode 100644 index aa2be1c0113..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/config.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "errors" - "time" - - "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" -) - -type config struct { - Interval time.Duration `config:"interval" validate:"required"` - Auth *authConfig `config:"auth"` - Request *requestConfig `config:"request" validate:"required"` - Response *responseConfig `config:"response"` - Cursor cursorConfig `config:"cursor"` -} - -type cursorConfig map[string]cursorEntry - -type cursorEntry struct { - Value *valueTpl `config:"value"` - Default *valueTpl `config:"default"` - IgnoreEmptyValue *bool `config:"ignore_empty_value"` -} - -func (ce cursorEntry) mustIgnoreEmptyValue() bool { - return ce.IgnoreEmptyValue == nil || *ce.IgnoreEmptyValue -} - -func (c config) Validate() error { - if c.Interval <= 0 { - return errors.New("interval must be greater than 0") - } - return nil -} - -func defaultConfig() config { - maxAttempts := 5 - waitMin := time.Second - waitMax := time.Minute - transport := httpcommon.DefaultHTTPTransportSettings() - transport.Timeout = 30 * time.Second - - return config{ - Interval: time.Minute, - Auth: &authConfig{}, - Request: &requestConfig{ - Method: "GET", - Retry: retryConfig{ - MaxAttempts: &maxAttempts, - WaitMin: &waitMin, - WaitMax: &waitMax, - }, - RedirectForwardHeaders: false, - RedirectMaxRedirects: 10, - Transport: transport, - }, - Response: &responseConfig{}, - } -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go deleted file mode 100644 index 4110430b25e..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "context" - "errors" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/oauth2/google" - - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestProviderCanonical(t *testing.T) { - const ( - a oAuth2Provider = "gOoGle" - b oAuth2Provider = "google" - ) - - assert.Equal(t, a.canonical(), b.canonical()) -} - -func TestGetProviderIsCanonical(t *testing.T) { - const expected oAuth2Provider = "google" - - oauth2 := oAuth2Config{Provider: "GOogle"} - assert.Equal(t, expected, oauth2.getProvider()) -} - -func TestIsEnabled(t *testing.T) { - oauth2 := oAuth2Config{} - if !oauth2.isEnabled() { - t.Fatal("OAuth2 should be enabled by default") - } - - var enabled = false - oauth2.Enabled = &enabled - - assert.False(t, oauth2.isEnabled()) - - enabled = true - - assert.True(t, oauth2.isEnabled()) -} - -func TestGetTokenURL(t *testing.T) { - const expected = "http://localhost" - oauth2 := oAuth2Config{TokenURL: "http://localhost"} - assert.Equal(t, expected, oauth2.getTokenURL()) -} - -func TestGetTokenURLWithAzure(t *testing.T) { - const expectedWithoutTenantID = "http://localhost" - oauth2 := oAuth2Config{TokenURL: "http://localhost", Provider: "azure"} - - assert.Equal(t, expectedWithoutTenantID, oauth2.getTokenURL()) - - oauth2.TokenURL = "" - oauth2.AzureTenantID = "a_tenant_id" - const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" - - assert.Equal(t, expectedWithTenantID, oauth2.getTokenURL()) -} - -func TestGetEndpointParams(t *testing.T) { - var expected = map[string][]string{"foo": {"bar"}} - oauth2 := oAuth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} - assert.Equal(t, expected, oauth2.getEndpointParams()) -} - -func TestGetEndpointParamsWithAzure(t *testing.T) { - var expectedWithoutResource = map[string][]string{"foo": {"bar"}} - oauth2 := oAuth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} - - assert.Equal(t, expectedWithoutResource, oauth2.getEndpointParams()) - - oauth2.AzureResource = "baz" - var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} - - assert.Equal(t, expectedWithResource, oauth2.getEndpointParams()) -} - -func TestConfigFailsWithInvalidMethod(t *testing.T) { - m := map[string]interface{}{ - "request.method": "DELETE", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") - } -} - -func TestConfigMustFailWithInvalidURL(t *testing.T) { - m := map[string]interface{}{ - "request.url": "::invalid::", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - err := cfg.Unpack(&conf) - assert.EqualError(t, err, `parse "::invalid::": missing protocol scheme accessing 'request.url'`) -} - -func TestConfigOauth2Validation(t *testing.T) { - cases := []struct { - name string - expectedErr string - input map[string]interface{} - setup func() - teardown func() - }{ - { - name: "can't set oauth2 and basic auth together", - expectedErr: "only one kind of auth can be enabled accessing 'auth'", - input: map[string]interface{}{ - "auth.basic.user": "user", - "auth.basic.password": "pass", - "auth.oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", - }, - }, - }, - }, - { - name: "can set oauth2 and basic auth together if oauth2 is disabled", - input: map[string]interface{}{ - "auth.basic.user": "user", - "auth.basic.password": "pass", - "auth.oauth2": map[string]interface{}{ - "enabled": false, - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", - }, - }, - }, - }, - { - name: "token_url and client credentials must be set", - expectedErr: "both token_url and client credentials must be provided accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{}, - }, - }, - { - name: "must fail with an unknown provider", - expectedErr: "unknown provider \"unknown\" accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "unknown", - }, - }, - }, - { - name: "azure must have either tenant_id or token_url", - expectedErr: "at least one of token_url or tenant_id must be provided accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "azure", - }, - }, - }, - { - name: "azure must have only one of token_url and tenant_id", - expectedErr: "only one of token_url and tenant_id can be used accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", - "token_url": "localhost", - }, - }, - }, - { - name: "azure must have client credentials set", - expectedErr: "client credentials must be provided accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", - }, - }, - }, - { - name: "azure config is valid", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "azure", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", - }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", - }, - }, - }, - { - name: "google can't have token_url or client credentials set", - expectedErr: "none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", - }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", - "token_url": "localhost", - }, - }, - }, - { - name: "google must fail if no ADC available", - expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - }, - }, - setup: func() { - // we change the default function to force a failure - findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { - return nil, errors.New("failed") - } - }, - teardown: func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }, - }, - { - name: "google must fail if credentials file not found", - expectedErr: "the file \"./wrong\" cannot be found accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./wrong", - }, - }, - }, - { - name: "google must fail if ADC is wrongly set", - expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - }, - }, - setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, - }, - { - name: "google must work if ADC is set up", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - }, - }, - setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, - }, - { - name: "google must work if credentials_file is correct", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/credentials.json", - }, - }, - }, - { - name: "google must work if jwt_file is correct", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.jwt_file": "./testdata/credentials.json", - }, - }, - }, - { - name: "google must work if credentials_json is correct", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": `{ - "type": "service_account", - "project_id": "foo", - "private_key_id": "x", - "client_email": "foo@bar.com", - "client_id": "0" - }`, - }, - }, - }, - { - name: "google must fail if credentials_json is not a valid JSON", - expectedErr: "the field can't be converted to valid JSON accessing 'auth.oauth2.google.credentials_json'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": `invalid`, - }, - }, - }, - { - name: "google must fail if the provided credentials file is not a valid JSON", - expectedErr: "the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/invalid_credentials.json", - }, - }, - }, - { - name: "google must fail if the delegated_account is set without jwt_file", - expectedErr: "google.delegated_account can only be provided with a jwt_file accessing 'auth.oauth2'", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/credentials.json", - "google.delegated_account": "delegated@account.com", - }, - }, - }, - { - name: "google must work with delegated_account and a valid jwt_file", - input: map[string]interface{}{ - "auth.oauth2": map[string]interface{}{ - "provider": "google", - "google.jwt_file": "./testdata/credentials.json", - "google.delegated_account": "delegated@account.com", - }, - }, - }, - } - - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - if c.setup != nil { - c.setup() - } - - if c.teardown != nil { - defer c.teardown() - } - - c.input["request.url"] = "localhost" - cfg := common.MustNewConfigFrom(c.input) - conf := defaultConfig() - err := cfg.Unpack(&conf) - - switch { - case c.expectedErr == "": - if err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } - - case c.expectedErr != "": - if err == nil || err.Error() != c.expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) - } - } - }) - } -} - -func TestCursorEntryConfig(t *testing.T) { - in := map[string]interface{}{ - "entry1": map[string]interface{}{ - "ignore_empty_value": true, - }, - "entry2": map[string]interface{}{ - "ignore_empty_value": false, - }, - "entry3": map[string]interface{}{ - "ignore_empty_value": nil, - }, - "entry4": map[string]interface{}{}, - } - cfg := common.MustNewConfigFrom(in) - conf := cursorConfig{} - require.NoError(t, cfg.Unpack(&conf)) - assert.True(t, conf["entry1"].mustIgnoreEmptyValue()) - assert.False(t, conf["entry2"].mustIgnoreEmptyValue()) - assert.True(t, conf["entry3"].mustIgnoreEmptyValue()) - assert.True(t, conf["entry4"].mustIgnoreEmptyValue()) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go deleted file mode 100644 index 9cbd017c6ba..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" - "net/url" - "time" - - retryablehttp "github.com/hashicorp/go-retryablehttp" - "go.uber.org/zap" - - v2 "github.com/elastic/beats/v7/filebeat/input/v2" - inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" - "github.com/elastic/beats/v7/libbeat/common/useragent" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/go-concert/ctxtool" - "github.com/elastic/go-concert/timed" -) - -const ( - inputName = "httpjson" -) - -var ( - userAgent = useragent.UserAgent("Filebeat") - - // for testing - timeNow = time.Now -) - -type retryLogger struct { - log *logp.Logger -} - -func newRetryLogger(log *logp.Logger) *retryLogger { - return &retryLogger{ - log: log.Named("retryablehttp").WithOptions(zap.AddCallerSkip(1)), - } -} - -func (log *retryLogger) Error(format string, args ...interface{}) { - log.log.Errorf(format, args...) -} - -func (log *retryLogger) Info(format string, args ...interface{}) { - log.log.Infof(format, args...) -} - -func (log *retryLogger) Debug(format string, args ...interface{}) { - log.log.Debugf(format, args...) -} - -func (log *retryLogger) Warn(format string, args ...interface{}) { - log.log.Warnf(format, args...) -} - -func test(url *url.URL) error { - port := func() string { - if url.Port() != "" { - return url.Port() - } - switch url.Scheme { - case "https": - return "443" - } - return "80" - }() - - _, err := net.DialTimeout("tcp", net.JoinHostPort(url.Hostname(), port), time.Second) - if err != nil { - return fmt.Errorf("url %q is unreachable", url) - } - - return nil -} - -func run( - ctx v2.Context, - config config, - publisher inputcursor.Publisher, - cursor *inputcursor.Cursor, -) error { - log := ctx.Logger.With("input_url", config.Request.URL) - - stdCtx := ctxtool.FromCanceller(ctx.Cancelation) - - httpClient, err := newHTTPClient(stdCtx, config, log) - if err != nil { - return err - } - - requestFactory := newRequestFactory(config.Request, config.Auth, log) - pagination := newPagination(config, httpClient, log) - responseProcessor := newResponseProcessor(config.Response, pagination, log) - requester := newRequester(httpClient, requestFactory, responseProcessor, log) - - trCtx := emptyTransformContext() - trCtx.cursor = newCursor(config.Cursor, log) - trCtx.cursor.load(cursor) - - doFunc := func() error { - log.Info("Process another repeated request.") - - if err := requester.doRequest(stdCtx, trCtx, publisher); err != nil { - log.Errorf("Error while processing http request: %v", err) - } - - if stdCtx.Err() != nil { - return err - } - - return nil - } - - // we trigger the first call immediately, - // then we schedule it on the given interval using timed.Periodic - if err = doFunc(); err == nil { - err = timed.Periodic(stdCtx, config.Interval, doFunc) - } - - log.Infof("Input stopped because context was cancelled with: %v", err) - - return nil -} - -func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpClient, error) { - // Make retryable HTTP client - netHTTPClient, err := config.Request.Transport.Client( - httpcommon.WithAPMHTTPInstrumentation(), - httpcommon.WithKeepaliveSettings{Disable: true}, - ) - if err != nil { - return nil, err - } - - netHTTPClient.CheckRedirect = checkRedirect(config.Request, log) - - client := &retryablehttp.Client{ - HTTPClient: netHTTPClient, - Logger: newRetryLogger(log), - RetryWaitMin: config.Request.Retry.getWaitMin(), - RetryWaitMax: config.Request.Retry.getWaitMax(), - RetryMax: config.Request.Retry.getMaxAttempts(), - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, - } - - limiter := newRateLimiterFromConfig(config.Request.RateLimit, log) - - if config.Auth.OAuth2.isEnabled() { - authClient, err := config.Auth.OAuth2.client(ctx, client.StandardClient()) - if err != nil { - return nil, err - } - return &httpClient{client: authClient, limiter: limiter}, nil - } - - return &httpClient{client: client.StandardClient(), limiter: limiter}, nil -} - -func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error { - return func(req *http.Request, via []*http.Request) error { - log.Debug("http client: checking redirect") - if len(via) >= config.RedirectMaxRedirects { - log.Debug("http client: max redirects exceeded") - return fmt.Errorf("stopped after %d redirects", config.RedirectMaxRedirects) - } - - if !config.RedirectForwardHeaders || len(via) == 0 { - log.Debugf("http client: nothing to do while checking redirects - forward_headers: %v, via: %#v", config.RedirectForwardHeaders, via) - return nil - } - - prev := via[len(via)-1] // previous request to get headers from - - log.Debugf("http client: forwarding headers from previous request: %#v", prev.Header) - req.Header = prev.Header.Clone() - - for _, k := range config.RedirectHeadersBanList { - log.Debugf("http client: ban header %v", k) - req.Header.Del(k) - } - - return nil - } -} - -func makeEvent(body common.MapStr) (beat.Event, error) { - bodyBytes, err := json.Marshal(body) - if err != nil { - return beat.Event{}, err - } - now := timeNow() - fields := common.MapStr{ - "event": common.MapStr{ - "created": now, - }, - "message": string(bodyBytes), - } - - return beat.Event{ - Timestamp: now, - Fields: fields, - }, nil -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go b/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go deleted file mode 100644 index db454f6776d..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - v2 "github.com/elastic/beats/v7/filebeat/input/v2" - inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - "github.com/elastic/beats/v7/libbeat/common" -) - -type cursorInput struct{} - -func (cursorInput) Name() string { - return "httpjson-cursor" -} - -type source struct { - config config -} - -func (src source) Name() string { - return src.config.Request.URL.String() -} - -func cursorConfigure(cfg *common.Config) ([]inputcursor.Source, inputcursor.Input, error) { - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - return nil, nil, err - } - sources, inp := newCursorInput(conf) - return sources, inp, nil -} - -func newCursorInput(config config) ([]inputcursor.Source, inputcursor.Input) { - // we only allow one url per config, if we wanted to allow more than one - // each source should hold only one url - return []inputcursor.Source{&source{config: config}}, &cursorInput{} -} - -func (in *cursorInput) Test(src inputcursor.Source, _ v2.TestContext) error { - return test((src.(*source)).config.Request.URL.URL) -} - -// Run starts the input and blocks until it ends the execution. -// It will return on context cancellation, any other error will be retried. -func (in *cursorInput) Run( - ctx v2.Context, - src inputcursor.Source, - cursor inputcursor.Cursor, - publisher inputcursor.Publisher, -) error { - s := src.(*source) - return run(ctx, s.config, publisher, &cursor) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go deleted file mode 100644 index bdf04b108ed..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "go.uber.org/multierr" - - "github.com/elastic/go-concert/unison" - - v2 "github.com/elastic/beats/v7/filebeat/input/v2" - inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -// inputManager wraps one stateless input manager -// and one cursor input manager. It will create one or the other -// based on the config that is passed. -type InputManager struct { - stateless *stateless.InputManager - cursor *inputcursor.InputManager -} - -var _ v2.InputManager = InputManager{} - -func NewInputManager(log *logp.Logger, store inputcursor.StateStore) InputManager { - sim := stateless.NewInputManager(statelessConfigure) - return InputManager{ - stateless: &sim, - cursor: &inputcursor.InputManager{ - Logger: log, - StateStore: store, - Type: inputName, - Configure: cursorConfigure, - }, - } -} - -// Init initializes both wrapped input managers. -func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { - registerRequestTransforms() - registerResponseTransforms() - registerPaginationTransforms() - registerEncoders() - registerDecoders() - return multierr.Append( - m.stateless.Init(grp, mode), - m.cursor.Init(grp, mode), - ) -} - -// Create creates a cursor input manager if the config has a date cursor set up, -// otherwise it creates a stateless input manager. -func (m InputManager) Create(cfg *common.Config) (v2.Input, error) { - config := defaultConfig() - if err := cfg.Unpack(&config); err != nil { - return nil, err - } - if len(config.Cursor) == 0 { - return m.stateless.Create(cfg) - } - return m.cursor.Create(cfg) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go b/x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go deleted file mode 100644 index 4a871d62d77..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - v2 "github.com/elastic/beats/v7/filebeat/input/v2" - stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" -) - -type statelessInput struct { - config config -} - -func (statelessInput) Name() string { - return "httpjson-stateless" -} - -func statelessConfigure(cfg *common.Config) (stateless.Input, error) { - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - return nil, err - } - return newStatelessInput(conf) -} - -func newStatelessInput(config config) (*statelessInput, error) { - return &statelessInput{config: config}, nil -} - -func (in *statelessInput) Test(v2.TestContext) error { - return test(in.config.Request.URL.URL) -} - -type statelessPublisher struct { - wrapped stateless.Publisher -} - -func (pub statelessPublisher) Publish(event beat.Event, _ interface{}) error { - pub.wrapped.Publish(event) - return nil -} - -// Run starts the input and blocks until it ends the execution. -// It will return on context cancellation, any other error will be retried. -func (in *statelessInput) Run(ctx v2.Context, publisher stateless.Publisher) error { - pub := statelessPublisher{wrapped: publisher} - return run(ctx, in.config, pub, nil) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go deleted file mode 100644 index 788a3f2d1af..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go +++ /dev/null @@ -1,641 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "context" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "golang.org/x/sync/errgroup" - - v2 "github.com/elastic/beats/v7/filebeat/input/v2" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" - beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" -) - -func TestInput(t *testing.T) { - testCases := []struct { - name string - setupServer func(*testing.T, http.HandlerFunc, map[string]interface{}) - baseConfig map[string]interface{} - handler http.HandlerFunc - expected []string - }{ - { - name: "Test simple GET request", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - }, - handler: defaultHandler("GET", ""), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, - }, - { - name: "Test simple HTTPS GET request", - setupServer: newTestServer(httptest.NewTLSServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "request.ssl.verification_mode": "none", - }, - handler: defaultHandler("GET", ""), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, - }, - { - name: "Test request honors rate limit", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "http_method": "GET", - "request.rate_limit.limit": `[[.last_response.header.Get "X-Rate-Limit-Limit"]]`, - "request.rate_limit.remaining": `[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`, - "request.rate_limit.reset": `[[.last_response.header.Get "X-Rate-Limit-Reset"]]`, - }, - handler: rateLimitHandler(), - expected: []string{`{"hello":"world"}`}, - }, - { - name: "Test request retries when failed", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - }, - handler: retryHandler(), - expected: []string{`{"hello":"world"}`}, - }, - { - name: "Test POST request with body", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "POST", - "request.body": map[string]interface{}{ - "test": "abc", - }, - }, - handler: defaultHandler("POST", `{"test":"abc"}`), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, - }, - { - name: "Test repeated POST requests", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": "100ms", - "request.method": "POST", - }, - handler: defaultHandler("POST", ""), - expected: []string{ - `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`, - `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`, - }, - }, - { - name: "Test split by json objects array", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.hello", - }, - }, - handler: defaultHandler("GET", ""), - expected: []string{`{"world":"moon"}`, `{"space":[{"cake":"pumpkin"}]}`}, - }, - { - name: "Test split by json objects array with keep parent", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.hello", - "keep_parent": true, - }, - }, - handler: defaultHandler("GET", ""), - expected: []string{ - `{"hello":{"world":"moon"}}`, - `{"hello":{"space":[{"cake":"pumpkin"}]}}`, - }, - }, - { - name: "Test nested split", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.hello", - "split": map[string]interface{}{ - "target": "body.space", - "keep_parent": true, - }, - }, - }, - handler: defaultHandler("GET", ""), - expected: []string{ - `{"world":"moon"}`, - `{"space":{"cake":"pumpkin"}}`, - }, - }, - { - name: "Test split events by not found", - setupServer: newTestServer(httptest.NewServer), - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.unknown", - }, - }, - handler: defaultHandler("GET", ""), - expected: []string{}, - }, - { - name: "Test date cursor", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerRequestTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - // mock timeNow func to return a fixed value - timeNow = func() time.Time { - t, _ := time.Parse(time.RFC3339, "2002-10-02T15:00:00Z") - return t - } - - server := httptest.NewServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - t.Cleanup(func() { timeNow = time.Now }) - }, - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "request.transforms": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "url.params.$filter", - "value": "alertCreationTime ge [[.cursor.timestamp]]", - "default": `alertCreationTime ge [[formatDate (now (parseDuration "-10m")) "2006-01-02T15:04:05Z"]]`, - }, - }, - }, - "cursor": map[string]interface{}{ - "timestamp": map[string]interface{}{ - "value": `[[index .last_response.body "@timestamp"]]`, - }, - }, - }, - handler: dateCursorHandler(), - expected: []string{ - `{"@timestamp":"2002-10-02T15:00:00Z","foo":"bar"}`, - `{"@timestamp":"2002-10-02T15:00:01Z","foo":"bar"}`, - `{"@timestamp":"2002-10-02T15:00:02Z","foo":"bar"}`, - }, - }, - { - name: "Test pagination", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerPaginationTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - server := httptest.NewServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": time.Second, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.items", - }, - "response.pagination": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "url.params.page", - "value": "[[.last_response.body.nextPageToken]]", - }, - }, - }, - }, - handler: paginationHandler(), - expected: []string{`{"foo":"a"}`, `{"foo":"b"}`}, - }, - { - name: "Test first event", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerPaginationTransforms() - registerResponseTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - server := httptest.NewServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.split": map[string]interface{}{ - "target": "body.items", - "transforms": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "body.first", - "value": "[[.cursor.first]]", - "default": "none", - }, - }, - }, - }, - "response.pagination": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "url.params.page", - "value": "[[.last_response.body.nextPageToken]]", - "fail_on_template_error": true, - }, - }, - }, - "cursor": map[string]interface{}{ - "first": map[string]interface{}{ - "value": "[[.first_event.foo]]", - }, - }, - }, - handler: paginationHandler(), - expected: []string{`{"first":"none", "foo":"a"}`, `{"first":"a", "foo":"b"}`, `{"first":"a", "foo":"c"}`, `{"first":"c", "foo":"d"}`}, - }, - { - name: "Test pagination with array response", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerPaginationTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - server := httptest.NewServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "GET", - "response.pagination": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "url.params.page", - "value": `[[index (index .last_response.body 0) "nextPageToken"]]`, - }, - }, - }, - }, - handler: paginationArrayHandler(), - expected: []string{`{"nextPageToken":"bar","foo":"bar"}`, `{"foo":"bar"}`, `{"foo":"bar"}`}, - }, - { - name: "Test oauth2", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - server := httptest.NewServer(h) - config["request.url"] = server.URL - config["auth.oauth2.token_url"] = server.URL + "/token" - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "POST", - "auth.oauth2.client.id": "a_client_id", - "auth.oauth2.client.secret": "a_client_secret", - "auth.oauth2.endpoint_params": map[string]interface{}{ - "param1": "v1", - }, - "auth.oauth2.scopes": []string{"scope1", "scope2"}, - }, - handler: oauth2Handler, - expected: []string{`{"hello": "world"}`}, - }, - { - name: "Test request transforms can access state from previous transforms", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerRequestTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - server := httptest.NewServer(h) - config["request.url"] = server.URL + "/test-path" - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": 1, - "request.method": "POST", - "request.transforms": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "header.X-Foo", - "value": "foo", - }, - }, - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "body.bar", - "value": `[[.header.Get "X-Foo"]]`, - }, - }, - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "body.url.path", - "value": `[[.url.Path]]`, - }, - }, - }, - }, - handler: defaultHandler("POST", `{"bar":"foo","url":{"path":"/test-path"}}`), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, - }, - { - name: "Test response transforms can't access request state from previous transforms", - setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - registerRequestTransforms() - registerResponseTransforms() - t.Cleanup(func() { registeredTransforms = newRegistry() }) - server := httptest.NewServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - }, - baseConfig: map[string]interface{}{ - "interval": 10, - "request.method": "GET", - "request.transforms": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "header.X-Foo", - "value": "foo", - }, - }, - }, - "response.transforms": []interface{}{ - map[string]interface{}{ - "set": map[string]interface{}{ - "target": "body.bar", - "value": `[[.header.Get "X-Foo"]]`, - }, - }, - }, - }, - handler: defaultHandler("GET", ""), - expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, - }, - } - - for _, testCase := range testCases { - tc := testCase - t.Run(tc.name, func(t *testing.T) { - tc.setupServer(t, tc.handler, tc.baseConfig) - - cfg := common.MustNewConfigFrom(tc.baseConfig) - - conf := defaultConfig() - assert.NoError(t, cfg.Unpack(&conf)) - - input, err := newStatelessInput(conf) - - assert.NoError(t, err) - assert.Equal(t, "httpjson-stateless", input.Name()) - assert.NoError(t, input.Test(v2.TestContext{})) - - chanClient := beattest.NewChanClient(len(tc.expected)) - t.Cleanup(func() { _ = chanClient.Close() }) - - ctx, cancel := newV2Context() - t.Cleanup(cancel) - - var g errgroup.Group - g.Go(func() error { - return input.Run(ctx, chanClient) - }) - - timeout := time.NewTimer(5 * time.Second) - t.Cleanup(func() { _ = timeout.Stop() }) - - if len(tc.expected) == 0 { - cancel() - assert.NoError(t, g.Wait()) - return - } - - var receivedCount int - wait: - for { - select { - case <-timeout.C: - t.Errorf("timed out waiting for %d events", len(tc.expected)) - cancel() - return - case got := <-chanClient.Channel: - val, err := got.Fields.GetValue("message") - assert.NoError(t, err) - assert.JSONEq(t, tc.expected[receivedCount], val.(string)) - receivedCount += 1 - if receivedCount == len(tc.expected) { - cancel() - break wait - } - } - } - assert.NoError(t, g.Wait()) - }) - } -} - -func newTestServer( - newServer func(http.Handler) *httptest.Server, -) func(*testing.T, http.HandlerFunc, map[string]interface{}) { - return func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { - server := newServer(h) - config["request.url"] = server.URL - t.Cleanup(server.Close) - } -} - -func newV2Context() (v2.Context, func()) { - ctx, cancel := context.WithCancel(context.Background()) - return v2.Context{ - Logger: logp.NewLogger("httpjson_test"), - ID: "test_id", - Cancelation: ctx, - }, cancel -} - -func defaultHandler(expectedMethod, expectedBody string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - msg := `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}` - switch { - case r.Method != expectedMethod: - w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected method was %q"}`, expectedMethod) - case expectedBody != "": - body, _ := ioutil.ReadAll(r.Body) - r.Body.Close() - if expectedBody != string(body) { - w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected body was %q"}`, expectedBody) - } - } - - _, _ = w.Write([]byte(msg)) - } -} - -func rateLimitHandler() http.HandlerFunc { - var isRetry bool - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - if isRetry { - _, _ = w.Write([]byte(`{"hello":"world"}`)) - return - } - w.Header().Set("X-Rate-Limit-Limit", "0") - w.Header().Set("X-Rate-Limit-Remaining", "0") - w.Header().Set("X-Rate-Limit-Reset", fmt.Sprint(time.Now().Unix())) - w.WriteHeader(http.StatusTooManyRequests) - isRetry = true - _, _ = w.Write([]byte(`{"error":"too many requests"}`)) - } -} - -func retryHandler() http.HandlerFunc { - count := 0 - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - if count == 2 { - _, _ = w.Write([]byte(`{"hello":"world"}`)) - return - } - w.WriteHeader(rand.Intn(100) + 500) - count += 1 - } -} - -func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - _ = r.ParseForm() - switch { - case r.Method != "POST": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong method"}`)) - case r.FormValue("grant_type") != "client_credentials": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong grant_type"}`)) - case r.FormValue("client_id") != "a_client_id": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong client_id"}`)) - case r.FormValue("client_secret") != "a_client_secret": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong client_secret"}`)) - case r.FormValue("scope") != "scope1 scope2": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong scope"}`)) - case r.FormValue("param1") != "v1": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong param1"}`)) - default: - _, _ = w.Write([]byte(`{"token_type": "Bearer", "expires_in": "60", "access_token": "abcd"}`)) - } -} - -func oauth2Handler(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/token" { - oauth2TokenHandler(w, r) - return - } - - w.Header().Set("content-type", "application/json") - switch { - case r.Method != "POST": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong method"}`)) - case r.Header.Get("Authorization") != "Bearer abcd": - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong bearer"}`)) - default: - _, _ = w.Write([]byte(`{"hello":"world"}`)) - } -} - -func dateCursorHandler() http.HandlerFunc { - var count int - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - switch count { - case 0: - if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T14:50:00Z" { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong initial cursor value"`)) - return - } - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","foo":"bar"}`)) - case 1: - if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T15:00:00Z" { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong cursor value"`)) - return - } - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","foo":"bar"}`)) - case 2: - if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T15:00:01Z" { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong cursor value"`)) - return - } - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:02Z","foo":"bar"}`)) - } - count += 1 - } -} - -func paginationHandler() http.HandlerFunc { - var count int - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - switch count { - case 0: - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","nextPageToken":"bar","items":[{"foo":"a"}]}`)) - case 1: - if r.URL.Query().Get("page") != "bar" { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) - return - } - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","items":[{"foo":"b"}]}`)) - case 2: - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:02Z","items":[{"foo":"c"}]}`)) - case 3: - _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:03Z","items":[{"foo":"d"}]}`)) - } - count += 1 - } -} - -func paginationArrayHandler() http.HandlerFunc { - var count int - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("content-type", "application/json") - switch count { - case 0: - _, _ = w.Write([]byte(`[{"nextPageToken":"bar","foo":"bar"},{"foo":"bar"}]`)) - case 1: - if r.URL.Query().Get("page") != "bar" { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) - return - } - _, _ = w.Write([]byte(`[{"foo":"bar"}]`)) - } - count += 1 - } -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go deleted file mode 100644 index 6ea063a10af..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "context" - "io/ioutil" - "net/http" - "net/url" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -const paginationNamespace = "pagination" - -func registerPaginationTransforms() { - registerTransform(paginationNamespace, appendName, newAppendPagination) - registerTransform(paginationNamespace, deleteName, newDeletePagination) - registerTransform(paginationNamespace, setName, newSetRequestPagination) -} - -type pagination struct { - log *logp.Logger - httpClient *httpClient - requestFactory *requestFactory - decoder decoderFunc -} - -func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pagination { - pagination := &pagination{httpClient: httpClient, log: log} - if config.Response == nil { - return pagination - } - - pagination.decoder = registeredDecoders[config.Response.DecodeAs] - - if len(config.Response.Pagination) == 0 { - return pagination - } - - rts, _ := newBasicTransformsFromConfig(config.Request.Transforms, requestNamespace, log) - pts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace, log) - - body := func() *common.MapStr { - if config.Response.RequestBodyOnPagination { - return config.Request.Body - } - return &common.MapStr{} - }() - - requestFactory := newPaginationRequestFactory( - config.Request.Method, - config.Request.EncodeAs, - *config.Request.URL.URL, - body, - append(rts, pts...), - config.Auth, - log, - ) - pagination.requestFactory = requestFactory - return pagination -} - -func newPaginationRequestFactory(method, encodeAs string, url url.URL, body *common.MapStr, ts []basicTransform, authConfig *authConfig, log *logp.Logger) *requestFactory { - // config validation already checked for errors here - rf := &requestFactory{ - url: url, - method: method, - body: body, - transforms: ts, - log: log, - encoder: registeredEncoders[encodeAs], - } - if authConfig != nil && authConfig.Basic.isEnabled() { - rf.user = authConfig.Basic.User - rf.password = authConfig.Basic.Password - } - return rf -} - -type pageIterator struct { - pagination *pagination - - stdCtx context.Context - trCtx *transformContext - - resp *http.Response - - isFirst bool - done bool - - n int64 -} - -func (p *pagination) newPageIterator(stdCtx context.Context, trCtx *transformContext, resp *http.Response) *pageIterator { - return &pageIterator{ - pagination: p, - stdCtx: stdCtx, - trCtx: trCtx, - resp: resp, - isFirst: true, - } -} - -func (iter *pageIterator) next() (*response, bool, error) { - if iter == nil || iter.resp == nil || iter.done { - return nil, false, nil - } - - if iter.isFirst { - iter.isFirst = false - tr, err := iter.getPage() - if err != nil { - return nil, false, err - } - if iter.pagination.requestFactory == nil { - iter.done = true - } - return tr, true, nil - } - - httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) - if err != nil { - if err == errNewURLValueNotSet || - err == errEmptyTemplateResult || - err == errExecutingTemplate { - // if this error happens here it means a transform - // did not find any new value and we can stop paginating without error - iter.done = true - return nil, false, nil - } - return nil, false, err - } - - resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) - if err != nil { - return nil, false, err - } - - iter.resp = resp - - r, err := iter.getPage() - if err != nil { - return nil, false, err - } - - if r.body == nil { - iter.pagination.log.Debug("finished pagination because there is no body") - iter.done = true - return nil, false, nil - } - - return r, true, nil -} - -func (iter *pageIterator) getPage() (*response, error) { - bodyBytes, err := ioutil.ReadAll(iter.resp.Body) - if err != nil { - return nil, err - } - iter.resp.Body.Close() - iter.n += 1 - - var r response - r.header = iter.resp.Header - r.url = *iter.resp.Request.URL - r.page = iter.n - - if len(bodyBytes) > 0 { - if iter.pagination.decoder != nil { - err = iter.pagination.decoder(bodyBytes, &r) - } else { - err = decode(iter.resp.Header.Get("Content-Type"), bodyBytes, &r) - } - if err != nil { - return nil, err - } - } - - return &r, nil -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go deleted file mode 100644 index 569c0c84f18..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "context" - "errors" - "fmt" - "net/http" - "strconv" - "time" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -type rateLimiter struct { - log *logp.Logger - - limit *valueTpl - reset *valueTpl - remaining *valueTpl -} - -func newRateLimiterFromConfig(config *rateLimitConfig, log *logp.Logger) *rateLimiter { - if config == nil { - return nil - } - - return &rateLimiter{ - log: log, - limit: config.Limit, - reset: config.Reset, - remaining: config.Remaining, - } -} - -func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, error)) (*http.Response, error) { - for { - resp, err := f() - if err != nil { - return nil, err - } - - if err != nil { - return nil, fmt.Errorf("failed to read http.response.body: %w", err) - } - - if r == nil || resp.StatusCode == http.StatusOK { - return resp, nil - } - - if resp.StatusCode != http.StatusTooManyRequests { - return nil, fmt.Errorf("http request was unsuccessful with a status code %d", resp.StatusCode) - } - - if err := r.applyRateLimit(ctx, resp); err != nil { - return nil, err - } - } -} - -// applyRateLimit applies appropriate rate limit if specified in the HTTP Header of the response -func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) error { - epoch, err := r.getRateLimit(resp) - if err != nil { - return err - } - - t := time.Unix(epoch, 0) - w := time.Until(t) - if epoch == 0 || w <= 0 { - r.log.Debugf("Rate Limit: No need to apply rate limit.") - return nil - } - r.log.Debugf("Rate Limit: Wait until %v for the rate limit to reset.", t) - ticker := time.NewTicker(w) - defer ticker.Stop() - - select { - case <-ctx.Done(): - r.log.Info("Context done.") - return nil - case <-ticker.C: - r.log.Debug("Rate Limit: time is up.") - return nil - } -} - -// getRateLimit gets the rate limit value if specified in the response, -// and returns an int64 value in seconds since unix epoch for rate limit reset time. -// When there is a remaining rate limit quota, or when the rate limit reset time has expired, it -// returns 0 for the epoch value. -func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { - if r == nil { - return 0, nil - } - - if r.remaining == nil { - return 0, nil - } - - tr := transformable{} - ctx := emptyTransformContext() - ctx.updateLastResponse(response{header: resp.Header.Clone()}) - - remaining, _ := r.remaining.Execute(ctx, tr, nil, r.log) - if remaining == "" { - return 0, errors.New("remaining value is empty") - } - m, err := strconv.ParseInt(remaining, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse rate-limit remaining value: %w", err) - } - - if m != 0 { - return 0, nil - } - - if r.reset == nil { - r.log.Warn("reset rate limit is not set") - return 0, nil - } - - reset, _ := r.reset.Execute(ctx, tr, nil, r.log) - if reset == "" { - return 0, errors.New("reset value is empty") - } - - epoch, err := strconv.ParseInt(reset, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) - } - - if timeNow().Unix() > epoch { - return 0, nil - } - - return epoch, nil -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go deleted file mode 100644 index 8e9e0cb0db8..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "net/http" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -// Test getRateLimit function with a remaining quota, expect to receive 0, nil. -func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { - header := make(http.Header) - header.Add("X-Rate-Limit-Limit", "120") - header.Add("X-Rate-Limit-Remaining", "118") - header.Add("X-Rate-Limit-Reset", "1581658643") - tplLimit := &valueTpl{} - tplReset := &valueTpl{} - tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) - rateLimit := &rateLimiter{ - limit: tplLimit, - reset: tplReset, - remaining: tplRemaining, - log: logp.NewLogger(""), - } - resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) - assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) -} - -func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { - header := make(http.Header) - header.Add("X-Rate-Limit-Limit", "10") - header.Add("X-Rate-Limit-Remaining", "0") - header.Add("X-Rate-Limit-Reset", "1581658643") - tplLimit := &valueTpl{} - tplReset := &valueTpl{} - tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) - rateLimit := &rateLimiter{ - limit: tplLimit, - reset: tplReset, - remaining: tplRemaining, - log: logp.NewLogger(""), - } - resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) - assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) -} - -func TestGetRateLimitReturnsResetValue(t *testing.T) { - epoch := int64(1604582732 + 100) - timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } - t.Cleanup(func() { timeNow = time.Now }) - - header := make(http.Header) - header.Add("X-Rate-Limit-Limit", "10") - header.Add("X-Rate-Limit-Remaining", "0") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(epoch, 10)) - tplLimit := &valueTpl{} - tplReset := &valueTpl{} - tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) - rateLimit := &rateLimiter{ - limit: tplLimit, - reset: tplReset, - remaining: tplRemaining, - log: logp.NewLogger(""), - } - resp := &http.Response{Header: header} - epoch2, err := rateLimit.getRateLimit(resp) - assert.NoError(t, err) - assert.EqualValues(t, 1604582832, epoch2) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json b/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json deleted file mode 100644 index 2b5fdd89e5c..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "service_account", - "project_id": "foo", - "private_key_id": "x", - "client_email": "foo@bar.com", - "client_id": "0" -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json b/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json deleted file mode 100644 index 9977a2836c1..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json +++ /dev/null @@ -1 +0,0 @@ -invalid diff --git a/x-pack/filebeat/input/httpjson/pagination.go b/x-pack/filebeat/input/httpjson/pagination.go index 02bdb4b13de..a4fd8db5971 100644 --- a/x-pack/filebeat/input/httpjson/pagination.go +++ b/x-pack/filebeat/input/httpjson/pagination.go @@ -5,121 +5,180 @@ package httpjson import ( - "errors" - "fmt" + "context" + "io/ioutil" "net/http" "net/url" - "regexp" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) +const paginationNamespace = "pagination" + +func registerPaginationTransforms() { + registerTransform(paginationNamespace, appendName, newAppendPagination) + registerTransform(paginationNamespace, deleteName, newDeletePagination) + registerTransform(paginationNamespace, setName, newSetRequestPagination) +} + type pagination struct { - extraBodyContent common.MapStr - header *headerConfig - idField string - requestField string - urlField string - url string + log *logp.Logger + httpClient *httpClient + requestFactory *requestFactory + decoder decoderFunc } -func newPaginationFromConfig(config config) *pagination { - if !config.Pagination.isEnabled() { - return nil +func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pagination { + pagination := &pagination{httpClient: httpClient, log: log} + if config.Response == nil { + return pagination } - return &pagination{ - extraBodyContent: config.Pagination.ExtraBodyContent.Clone(), - header: config.Pagination.Header, - idField: config.Pagination.IDField, - requestField: config.Pagination.RequestField, - urlField: config.Pagination.URLField, - url: config.Pagination.URL, - } -} -func (p *pagination) nextRequestInfo(ri *requestInfo, response response, lastObj common.MapStr) (*requestInfo, bool, error) { - if p == nil { - return ri, false, nil + pagination.decoder = registeredDecoders[config.Response.DecodeAs] + + if len(config.Response.Pagination) == 0 { + return pagination } - if p.header == nil { - var err error - // Pagination control using HTTP Body fields - if err = p.setRequestInfoFromBody(response.body, lastObj, ri); err != nil { - // if the field is not found, there is no next page - if errors.Is(err, common.ErrKeyNotFound) { - return ri, false, nil - } - return ri, false, err + rts, _ := newBasicTransformsFromConfig(config.Request.Transforms, requestNamespace, log) + pts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace, log) + + body := func() *common.MapStr { + if config.Response.RequestBodyOnPagination { + return config.Request.Body } + return &common.MapStr{} + }() - return ri, true, nil - } + requestFactory := newPaginationRequestFactory( + config.Request.Method, + config.Request.EncodeAs, + *config.Request.URL.URL, + body, + append(rts, pts...), + config.Auth, + log, + ) + pagination.requestFactory = requestFactory + return pagination +} - // Pagination control using HTTP Header - url, err := getNextLinkFromHeader(response.header, p.header.FieldName, p.header.RegexPattern) - if err != nil { - return ri, false, fmt.Errorf("failed to retrieve the next URL for pagination: %w", err) +func newPaginationRequestFactory(method, encodeAs string, url url.URL, body *common.MapStr, ts []basicTransform, authConfig *authConfig, log *logp.Logger) *requestFactory { + // config validation already checked for errors here + rf := &requestFactory{ + url: url, + method: method, + body: body, + transforms: ts, + log: log, + encoder: registeredEncoders[encodeAs], } - if ri.url == url || url == "" { - return ri, false, nil + if authConfig != nil && authConfig.Basic.isEnabled() { + rf.user = authConfig.Basic.User + rf.password = authConfig.Basic.Password } + return rf +} + +type pageIterator struct { + pagination *pagination + + stdCtx context.Context + trCtx *transformContext - ri.url = url + resp *http.Response - return ri, true, nil + isFirst bool + done bool + + n int64 } -// getNextLinkFromHeader retrieves the next URL for pagination from the HTTP Header of the response -func getNextLinkFromHeader(header http.Header, fieldName string, re *regexp.Regexp) (string, error) { - links, ok := header[http.CanonicalHeaderKey(fieldName)] - if !ok { - return "", fmt.Errorf("field %s does not exist in the HTTP Header", fieldName) +func (p *pagination) newPageIterator(stdCtx context.Context, trCtx *transformContext, resp *http.Response) *pageIterator { + return &pageIterator{ + pagination: p, + stdCtx: stdCtx, + trCtx: trCtx, + resp: resp, + isFirst: true, } - for _, link := range links { - matchArray := re.FindAllStringSubmatch(link, -1) - if len(matchArray) == 1 { - return matchArray[0][1], nil +} + +func (iter *pageIterator) next() (*response, bool, error) { + if iter == nil || iter.resp == nil || iter.done { + return nil, false, nil + } + + if iter.isFirst { + iter.isFirst = false + tr, err := iter.getPage() + if err != nil { + return nil, false, err + } + if iter.pagination.requestFactory == nil { + iter.done = true } + return tr, true, nil } - return "", nil -} -// createRequestInfoFromBody creates a new RequestInfo for a new HTTP request in pagination based on HTTP response body -func (p *pagination) setRequestInfoFromBody(response, last common.MapStr, ri *requestInfo) error { - // we try to get it from last element, if not found, from the original response - v, err := last.GetValue(p.idField) - if err == common.ErrKeyNotFound { - v, err = response.GetValue(p.idField) + httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) + if err != nil { + if err == errNewURLValueNotSet || + err == errEmptyTemplateResult || + err == errExecutingTemplate { + // if this error happens here it means a transform + // did not find any new value and we can stop paginating without error + iter.done = true + return nil, false, nil + } + return nil, false, err } + resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) if err != nil { - return fmt.Errorf("failed to retrieve id_field for pagination: %w", err) + return nil, false, err } - if p.requestField != "" { - _, _ = ri.contentMap.Put(p.requestField, v) - if p.url != "" { - ri.url = p.url - } - } else if p.urlField != "" { - url, err := url.Parse(ri.url) - if err == nil { - q := url.Query() - q.Set(p.urlField, fmt.Sprint(v)) - url.RawQuery = q.Encode() - ri.url = url.String() + iter.resp = resp + + r, err := iter.getPage() + if err != nil { + return nil, false, err + } + + if r.body == nil { + iter.pagination.log.Debug("finished pagination because there is no body") + iter.done = true + return nil, false, nil + } + + return r, true, nil +} + +func (iter *pageIterator) getPage() (*response, error) { + bodyBytes, err := ioutil.ReadAll(iter.resp.Body) + if err != nil { + return nil, err + } + iter.resp.Body.Close() + iter.n += 1 + + var r response + r.header = iter.resp.Header + r.url = *iter.resp.Request.URL + r.page = iter.n + + if len(bodyBytes) > 0 { + if iter.pagination.decoder != nil { + err = iter.pagination.decoder(bodyBytes, &r) + } else { + err = decode(iter.resp.Header.Get("Content-Type"), bodyBytes, &r) } - } else { - switch vt := v.(type) { - case string: - ri.url = vt - default: - return errors.New("pagination ID is not of string type") + if err != nil { + return nil, err } } - if len(p.extraBodyContent) > 0 { - ri.contentMap.Update(common.MapStr(p.extraBodyContent)) - } - return nil + + return &r, nil } diff --git a/x-pack/filebeat/input/httpjson/pagination_test.go b/x-pack/filebeat/input/httpjson/pagination_test.go deleted file mode 100644 index 17dcae4fc62..00000000000 --- a/x-pack/filebeat/input/httpjson/pagination_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "net/http" - "regexp" - "testing" - - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestGetNextLinkFromHeader(t *testing.T) { - header := make(http.Header) - header.Add("link", "; rel=\"self\"") - header.Add("Link", "; rel=\"next\"") - re, _ := regexp.Compile("<([^>]+)>; *rel=\"next\"(?:,|$)") - url, err := getNextLinkFromHeader(header, "Link", re) - if url != "https://dev-168980.okta.com/api/v1/logs?after=1581658181086_1" { - t.Fatal("Failed to test getNextLinkFromHeader. URL " + url + " is not expected") - } - if err != nil { - t.Fatal("Failed to test getNextLinkFromHeader with error:", err) - } -} - -func TestCreateRequestInfoFromBody(t *testing.T) { - m := map[string]interface{}{ - "id": 100, - } - extraBodyContent := common.MapStr{"extra_body": "abc"} - pagination := &pagination{ - idField: "id", - requestField: "pagination_id", - extraBodyContent: extraBodyContent, - url: "https://test-123", - } - ri := &requestInfo{ - url: "", - contentMap: common.MapStr{}, - headers: common.MapStr{}, - } - _ = pagination.setRequestInfoFromBody( - common.MapStr(m), - common.MapStr(m), - ri, - ) - if ri.url != "https://test-123" { - t.Fatal("Failed to test createRequestInfoFromBody. URL should be https://test-123.") - } - p, err := ri.contentMap.GetValue("pagination_id") - if err != nil { - t.Fatal("Failed to test createRequestInfoFromBody with error", err) - } - switch pt := p.(type) { - case int: - if pt != 100 { - t.Fatalf("Failed to test createRequestInfoFromBody. pagination_id value %d should be 100.", pt) - } - default: - t.Fatalf("Failed to test createRequestInfoFromBody. pagination_id value %T should be int.", pt) - } - b, err := ri.contentMap.GetValue("extra_body") - if err != nil { - t.Fatal("Failed to test createRequestInfoFromBody with error", err) - } - switch bt := b.(type) { - case string: - if bt != "abc" { - t.Fatalf("Failed to test createRequestInfoFromBody. extra_body value %s does not match \"abc\".", bt) - } - default: - t.Fatalf("Failed to test createRequestInfoFromBody. extra_body type %T should be string.", bt) - } -} diff --git a/x-pack/filebeat/input/httpjson/rate_limiter.go b/x-pack/filebeat/input/httpjson/rate_limiter.go index 93c2b4a3fe7..ab2ef2715e9 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter.go @@ -6,6 +6,7 @@ package httpjson import ( "context" + "errors" "fmt" "net/http" "strconv" @@ -17,32 +18,31 @@ import ( type rateLimiter struct { log *logp.Logger - limit string - reset string - remaining string + limit *valueTpl + reset *valueTpl + remaining *valueTpl } -func newRateLimiterFromConfig(config config, log *logp.Logger) *rateLimiter { - if config.RateLimit == nil { +func newRateLimiterFromConfig(config *rateLimitConfig, log *logp.Logger) *rateLimiter { + if config == nil { return nil } return &rateLimiter{ log: log, - limit: config.RateLimit.Limit, - reset: config.RateLimit.Reset, - remaining: config.RateLimit.Remaining, + limit: config.Limit, + reset: config.Reset, + remaining: config.Remaining, } } -func (r *rateLimiter) execute(ctx context.Context, f func(context.Context) (*http.Response, error)) (*http.Response, error) { +func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, error)) (*http.Response, error) { for { - resp, err := f(ctx) + resp, err := f() if err != nil { return nil, err } - header := resp.Header if err != nil { return nil, fmt.Errorf("failed to read http.response.body: %w", err) } @@ -55,15 +55,15 @@ func (r *rateLimiter) execute(ctx context.Context, f func(context.Context) (*htt return nil, fmt.Errorf("http request was unsuccessful with a status code %d", resp.StatusCode) } - if err := r.applyRateLimit(ctx, header); err != nil { + if err := r.applyRateLimit(ctx, resp); err != nil { return nil, err } } } // applyRateLimit applies appropriate rate limit if specified in the HTTP Header of the response -func (r *rateLimiter) applyRateLimit(ctx context.Context, header http.Header) error { - epoch, err := r.getRateLimit(header) +func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) error { + epoch, err := r.getRateLimit(resp) if err != nil { return err } @@ -88,22 +88,26 @@ func (r *rateLimiter) applyRateLimit(ctx context.Context, header http.Header) er } } -// getRateLimit gets the rate limit value if specified in the HTTP Header of the response, +// getRateLimit gets the rate limit value if specified in the response, // and returns an int64 value in seconds since unix epoch for rate limit reset time. // When there is a remaining rate limit quota, or when the rate limit reset time has expired, it // returns 0 for the epoch value. -func (r *rateLimiter) getRateLimit(header http.Header) (int64, error) { +func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { if r == nil { return 0, nil } - if r.remaining == "" { + if r.remaining == nil { return 0, nil } - remaining := header.Get(r.remaining) + tr := transformable{} + ctx := emptyTransformContext() + ctx.updateLastResponse(response{header: resp.Header.Clone()}) + + remaining, _ := r.remaining.Execute(ctx, tr, nil, r.log) if remaining == "" { - return 0, fmt.Errorf("field %s does not exist in the HTTP Header, or is empty", r.remaining) + return 0, errors.New("remaining value is empty") } m, err := strconv.ParseInt(remaining, 10, 64) if err != nil { @@ -114,15 +118,22 @@ func (r *rateLimiter) getRateLimit(header http.Header) (int64, error) { return 0, nil } - reset := header.Get(r.reset) + if r.reset == nil { + r.log.Warn("reset rate limit is not set") + return 0, nil + } + + reset, _ := r.reset.Execute(ctx, tr, nil, r.log) if reset == "" { - return 0, fmt.Errorf("field %s does not exist in the HTTP Header, or is empty", r.reset) + return 0, errors.New("reset value is empty") } + epoch, err := strconv.ParseInt(reset, 10, 64) if err != nil { return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) } - if time.Until(time.Unix(epoch, 0)) <= 0 { + + if timeNow().Unix() > epoch { return 0, nil } diff --git a/x-pack/filebeat/input/httpjson/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/rate_limiter_test.go index e349e725f2f..7ede16514d4 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter_test.go @@ -9,56 +9,82 @@ import ( "strconv" "testing" "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/logp" ) // Test getRateLimit function with a remaining quota, expect to receive 0, nil. -func TestGetRateLimitCase1(t *testing.T) { +func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "118") header.Add("X-Rate-Limit-Reset", "1581658643") + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ - limit: "X-Rate-Limit-Limit", - reset: "X-Rate-Limit-Reset", - remaining: "X-Rate-Limit-Remaining", - } - epoch, err := rateLimit.getRateLimit(header) - if err != nil || epoch != 0 { - t.Fatal("Failed to test getRateLimit.") + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + log: logp.NewLogger(""), } + resp := &http.Response{Header: header} + epoch, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 0, epoch) } -// Test getRateLimit function with a past time, expect to receive 0, nil. -func TestGetRateLimitCase2(t *testing.T) { +func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "10") header.Add("X-Rate-Limit-Remaining", "0") header.Add("X-Rate-Limit-Reset", "1581658643") + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ - limit: "X-Rate-Limit-Limit", - reset: "X-Rate-Limit-Reset", - remaining: "X-Rate-Limit-Remaining", - } - epoch, err := rateLimit.getRateLimit(header) - if err != nil || epoch != 0 { - t.Fatal("Failed to test getRateLimit.") + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + log: logp.NewLogger(""), } + resp := &http.Response{Header: header} + epoch, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 0, epoch) } -// Test getRateLimit function with a time yet to come, expect to receive , nil. -func TestGetRateLimitCase3(t *testing.T) { - epoch := time.Now().Unix() + 100 +func TestGetRateLimitReturnsResetValue(t *testing.T) { + epoch := int64(1604582732 + 100) + timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } + t.Cleanup(func() { timeNow = time.Now }) + header := make(http.Header) header.Add("X-Rate-Limit-Limit", "10") header.Add("X-Rate-Limit-Remaining", "0") header.Add("X-Rate-Limit-Reset", strconv.FormatInt(epoch, 10)) + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ - limit: "X-Rate-Limit-Limit", - reset: "X-Rate-Limit-Reset", - remaining: "X-Rate-Limit-Remaining", - } - epoch2, err := rateLimit.getRateLimit(header) - if err != nil || epoch2 != epoch { - t.Fatal("Failed to test getRateLimit.") + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + log: logp.NewLogger(""), } + resp := &http.Response{Header: header} + epoch2, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 1604582832, epoch2) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/request.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/request.go rename to x-pack/filebeat/input/httpjson/request.go index 921b13b9ab7..0c5fbc39fa2 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "bytes" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request_test.go b/x-pack/filebeat/input/httpjson/request_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/request_test.go rename to x-pack/filebeat/input/httpjson/request_test.go index 6e35d5cc45f..34ec55f5274 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request_test.go +++ b/x-pack/filebeat/input/httpjson/request_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "context" diff --git a/x-pack/filebeat/input/httpjson/requester.go b/x-pack/filebeat/input/httpjson/requester.go deleted file mode 100644 index 9e0e26edf2d..00000000000 --- a/x-pack/filebeat/input/httpjson/requester.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "strings" - - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -type requestInfo struct { - url string - contentMap common.MapStr - headers common.MapStr -} - -type requester struct { - log *logp.Logger - client *http.Client - dateCursor *dateCursor - rateLimiter *rateLimiter - pagination *pagination - - method string - reqBody common.MapStr - headers common.MapStr - noHTTPBody bool - apiKey string - authScheme string - jsonObjects string - splitEventsBy string - - cursorState cursorState -} - -func newRequester( - config config, - rateLimiter *rateLimiter, - dateCursor *dateCursor, - pagination *pagination, - client *http.Client, - log *logp.Logger) *requester { - return &requester{ - log: log, - client: client, - rateLimiter: rateLimiter, - dateCursor: dateCursor, - pagination: pagination, - method: config.HTTPMethod, - reqBody: config.HTTPRequestBody.Clone(), - headers: config.HTTPHeaders.Clone(), - noHTTPBody: config.NoHTTPBody, - apiKey: config.APIKey, - authScheme: config.AuthenticationScheme, - splitEventsBy: config.SplitEventsBy, - jsonObjects: config.JSONObjects, - } -} - -type response struct { - header http.Header - body common.MapStr -} - -// processHTTPRequest processes HTTP request, and handles pagination if enabled -func (r *requester) processHTTPRequest(ctx context.Context, publisher cursor.Publisher) error { - ri := &requestInfo{ - url: r.dateCursor.getURL(r.cursorState.LastDateCursorValue), - contentMap: common.MapStr{}, - headers: r.headers, - } - - if r.method == "POST" && r.reqBody != nil { - ri.contentMap.Update(common.MapStr(r.reqBody)) - } - - var ( - m, v interface{} - response response - lastObj common.MapStr - ) - - // always request at least once - hasNext := true - - for hasNext { - resp, err := r.rateLimiter.execute( - ctx, - func(ctx context.Context) (*http.Response, error) { - req, err := r.createHTTPRequest(ctx, ri) - if err != nil { - return nil, fmt.Errorf("failed to create http request: %w", err) - } - msg, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to execute http client.Do: %w", err) - } - return msg, nil - }, - ) - if err != nil { - return err - } - - response.header = resp.Header - responseData, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("failed to read http response: %w", err) - } - _ = resp.Body.Close() - - if err = json.Unmarshal(responseData, &m); err != nil { - r.log.Debug("failed to unmarshal http.response.body", string(responseData)) - return fmt.Errorf("failed to unmarshal http.response.body: %w", err) - } - - switch obj := m.(type) { - // Top level Array - case []interface{}: - lastObj, err = r.processEventArray(publisher, obj) - if err != nil { - return err - } - case map[string]interface{}: - response.body = obj - if r.jsonObjects == "" { - lastObj, err = r.processEventArray(publisher, []interface{}{obj}) - if err != nil { - return err - } - } else { - v, err = common.MapStr(obj).GetValue(r.jsonObjects) - if err != nil { - if err == common.ErrKeyNotFound { - break - } - return err - } - switch ts := v.(type) { - case []interface{}: - lastObj, err = r.processEventArray(publisher, ts) - if err != nil { - return err - } - default: - return fmt.Errorf("content of %s is not a valid array", r.jsonObjects) - } - } - default: - r.log.Debug("http.response.body is not a valid JSON object", string(responseData)) - return fmt.Errorf("http.response.body is not a valid JSON object, but a %T", obj) - } - - ri, hasNext, err = r.pagination.nextRequestInfo(ri, response, lastObj) - if err != nil { - return err - } - - if lastObj != nil && r.dateCursor.enabled { - r.updateCursorState(ri.url, r.dateCursor.getNextValue(common.MapStr(lastObj))) - } - } - - return nil -} - -// createHTTPRequest creates an HTTP/HTTPs request for the input -func (r *requester) createHTTPRequest(ctx context.Context, ri *requestInfo) (*http.Request, error) { - var body io.Reader - if len(ri.contentMap) == 0 || r.noHTTPBody { - body = nil - } else { - b, err := json.Marshal(ri.contentMap) - if err != nil { - return nil, err - } - body = bytes.NewReader(b) - } - req, err := http.NewRequest(r.method, ri.url, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - if r.apiKey != "" { - if r.authScheme != "" { - req.Header.Set("Authorization", r.authScheme+" "+r.apiKey) - } else { - req.Header.Set("Authorization", r.apiKey) - } - } - for k, v := range ri.headers { - switch vv := v.(type) { - case string: - req.Header.Set(k, vv) - default: - } - } - return req, nil -} - -// processEventArray publishes an event for each object contained in the array. It returns the last object in the array and an error if any. -func (r *requester) processEventArray(publisher cursor.Publisher, events []interface{}) (map[string]interface{}, error) { - var last map[string]interface{} - for _, t := range events { - switch v := t.(type) { - case map[string]interface{}: - for _, e := range splitEvent(r.splitEventsBy, v) { - last = e - d, err := json.Marshal(e) - if err != nil { - return nil, fmt.Errorf("failed to marshal %+v: %w", e, err) - } - if err := publisher.Publish(makeEvent(string(d)), r.cursorState); err != nil { - return nil, fmt.Errorf("failed to publish: %w", err) - } - } - default: - return nil, fmt.Errorf("expected only JSON objects in the array but got a %T", v) - } - } - return last, nil -} - -func splitEvent(splitKey string, event map[string]interface{}) []map[string]interface{} { - m := common.MapStr(event) - - // NOTE: this notation is only used internally, not meant to be documented - // and will be removed in the next release - keys := strings.SplitN(splitKey, "..", 2) - if len(keys) < 2 { - // we append an empty key to force the recursive call - keys = append(keys, "") - } - - hasSplitKey, _ := m.HasKey(keys[0]) - if keys[0] == "" || !hasSplitKey { - return []map[string]interface{}{event} - } - - splitOnIfc, _ := m.GetValue(keys[0]) - splitOn, ok := splitOnIfc.([]interface{}) - // if not an array or is empty, we do nothing - if !ok || len(splitOn) == 0 { - return []map[string]interface{}{event} - } - - var events []map[string]interface{} - for _, split := range splitOn { - s, ok := split.(map[string]interface{}) - // if not an object, we do nothing - if !ok { - return []map[string]interface{}{event} - } - - // call splitEvent recursively for each part - for _, nestedSplit := range splitEvent(keys[1], s) { - mm := m.Clone() - if _, err := mm.Put(keys[0], nestedSplit); err != nil { - return []map[string]interface{}{event} - } - events = append(events, mm) - } - } - - return events -} - -type cursorState struct { - LastCalledURL string - LastDateCursorValue string -} - -func (r *requester) updateCursorState(url, value string) { - r.cursorState.LastCalledURL = url - r.cursorState.LastDateCursorValue = value -} - -func (r *requester) loadCursor(c *cursor.Cursor, log *logp.Logger) { - if c == nil || c.IsNew() { - return - } - - if err := c.Unpack(&r.cursorState); err != nil { - log.Errorf("Reset http cursor state. Failed to read from registry: %v", err) - } -} diff --git a/x-pack/filebeat/input/httpjson/requester_test.go b/x-pack/filebeat/input/httpjson/requester_test.go deleted file mode 100644 index 31e65a57c73..00000000000 --- a/x-pack/filebeat/input/httpjson/requester_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package httpjson - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSplitEventsBy(t *testing.T) { - event := map[string]interface{}{ - "this": "is kept", - "alerts": []interface{}{ - map[string]interface{}{ - "this_is": "also kept", - "entities": []interface{}{ - map[string]interface{}{ - "something": "something", - }, - map[string]interface{}{ - "else": "else", - }, - }, - }, - map[string]interface{}{ - "this_is": "also kept 2", - "entities": []interface{}{ - map[string]interface{}{ - "something": "something 2", - }, - map[string]interface{}{ - "else": "else 2", - }, - }, - }, - }, - } - - expectedEvents := []map[string]interface{}{ - { - "this": "is kept", - "alerts": map[string]interface{}{ - "this_is": "also kept", - "entities": map[string]interface{}{ - "something": "something", - }, - }, - }, - { - "this": "is kept", - "alerts": map[string]interface{}{ - "this_is": "also kept", - "entities": map[string]interface{}{ - "else": "else", - }, - }, - }, - { - "this": "is kept", - "alerts": map[string]interface{}{ - "this_is": "also kept 2", - "entities": map[string]interface{}{ - "something": "something 2", - }, - }, - }, - { - "this": "is kept", - "alerts": map[string]interface{}{ - "this_is": "also kept 2", - "entities": map[string]interface{}{ - "else": "else 2", - }, - }, - }, - } - - const key = "alerts..entities" - - got := splitEvent(key, event) - - assert.Equal(t, expectedEvents, got) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response.go b/x-pack/filebeat/input/httpjson/response.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/response.go rename to x-pack/filebeat/input/httpjson/response.go index 7513c8476b7..21fd85e20f6 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response.go +++ b/x-pack/filebeat/input/httpjson/response.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "context" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response_test.go b/x-pack/filebeat/input/httpjson/response_test.go similarity index 98% rename from x-pack/filebeat/input/httpjson/internal/v2/response_test.go rename to x-pack/filebeat/input/httpjson/response_test.go index f3dd507b394..a855ea87f64 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response_test.go +++ b/x-pack/filebeat/input/httpjson/response_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/split.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/split.go rename to x-pack/filebeat/input/httpjson/split.go index 56c89f7f9ef..c85b25a4341 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/split.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "errors" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/split_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/split_test.go rename to x-pack/filebeat/input/httpjson/split_test.go index c385771667b..3e5bff77c79 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/split_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "testing" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/transform.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform.go rename to x-pack/filebeat/input/httpjson/transform.go index 577edc2ba85..d64e57f4390 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/transform.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "encoding/json" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/transform_append.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_append.go rename to x-pack/filebeat/input/httpjson/transform_append.go index f289bcd4ef7..6470d0717a4 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/transform_append.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "fmt" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go b/x-pack/filebeat/input/httpjson/transform_append_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go rename to x-pack/filebeat/input/httpjson/transform_append_test.go index 127492cb262..023798bc719 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go +++ b/x-pack/filebeat/input/httpjson/transform_append_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/transform_delete.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go rename to x-pack/filebeat/input/httpjson/transform_delete.go index c8c54b8141e..a44960074e4 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go +++ b/x-pack/filebeat/input/httpjson/transform_delete.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "fmt" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go b/x-pack/filebeat/input/httpjson/transform_delete_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go rename to x-pack/filebeat/input/httpjson/transform_delete_test.go index 22cbce310f9..86985717aad 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go +++ b/x-pack/filebeat/input/httpjson/transform_delete_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go b/x-pack/filebeat/input/httpjson/transform_registry.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go rename to x-pack/filebeat/input/httpjson/transform_registry.go index f0073f29277..0a41262b64b 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go +++ b/x-pack/filebeat/input/httpjson/transform_registry.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "errors" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/transform_set.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_set.go rename to x-pack/filebeat/input/httpjson/transform_set.go index c38f1719faf..779ed0b5519 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/transform_set.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "fmt" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/transform_set_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go rename to x-pack/filebeat/input/httpjson/transform_set_test.go index 6a3a2d8915c..f459dbeb82f 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go +++ b/x-pack/filebeat/input/httpjson/transform_set_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go b/x-pack/filebeat/input/httpjson/transform_target.go similarity index 98% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_target.go rename to x-pack/filebeat/input/httpjson/transform_target.go index 2fd6d83d3c0..eb7a45a891a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go +++ b/x-pack/filebeat/input/httpjson/transform_target.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "fmt" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go b/x-pack/filebeat/input/httpjson/transform_target_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go rename to x-pack/filebeat/input/httpjson/transform_target_test.go index 2042c8dab38..5323853def8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go +++ b/x-pack/filebeat/input/httpjson/transform_target_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "testing" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go b/x-pack/filebeat/input/httpjson/transform_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/transform_test.go rename to x-pack/filebeat/input/httpjson/transform_test.go index 2ac43b3faf9..cbe90614a8f 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go +++ b/x-pack/filebeat/input/httpjson/transform_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go rename to x-pack/filebeat/input/httpjson/value_tpl.go index ed37af4f3dd..4dfdffe0983 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "bytes" diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go similarity index 99% rename from x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go rename to x-pack/filebeat/input/httpjson/value_tpl_test.go index 14caa3ca1d3..f925c8def9a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package v2 +package httpjson import ( "net/http" diff --git a/x-pack/filebeat/module/cisco/amp/config/config.yml b/x-pack/filebeat/module/cisco/amp/config/config.yml index e125fb7dc2c..49c4cac717e 100644 --- a/x-pack/filebeat/module/cisco/amp/config/config.yml +++ b/x-pack/filebeat/module/cisco/amp/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} {{ if .client_id }} diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml index 6cb67fd728e..48c8d847f9c 100644 --- a/x-pack/filebeat/module/google_workspace/admin/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/admin/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml index 941e9cb4383..037fc38cabe 100644 --- a/x-pack/filebeat/module/google_workspace/drive/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/drive/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml index 62833d56d58..60dd85a169d 100644 --- a/x-pack/filebeat/module/google_workspace/groups/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/groups/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml index 09de8ef45d1..3c26a059bba 100644 --- a/x-pack/filebeat/module/google_workspace/login/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/login/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml index 87c754e5107..e5736f5f33b 100644 --- a/x-pack/filebeat/module/google_workspace/saml/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/saml/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml index 4807b79e8ed..65fdffb84c1 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml @@ -1,6 +1,5 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} auth.oauth2.provider: google auth.oauth2.google.jwt_file: {{ .jwt_file }} diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 7577ee3e932..41675191f3f 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: POST diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index f112eec3916..8beb7e70e34 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} {{ if .ssl }} diff --git a/x-pack/filebeat/module/snyk/audit/config/config.yml b/x-pack/filebeat/module/snyk/audit/config/config.yml index 7ff98b032f2..7476d4dd1cf 100644 --- a/x-pack/filebeat/module/snyk/audit/config/config.yml +++ b/x-pack/filebeat/module/snyk/audit/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} {{ if .audit_type }} {{ if eq .audit_type "group" }} diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml index 90af60fbad0..a78cc297493 100644 --- a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml +++ b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.url: {{ .url }} diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml index c699705e268..e7f4f8cc264 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: GET diff --git a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml index 2da071910c2..da01bc61c40 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: GET diff --git a/x-pack/filebeat/module/threatintel/anomali/config/config.yml b/x-pack/filebeat/module/threatintel/anomali/config/config.yml index 3668a0b2f55..a268ddb7559 100644 --- a/x-pack/filebeat/module/threatintel/anomali/config/config.yml +++ b/x-pack/filebeat/module/threatintel/anomali/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} {{ if .username }} diff --git a/x-pack/filebeat/module/threatintel/malwarebazaar/config/config.yml b/x-pack/filebeat/module/threatintel/malwarebazaar/config/config.yml index 8426de172c5..af6d77335d2 100644 --- a/x-pack/filebeat/module/threatintel/malwarebazaar/config/config.yml +++ b/x-pack/filebeat/module/threatintel/malwarebazaar/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: POST diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index aa5e6222d80..8efcd615f06 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: POST diff --git a/x-pack/filebeat/module/threatintel/otx/config/config.yml b/x-pack/filebeat/module/threatintel/otx/config/config.yml index 04edde00164..9e2f5169e6f 100644 --- a/x-pack/filebeat/module/threatintel/otx/config/config.yml +++ b/x-pack/filebeat/module/threatintel/otx/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: GET diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml index aff04f46146..08b2f682f3f 100644 --- a/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml +++ b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml @@ -1,7 +1,6 @@ {{ if eq .input "httpjson" }} type: httpjson -config_version: "2" interval: {{ .interval }} request.method: GET