diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 41de16190a8..8ff5fbee069 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -25,6 +25,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179 - Update to Golang 1.12.1. {pull}11330[11330] - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812] +- API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759] *Auditbeat* @@ -78,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706] - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993] - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] +- API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759] *Heartbeat* @@ -95,6 +97,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix ECS compliance of user.id field in system/users metricset {pull}19019[19019] - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718] - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457] +- API address and shard ID are required settings in the Cloud Foundry module. {pull}21759[21759] *Packetbeat* diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml index f4c4943d4bb..f3d58f079a1 100644 --- a/deploy/cloudfoundry/filebeat/filebeat.yml +++ b/deploy/cloudfoundry/filebeat/filebeat.yml @@ -11,7 +11,7 @@ filebeat.inputs: #doppler_address: ${DOPPLER_ADDRESS} #uaa_address: ${UAA_ADDRESS} #rlp_address: ${RLP_ADDRESS} - #shard_id: ${SHARD_ID} + shard_id: ${SHARD_ID} #version: v1 diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml index c89c00ce983..12cfee75f00 100644 --- a/deploy/cloudfoundry/metricbeat/metricbeat.yml +++ b/deploy/cloudfoundry/metricbeat/metricbeat.yml @@ -11,7 +11,8 @@ metricbeat.modules: #doppler_address: ${DOPPLER_ADDRESS} #uaa_address: ${UAA_ADDRESS} #rlp_address: ${RLP_ADDRESS} - #shard_id: ${SHARD_ID} + shard_id: ${SHARD_ID} + #version: v1 #================================ Outputs ===================================== diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc index c08efa30c5f..5aa3ef95837 100644 --- a/filebeat/docs/running-on-cloudfoundry.asciidoc +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -71,11 +71,3 @@ filebeat started 1/1 512M 1G Log events should start flowing to Elasticsearch. The events are annotated with metadata added by the <> processor. - - -[WARNING] -======================================= -*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that -{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` -will result in duplicate events being pushed to Elasticsearch. -======================================= diff --git a/metricbeat/docs/modules/cloudfoundry.asciidoc b/metricbeat/docs/modules/cloudfoundry.asciidoc index 4d153933c23..614c703d155 100644 --- a/metricbeat/docs/modules/cloudfoundry.asciidoc +++ b/metricbeat/docs/modules/cloudfoundry.asciidoc @@ -117,7 +117,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "". === `shard_id` Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events -from the RLP Gateway. Default: "(generated UUID)". +from the RLP Gateway. [float] ==== `version` @@ -152,6 +152,7 @@ metricbeat.modules: rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 ---- diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc index 5fdf20c9f72..59802ba505d 100644 --- a/metricbeat/docs/running-on-cloudfoundry.asciidoc +++ b/metricbeat/docs/running-on-cloudfoundry.asciidoc @@ -71,11 +71,3 @@ metricbeat started 1/1 512M 1G Metrics should start flowing to Elasticsearch. The events are annotated with metadata added by the <> processor. - - -[WARNING] -======================================= -*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that -{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` -will result in duplicate events being pushed to Elasticsearch. -======================================= diff --git a/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc b/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc index f8d5dd51015..551d0095b3a 100644 --- a/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc @@ -23,6 +23,7 @@ Example configurations: api_address: https://api.dev.cfdev.sh client_id: uaa-filebeat client_secret: verysecret + shard_id: filebeat ssl: verification_mode: none ---- @@ -34,6 +35,7 @@ Example configurations: api_address: https://api.dev.cfdev.sh client_id: uaa-filebeat client_secret: verysecret + shard_id: filebeat ssl.certificate_authorities: ["/etc/pki/cf/ca.pem"] ssl.certificate: "/etc/pki/cf/cert.pem" ssl.key: "/etc/pki/cf/cert.key" diff --git a/x-pack/libbeat/common/cloudfoundry/config.go b/x-pack/libbeat/common/cloudfoundry/config.go index 0724bdc66e1..2f15d0c7cf9 100644 --- a/x-pack/libbeat/common/cloudfoundry/config.go +++ b/x-pack/libbeat/common/cloudfoundry/config.go @@ -10,8 +10,6 @@ import ( "strings" "time" - "github.com/gofrs/uuid" - "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) @@ -32,14 +30,14 @@ type Config struct { TLS *tlscommon.Config `config:"ssl"` // Override URLs returned from the CF client - APIAddress string `config:"api_address"` + APIAddress string `config:"api_address" validate:"required"` DopplerAddress string `config:"doppler_address"` UaaAddress string `config:"uaa_address"` RlpAddress string `config:"rlp_address"` // ShardID when retrieving events from loggregator, sharing this ID across // multiple filebeats will shard the load of receiving and sending events. - ShardID string `config:"shard_id"` + ShardID string `config:"shard_id" validate:"required"` // Maximum amount of time to cache application objects from CF client. CacheDuration time.Duration `config:"cache_duration"` @@ -50,13 +48,6 @@ type Config struct { // InitDefaults initialize the defaults for the configuration. func (c *Config) InitDefaults() { - // If not provided by the user; subscription ID should be a unique string to avoid clustering by default. - // Default to using a UUID4 string. - uuid, err := uuid.NewV4() - if err != nil { - panic(err) - } - c.ShardID = uuid.String() c.CacheDuration = 120 * time.Second c.CacheRetryDelay = 20 * time.Second c.Version = ConsumerVersionV1 diff --git a/x-pack/libbeat/common/cloudfoundry/config_test.go b/x-pack/libbeat/common/cloudfoundry/config_test.go index bce98a0b642..79ba7d47cc1 100644 --- a/x-pack/libbeat/common/cloudfoundry/config_test.go +++ b/x-pack/libbeat/common/cloudfoundry/config_test.go @@ -22,26 +22,48 @@ func TestValidation(t *testing.T) { var noId Config assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&noId)) var noSecret Config assert.Error(t, ucfg.MustNewFrom(common.MapStr{ - "client_id": "client_id", + "api_address": "https://api.dev.cfdev.sh", + "client_id": "client_id", + "shard_id": "beats-test-1", }).Unpack(&noSecret)) + var noAPI Config + assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "client_id": "client_id", + "client_secret": "client_secret", + "shard_id": "beats-test-1", + }).Unpack(&noAPI)) + + var noShardID Config + assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", + "client_id": "client_id", + "client_secret": "client_secret", + }).Unpack(&noShardID)) + var valid Config assert.NoError(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_id": "client_id", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&valid)) } func TestInitDefaults(t *testing.T) { var cfCfg Config assert.NoError(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_id": "client_id", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&cfCfg)) - assert.Len(t, cfCfg.ShardID, 36) + assert.Equal(t, ConsumerVersionV1, cfCfg.Version) } diff --git a/x-pack/libbeat/common/cloudfoundry/test/config.go b/x-pack/libbeat/common/cloudfoundry/test/config.go index f7b9cd18ffb..45059fc559c 100644 --- a/x-pack/libbeat/common/cloudfoundry/test/config.go +++ b/x-pack/libbeat/common/cloudfoundry/test/config.go @@ -7,15 +7,23 @@ package test import ( "os" "testing" + + "github.com/gofrs/uuid" ) func GetConfigFromEnv(t *testing.T) map[string]interface{} { t.Helper() + shardID, err := uuid.NewV4() + if err != nil { + t.Fatalf("Unable to create a random shard ID: %v", err) + } + config := map[string]interface{}{ "api_address": lookupEnv(t, "CLOUDFOUNDRY_API_ADDRESS"), "client_id": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_ID"), "client_secret": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_SECRET"), + "shard_id": shardID.String(), "ssl.verification_mode": "none", } @@ -23,7 +31,6 @@ func GetConfigFromEnv(t *testing.T) map[string]interface{} { optionalConfig(config, "uaa_address", "CLOUDFOUNDRY_UAA_ADDRESS") optionalConfig(config, "rlp_address", "CLOUDFOUNDRY_RLP_ADDRESS") optionalConfig(config, "doppler_address", "CLOUDFOUNDRY_DOPPLER_ADDRESS") - optionalConfig(config, "shard_id", "CLOUDFOUNDRY_SHARD_ID") if t.Failed() { t.FailNow() diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go index c178ea04325..a6b8bd16566 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go @@ -5,6 +5,7 @@ package add_cloudfoundry_metadata import ( + "github.com/gofrs/uuid" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/beat" @@ -32,6 +33,11 @@ const selector = "add_cloudfoundry_metadata" // New constructs a new add_cloudfoundry_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { var config cloudfoundry.Config + + // ShardID is required in cloudfoundry config to consume from the firehose, + // but not for metadata requests, randomly generate one and use it. + config.ShardID = uuid.Must(uuid.NewV4()).String() + if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b7b62643353..41552410a38 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -398,6 +398,7 @@ metricbeat.modules: rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 #----------------------------- CockroachDB Module ----------------------------- diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml index be15db23b65..6299cfc8116 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml @@ -10,4 +10,5 @@ rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml index a2803b06d40..5ea86f3e8de 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml @@ -7,3 +7,4 @@ api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc index 4d908802358..752b2aaea0c 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc @@ -107,7 +107,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "". === `shard_id` Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events -from the RLP Gateway. Default: "(generated UUID)". +from the RLP Gateway. [float] ==== `version` diff --git a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled index ae540f20cfc..22a600e51d8 100644 --- a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled +++ b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled @@ -10,3 +10,4 @@ api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat