Merge remote-tracking branch 'origin/master' into postgresql-query-da…

…shboard

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update to ECS 1.7.0. {pull}22571[22571]
 - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 - Use alias to report container image in k8s metadata. {pull}24380[24380]
+- Set `cleanup_timeout` to zero by default in docker and kubernetes autodiscover in all beats except Filebeat where it is kept to 60 seconds. {pull}24681[24681]
 
 *Auditbeat*
 
@@ -372,15 +373,17 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
 - Fix Cisco ASA parser for message 722051. {pull}24410[24410]
 - Fix `google_workspace` pagination. {pull}24668[24668]
+- Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697]
+- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
+- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
+- Fix date parsing in GSuite/login fileset. {issue}24694[24694]
+- Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.  Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766]
 
 *Heartbeat*
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
 
-*Heartbeat*
-
-
 *Journalbeat*
 
 
@@ -491,6 +494,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218]
 - Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468]
 - Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502]
+- Accept text/plain type by default for prometheus client scraping. {pull}24622[24622]
 
 *Packetbeat*
 

Jenkinsfile

@@ -9,6 +9,7 @@ pipeline {
     AWS_REGION = "${params.awsRegion}"
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKERHUB_SECRET = 'secret/observability-team/ci/elastic-observability-dockerhub'
     DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
@@ -587,6 +588,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
   ]) {
     if(isDockerInstalled()) {
       dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io')
     }
     dir("${env.BASE_DIR}") {
       installTools(args)

filebeat/autodiscover/defaults.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package autodiscover
+
+import (
+	"time"
+
+	"github.com/elastic/beats/v7/libbeat/autodiscover/providers/docker"
+	"github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes"
+)
+
+func init() {
+	docker.DefaultCleanupTimeout = 60 * time.Second
+	kubernetes.DefaultCleanupTimeout = 60 * time.Second
+}

filebeat/autodiscover/imports.go

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package autodiscover
+
+import (
+	_ "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints"
+)

filebeat/beater/filebeat.go

@@ -54,8 +54,8 @@ import (
 	_ "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata"
 	_ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields"
 
-	// include all filebeat specific builders
-	_ "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints"
+	// include all filebeat specific autodiscover features
+	_ "github.com/elastic/beats/v7/filebeat/autodiscover"
 )
 
 const pipelinesWarning = "Filebeat is unable to load the Ingest Node pipelines for the configured" +

filebeat/docs/fields.asciidoc

@@ -21773,6 +21773,26 @@ type: keyword
 The total count of burst rate hits since the object was created or cleared
 
 
+type: keyword
+
+--
+
+*`cisco.asa.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.asa.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --
@@ -21991,6 +22011,26 @@ type: keyword
 The assigned DAP records
 
 
+type: keyword
+
+--
+
+*`cisco.ftd.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.ftd.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --

libbeat/autodiscover/providers/docker/config.go

@@ -40,12 +40,15 @@ type Config struct {
 	CleanupTimeout time.Duration           `config:"cleanup_timeout" validate:"positive"`
 }
 
+// Public variable, so specific beats (as Filebeat) can set a different cleanup timeout if they need it.
+var DefaultCleanupTimeout time.Duration = 0
+
 func defaultConfig() *Config {
 	return &Config{
 		Host:           "unix:///var/run/docker.sock",
 		Prefix:         "co.elastic",
 		Dedot:          true,
-		CleanupTimeout: 60 * time.Second,
+		CleanupTimeout: DefaultCleanupTimeout,
 	}
 }
 

libbeat/autodiscover/providers/kubernetes/config.go

@@ -57,11 +57,14 @@ type Config struct {
 	AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
 }
 
+// Public variable, so specific beats (as Filebeat) can set a different cleanup timeout if they need it.
+var DefaultCleanupTimeout time.Duration = 0
+
 func defaultConfig() *Config {
 	return &Config{
 		SyncPeriod:     10 * time.Minute,
 		Resource:       "pod",
-		CleanupTimeout: 60 * time.Second,
+		CleanupTimeout: DefaultCleanupTimeout,
 		Prefix:         "co.elastic",
 		Unique:         false,
 	}

libbeat/docs/shared-autodiscover.asciidoc

@@ -117,7 +117,13 @@ It has the following settings:
 `ssl`:: (Optional) SSL configuration to use when connecting to the Docker
 socket.
 `cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the
-running configuration for a container, 60s by default.
+running configuration for a container,
+ifeval::["{beatname_lc}"=="filebeat"]
+ 60s by default.
+endif::[]
+ifeval::["{beatname_lc}"!="filebeat"]
+ disabled by default.
+endif::[]
 `labels.dedot`:: (Optional) Default to be false. If set to true, replace dots in
  labels with `_`.
 
@@ -218,7 +224,13 @@ The `kubernetes` autodiscover provider has the following configuration settings:
   namespaces. It is unset by default. The namespace configuration only applies to
   kubernetes resources that are namespace scoped.
 `cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the
-running configuration for a container, 60s by default.
+running configuration for a container,
+ifeval::["{beatname_lc}"=="filebeat"]
+ 60s by default.
+endif::[]
+ifeval::["{beatname_lc}"!="filebeat"]
+ disabled by default.
+endif::[]
 `kube_config`:: (Optional) Use given config file as configuration for Kubernetes
   client. If kube_config is not set, KUBECONFIG environment variable will be
   checked and if not present it will fall back to InCluster.

libbeat/tests/compose/compose.go

@@ -112,7 +112,7 @@ func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo {
 	// Get host information
 	host, err := compose.HostInformation(service)
 	if err != nil {
-		t.Fatalf("getting host for %s", service)
+		t.Fatalf("getting host for %s: %v", service, err)
 	}
 
 	return host

metricbeat/helper/prometheus/prometheus.go

@@ -35,7 +35,7 @@ import (
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
-const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`
+const acceptHeader = `text/plain;version=0.0.4;q=0.5,*/*;q=0.1`
 
 // Prometheus helper retrieves prometheus formatted metrics
 type Prometheus interface {

x-pack/elastic-agent/pkg/agent/application/config.go

@@ -5,63 +5,10 @@
 package application
 
 import (
-	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 )
 
 type localConfig struct {
 	Fleet    *configuration.FleetAgentConfig `config:"fleet"`
 	Settings *configuration.SettingsConfig   `config:"agent" yaml:"agent"`
 }
-
-func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*configuration.FleetAgentConfig, error) {
-	cfg := configuration.DefaultFleetAgentConfig()
-	cfg.Enabled = true
-	cfg.AccessAPIKey = accessAPIKey
-	cfg.Kibana = kbn
-
-	if err := cfg.Valid(); err != nil {
-		return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig)
-	}
-	return cfg, nil
-}
-
-func createFleetServerBootstrapConfig(connStr string, policyID string, host string, port uint16, cert string, key string, esCA string) (*configuration.FleetAgentConfig, error) {
-	es, err := configuration.ElasticsearchFromConnStr(connStr)
-	if err != nil {
-		return nil, err
-	}
-	if esCA != "" {
-		es.TLS = &tlscommon.Config{
-			CAs: []string{esCA},
-		}
-	}
-	cfg := configuration.DefaultFleetAgentConfig()
-	cfg.Enabled = true
-	cfg.Server = &configuration.FleetServerConfig{
-		Bootstrap: true,
-		Output: configuration.FleetServerOutputConfig{
-			Elasticsearch: es,
-		},
-		Host: host,
-		Port: port,
-	}
-	if policyID != "" {
-		cfg.Server.Policy = &configuration.FleetServerPolicyConfig{ID: policyID}
-	}
-	if cert != "" || key != "" {
-		cfg.Server.TLS = &tlscommon.Config{
-			Certificate: tlscommon.CertificateConfig{
-				Certificate: cert,
-				Key:         key,
-			},
-		}
-	}
-
-	if err := cfg.Valid(); err != nil {
-		return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig)
-	}
-	return cfg, nil
-}

x-pack/elastic-agent/pkg/agent/application/config_test.go

@@ -5,12 +5,9 @@
 package application
 
 import (
-	"io/ioutil"
 	"testing"
 	"time"
 
-	"gopkg.in/yaml.v2"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -77,9 +74,3 @@ func mustWithConfigMode(standalone bool) *config.Config {
 		},
 	)
 }
-
-func dumpToYAML(t *testing.T, out string, in interface{}) {
-	b, err := yaml.Marshal(in)
-	require.NoError(t, err)
-	ioutil.WriteFile(out, b, 0600)
-}

x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go

@@ -16,8 +16,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-const semiLongString = ""
-
 func TestStreamCheck(t *testing.T) {
 	type testCase struct {
 		name      string