[Packetbeat] Refactor packetbeat for use with Elastic Agent (elastic#…

…22134) * Refactor packetbeat to support agent-based configuration * Add documentation changes and a Changelog entry * Update reference template * Fix funny merge * Incorporate feedback * use streams instead of inputs * support multiple sniffers * fix shutdown_timeout behavior (cherry picked from commit 8c05a41)
andrewstucki · Nov 11, 2020 · fd0e75c · fd0e75c
1 parent 41a62c3
commit fd0e75c
Show file tree

Hide file tree

Showing 59 changed files with 3,045 additions and 309 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -588,6 +588,14 @@ same journal. {pull}18467[18467]
 
 `host` metadata fields when processing network data from network tap or mirror
 port. {pull}19209[19209]
+- Add an example to packetbeat.yml of using the `forwarded` tag to disable
+  `host` metadata fields when processing network data from network tap or mirror
+  port. {pull}19209[19209]
+- Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
+- Add 100-continue support {issue}15830[15830] {pull}19349[19349]
+- Add initial SIP protocol support {pull}21221[21221]
+- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134]
+- Change build process for x-pack distribution {pull}21979[21979]
 
 
 *Functionbeat*

diff --git a/libbeat/cfgfile/list.go b/libbeat/cfgfile/list.go
@@ -157,7 +157,9 @@ func (r *RunnerList) Has(hash uint64) bool {
 // HashConfig hashes a given common.Config
 func HashConfig(c *common.Config) (uint64, error) {
 	var config map[string]interface{}
-	c.Unpack(&config)
+	if err := c.Unpack(&config); err != nil {
+		return 0, err
+	}
 	return hashstructure.Hash(config, nil)
 }
 

diff --git a/packetbeat/_meta/config/beat.reference.yml.tmpl b/packetbeat/_meta/config/beat.reference.yml.tmpl
@@ -63,6 +63,9 @@ packetbeat.flows:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where flow events are indexed.
+  #index: my-custom-flow-index
+
 {{header "Transaction protocols"}}
 
 packetbeat.protocols:
@@ -73,6 +76,9 @@ packetbeat.protocols:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-icmp-index
+
 - type: amqp
   # Enable AMQP monitoring. Default: true
   #enabled: true
@@ -113,6 +119,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-amqp-index
+
 - type: cassandra
   #Cassandra port for traffic monitoring.
   ports: [9042]
@@ -143,6 +152,9 @@ packetbeat.protocols:
   # This option indicates which Operator/Operators will be ignored.
   #ignored_ops: ["SUPPORTED","OPTIONS"]
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-cassandra-index
+
 - type: dhcpv4
   # Configure the DHCP for IPv4 ports.
   ports: [67, 68]
@@ -183,6 +195,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-dhcpv4-index
+
 - type: http
   # Enable HTTP monitoring. Default: true
   #enabled: true
@@ -257,6 +272,9 @@ packetbeat.protocols:
   # be trimmed to this size. Default is 10 MB.
   #max_message_size: 10485760
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-http-index
+
 - type: memcache
   # Enable memcache monitoring. Default: true
   #enabled: true
@@ -309,6 +327,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-memcache-index
+
 - type: mysql
   # Enable mysql monitoring. Default: true
   #enabled: true
@@ -332,6 +353,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-mysql-index
+
 - type: pgsql
   # Enable pgsql monitoring. Default: true
   #enabled: true
@@ -355,6 +379,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-pgsql-index
+
 - type: redis
   # Enable redis monitoring. Default: true
   #enabled: true
@@ -387,6 +414,9 @@ packetbeat.protocols:
   # large enough to allow for pipelining.
   #queue_max_messages: 20000
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-redis-index
+
 - type: thrift
   # Enable thrift monitoring. Default: true
   #enabled: true
@@ -445,6 +475,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-thrift-index
+
 - type: mongodb
   # Enable mongodb monitoring. Default: true
   #enabled: true
@@ -478,6 +511,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-mongodb-index
+
 - type: nfs
   # Enable NFS monitoring. Default: true
   #enabled: true
@@ -501,6 +537,9 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-nfs-index
+
 - type: tls
   # Enable TLS monitoring. Default: true
   #enabled: true
@@ -531,6 +570,9 @@ packetbeat.protocols:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-tls-index
+
 - type: sip
   # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports.
   ports: [5060]
@@ -544,6 +586,9 @@ packetbeat.protocols:
   # Preserve original contents in event.original
   keep_original: true
 
+  # Overrides where this protocol's events are indexed.
+  #index: my-custom-sip-index
+
 {{header "Monitored processes"}}
 
 # Packetbeat can enrich events with information about the process associated