Merge branch 'main' into url-extension-value-fix

.github/workflows/macos-xpack-heartbeat.yml

@@ -17,15 +17,17 @@ jobs:
   macos:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-go@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
       with:
         go-version-file: .go-version
     - uses: actions/setup-python@v4
       with:
         python-version: '3.9'
     - name: Install dependencies
-      run:  go install github.com/magefile/mage
+      run:  |
+        go install github.com/magefile/mage
+        cd ${{ env.BEAT_MODULE }} && npm install -g @elastic/synthetics
     - name: Run build
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test

.go-version

@@ -1 +1 @@
-1.21.7
+1.21.8

.golangci.yml

@@ -114,7 +114,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.7"
+    go: "1.21.8"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -132,19 +132,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.7"
+    go: "1.21.8"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.7"
+    go: "1.21.8"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.7"
+    go: "1.21.8"
 
   gosec:
     excludes:

CHANGELOG.next.asciidoc

@@ -10,7 +10,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
-- Upgrade to Go 1.21.7. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37913[37913]
+- Upgrade to Go 1.21.8. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}38209[38209]
 - add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack
 platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you
 know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`,
@@ -96,7 +96,10 @@ fields added to events containing the Beats version. {pull}37553[37553]
 - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799]
 - [threatintel] MISP pagination fixes {pull}37898[37898]
 - Fix file handle leak when handling errors in filestream {pull}37973[37973]
+- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094]
 - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116]
+- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125]
+- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181]
 - Fix the incorrect values generated by url_parts processor. {pull}38216[38216]
 
 *Heartbeat*
@@ -107,6 +110,7 @@ fields added to events containing the Beats version. {pull}37553[37553]
 
 *Metricbeat*
 
+- Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720]
 
 *Osquerybeat*
 

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.7
+FROM golang:1.21.8
 
 RUN \
     apt-get update \

catalog-info.yaml

@@ -637,7 +637,7 @@ spec:
     spec:
       # branch_configuration: "7.17"         #TODO: uncomment after tests
       pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml"
-      maximum_timeout_in_minutes: 240
+      maximum_timeout_in_minutes: 480
       provider_settings:
         trigger_mode: none # don't trigger jobs from github activity
         build_pull_request_forks: false

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.7
+FROM golang:1.21.8
 
 RUN \
     apt-get update \

libbeat/docs/version.asciidoc

@@ -1,6 +1,6 @@
 :stack-version: 8.13.0
 :doc-branch: main
-:go-version: 1.21.7
+:go-version: 1.21.8
 :release-state: unreleased
 :python: 3.7
 :docker: 1.12

libbeat/publisher/queue/memqueue/produce.go

@@ -36,9 +36,10 @@ type ackProducer struct {
 }
 
 type openState struct {
-	log    *logp.Logger
-	done   chan struct{}
-	events chan pushRequest
+	log       *logp.Logger
+	done      chan struct{}
+	queueDone <-chan struct{}
+	events    chan pushRequest
 }
 
 // producerID stores the order of events within a single producer, so multiple
@@ -58,9 +59,10 @@ type ackHandler func(count int)
 
 func newProducer(b *broker, cb ackHandler, dropCB func(interface{}), dropOnCancel bool) queue.Producer {
 	openState := openState{
-		log:    b.logger,
-		done:   make(chan struct{}),
-		events: b.pushChan,
+		log:       b.logger,
+		done:      make(chan struct{}),
+		queueDone: b.ctx.Done(),
+		events:    b.pushChan,
 	}
 
 	if cb != nil {
@@ -143,27 +145,40 @@ func (st *openState) Close() {
 func (st *openState) publish(req pushRequest) (queue.EntryID, bool) {
 	select {
 	case st.events <- req:
-		// If the output is blocked and the queue is full, `req` is written
-		// to `st.events`, however the queue never writes back to `req.resp`,
-		// which effectively blocks for ever. So we also need to select on the
-		// done channel to ensure we don't miss the shutdown signal.
+		// The events channel is buffered, which means we may successfully
+		// write to it even if the queue is shutting down. To avoid blocking
+		// forever during shutdown, we also have to wait on the queue's
+		// shutdown channel.
 		select {
 		case resp := <-req.resp:
 			return resp, true
-		case <-st.done:
+		case <-st.queueDone:
 			st.events = nil
 			return 0, false
 		}
 	case <-st.done:
 		st.events = nil
 		return 0, false
+	case <-st.queueDone:
+		st.events = nil
+		return 0, false
 	}
 }
 
 func (st *openState) tryPublish(req pushRequest) (queue.EntryID, bool) {
 	select {
 	case st.events <- req:
-		return <-req.resp, true
+		// The events channel is buffered, which means we may successfully
+		// write to it even if the queue is shutting down. To avoid blocking
+		// forever during shutdown, we also have to wait on the queue's
+		// shutdown channel.
+		select {
+		case resp := <-req.resp:
+			return resp, true
+		case <-st.queueDone:
+			st.events = nil
+			return 0, false
+		}
 	case <-st.done:
 		st.events = nil
 		return 0, false