Merge branch 'main' into refactor/drop-uber-atomic

.buildkite/filebeat/filebeat-pipeline.yml

@@ -27,6 +27,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Filebeat Check/Update"
+    key: "filebeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C filebeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "filebeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "filebeat-check-update"
+
   - group: "Filebeat Mandatory Tests"
     key: "filebeat-mandatory-tests"
     steps:

.buildkite/libbeat/pipeline.libbeat.yml

@@ -16,6 +16,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "libbeat Check/Update"
+    key: "libbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C libbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "libbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "libbeat-check-update"
+
   - group: "Mandatory Tests"
     key: "mandatory-tests"
     steps:

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -25,6 +25,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Packetbeat Check/Update"
+    key: "packetbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C packetbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "packetbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "packetbeat-check-update"
+
   - group: "packetbeat Mandatory Tests"
     key: "packetbeat-mandatory-tests"
     steps:

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
             "pipelineSlug": "beats",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"],
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
@@ -21,7 +21,7 @@
             "pipelineSlug": "beats-xpack-elastic-agent",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"],
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -26,6 +26,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "x-pack/filebeat Check/Update"
+    key: "x-pack-filebeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C x-pack/filebeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "x-pack-filebeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-filebeat-check-update"
+
+
   - group: "x-pack/filebeat Mandatory Tests"
     key: "x-pack-filebeat-mandatory-tests"
     steps:

.buildkite/x-pack/pipeline.xpack.libbeat.yml

@@ -24,6 +24,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "x-pack/libbeat Check/Update"
+    key: "x-pack-libbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C x-pack/libbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "x-pack/libbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-libbeat-check-update"
+
   - group: "x-pack/libbeat Mandatory Tests"
     key: "x-pack-libbeat-mandatory-tests"
 

.buildkite/x-pack/pipeline.xpack.packetbeat.yml

@@ -27,6 +27,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "x-pack/packetbeat Check/Update"
+    key: "x-pack-packetbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C x-pack/packetbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "x-pack/packetbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-packetbeat-check-update"
+
   - group: "x-pack/packetbeat Mandatory Tests"
     key: "x-pack-packetbeat-mandatory-tests"
 

CHANGELOG.next.asciidoc

@@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Removed deprecated Sophos UTM from Beats. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead. {pull}38037[38037]
 - Introduce input/netmetrics and refactor netflow input metrics {pull}38055[38055]
 - Update Salesforce module to use new Salesforce input. {pull}37509[37509]
+- Tag events that come from a filestream in "take over" mode. {pull}39828[39828]
 - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893]
 
 *Heartbeat*

filebeat/docs/howto/migrate-to-filestream.asciidoc

@@ -234,6 +234,13 @@ The example configuration shown earlier needs to be adjusted as well:
 |backoff.max
 |===
 
+=== Step 4
+
+The events produced by `filestream` input with `take_over: true` contain a `take_over` tag.
+You can filter on this tag in Kibana and see the events which came from a filestream in the "take_over" mode. 
+
+Once you start receiving events with this tag, you can remove `take_over: true` and restart the fileinput again.
+
 === If something went wrong
 
 If for whatever reason you'd like to revert the configuration after running the migrated configuration
@@ -247,3 +254,8 @@ and return to old `log` inputs the files that were taken by `filestream` inputs,
 6. Run Filebeat with the old configuration (no `filestream` inputs with `take_over: true`).
 
 NOTE: Reverting to backups might cause some events to repeat, depends on the amount of time the new configuration was running.
+
+=== Debugging on Kibana
+
+Events produced by `filestream` with `take_over: true` contains `take_over` tag.
+You can filter on this tag in Kibana and see the events which came from a filestream in the "take over" mode. 

filebeat/input/filestream/input.go

@@ -41,6 +41,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/reader/readfile/encoding"
 	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
 const pluginName = "filestream"
@@ -61,6 +62,7 @@ type filestream struct {
 	encodingFactory encoding.EncodingFactory
 	closerConfig    closerConfig
 	parsers         parser.Config
+	takeOver        bool
 }
 
 // Plugin creates a new filestream input plugin for creating a stateful input.
@@ -101,6 +103,7 @@ func configure(cfg *conf.C) (loginp.Prospector, loginp.Harvester, error) {
 		encodingFactory: encodingFactory,
 		closerConfig:    config.Close,
 		parsers:         config.Reader.Parsers,
+		takeOver:        config.TakeOver,
 	}
 
 	return prospector, filestream, nil
@@ -378,6 +381,11 @@ func (inp *filestream) readFromSource(
 
 		metrics.BytesProcessed.Add(uint64(message.Bytes))
 
+		// add "take_over" tag if `take_over` is set to true
+		if inp.takeOver {
+			_ = mapstr.AddTags(message.Fields, []string{"take_over"})
+		}
+
 		if err := p.Publish(message.ToEvent(), s); err != nil {
 			metrics.ProcessingErrors.Inc()
 			return err