diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 87f1925a372..849720d1e2d 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Filebeat Check/Update" + key: "filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "filebeat-check-update" + - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index fd485279858..7ebe5d76f98 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -16,6 +16,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "libbeat Check/Update" + key: "libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "libbeat-check-update" + - group: "Mandatory Tests" key: "mandatory-tests" steps: diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index c9a5c9f06f2..40e5f0f4b6f 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -25,6 +25,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Packetbeat Check/Update" + key: "packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "packetbeat-check-update" + - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" steps: diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 55affae4128..b2cbb06e70f 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -21,7 +21,7 @@ "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 1aa1c51ca5f..ce45e798239 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -26,6 +26,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/filebeat Check/Update" + key: "x-pack-filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack-filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-filebeat-check-update" + + - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 33356c78c46..93f57713ee3 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -24,6 +24,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/libbeat Check/Update" + key: "x-pack-libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-libbeat-check-update" + - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 542fee787c1..71027426992 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/packetbeat Check/Update" + key: "x-pack-packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-packetbeat-check-update" + - group: "x-pack/packetbeat Mandatory Tests" key: "x-pack-packetbeat-mandatory-tests" diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml deleted file mode 100644 index 930a04ec5e5..00000000000 --- a/.github/workflows/check-filebeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-filebeat - -on: - pull_request: - paths: - - '.github/workflows/check-filebeat.yml' - - 'filebeat/**' - - 'x-pack/filebeat/**' - -env: - BEAT_MODULE: 'filebeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libsystemd-dev - run: sudo apt-get install -y libsystemd-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml deleted file mode 100644 index 27e03701b85..00000000000 --- a/.github/workflows/check-libbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-libbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'libbeat/**' - - 'x-pack/libbeat/**' - -env: - BEAT_MODULE: 'libbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml deleted file mode 100644 index ba05b6c0160..00000000000 --- a/.github/workflows/check-packetbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-packetbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'packetbeat/**' - - 'x-pack/packetbeat/**' - -env: - BEAT_MODULE: 'packetbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e1214aa0e27..8e5cd3497e1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Removed deprecated Sophos UTM from Beats. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead. {pull}38037[38037] - Introduce input/netmetrics and refactor netflow input metrics {pull}38055[38055] - Update Salesforce module to use new Salesforce input. {pull}37509[37509] +- Tag events that come from a filestream in "take over" mode. {pull}39828[39828] - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893] *Heartbeat* diff --git a/filebeat/docs/howto/migrate-to-filestream.asciidoc b/filebeat/docs/howto/migrate-to-filestream.asciidoc index a57105adb3e..55c05a5b46d 100644 --- a/filebeat/docs/howto/migrate-to-filestream.asciidoc +++ b/filebeat/docs/howto/migrate-to-filestream.asciidoc @@ -234,6 +234,13 @@ The example configuration shown earlier needs to be adjusted as well: |backoff.max |=== +=== Step 4 + +The events produced by `filestream` input with `take_over: true` contain a `take_over` tag. +You can filter on this tag in Kibana and see the events which came from a filestream in the "take_over" mode. + +Once you start receiving events with this tag, you can remove `take_over: true` and restart the fileinput again. + === If something went wrong If for whatever reason you'd like to revert the configuration after running the migrated configuration @@ -247,3 +254,8 @@ and return to old `log` inputs the files that were taken by `filestream` inputs, 6. Run Filebeat with the old configuration (no `filestream` inputs with `take_over: true`). NOTE: Reverting to backups might cause some events to repeat, depends on the amount of time the new configuration was running. + +=== Debugging on Kibana + +Events produced by `filestream` with `take_over: true` contains `take_over` tag. +You can filter on this tag in Kibana and see the events which came from a filestream in the "take over" mode. \ No newline at end of file diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 0136b062b48..7da25654a25 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -41,6 +41,7 @@ import ( "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) const pluginName = "filestream" @@ -61,6 +62,7 @@ type filestream struct { encodingFactory encoding.EncodingFactory closerConfig closerConfig parsers parser.Config + takeOver bool } // Plugin creates a new filestream input plugin for creating a stateful input. @@ -101,6 +103,7 @@ func configure(cfg *conf.C) (loginp.Prospector, loginp.Harvester, error) { encodingFactory: encodingFactory, closerConfig: config.Close, parsers: config.Reader.Parsers, + takeOver: config.TakeOver, } return prospector, filestream, nil @@ -378,6 +381,11 @@ func (inp *filestream) readFromSource( metrics.BytesProcessed.Add(uint64(message.Bytes)) + // add "take_over" tag if `take_over` is set to true + if inp.takeOver { + _ = mapstr.AddTags(message.Fields, []string{"take_over"}) + } + if err := p.Publish(message.ToEvent(), s); err != nil { metrics.ProcessingErrors.Inc() return err diff --git a/filebeat/input/filestream/input_test.go b/filebeat/input/filestream/input_test.go index a1d9729c5aa..3dfe176ac01 100644 --- a/filebeat/input/filestream/input_test.go +++ b/filebeat/input/filestream/input_test.go @@ -35,6 +35,7 @@ import ( "github.com/elastic/beats/v7/libbeat/statestore/storetest" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) func BenchmarkFilestream(b *testing.B) { @@ -115,6 +116,48 @@ paths: }) } +func TestTakeOverTags(t *testing.T) { + testCases := []struct { + name string + takeOver bool + testFunc func(t *testing.T, event beat.Event) + }{ + { + name: "test-take_over-true", + takeOver: true, + testFunc: func(t *testing.T, event beat.Event) { + tags, err := event.GetValue("tags") + require.NoError(t, err) + require.Contains(t, tags, "take_over") + }, + }, + { + name: "test-take_over-false", + takeOver: false, + testFunc: func(t *testing.T, event beat.Event) { + _, err := event.GetValue("tags") + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + filename := generateFile(t, t.TempDir(), 5) + cfg := fmt.Sprintf(` +type: filestream +prospector.scanner.check_interval: 1s +take_over: %t +paths: + - %s`, testCase.takeOver, filename) + runner := createFilestreamTestRunner(context.Background(), t, testCase.name, cfg, 5, true) + events := runner(t) + for _, event := range events { + testCase.testFunc(t, event) + } + }) + } +} + // runFilestreamBenchmark runs the entire filestream input with the in-memory registry and the test pipeline. // `testID` must be unique for each test run // `cfg` must be a valid YAML string containing valid filestream configuration diff --git a/libbeat/docs/shared/opendashboards.asciidoc b/libbeat/docs/shared/opendashboards.asciidoc index 7e73fbc8f4b..1b942504998 100644 --- a/libbeat/docs/shared/opendashboards.asciidoc +++ b/libbeat/docs/shared/opendashboards.asciidoc @@ -13,7 +13,7 @@ include::{libbeat-dir}/tab-widgets/open-kibana-widget.asciidoc[] -- . In the side navigation, click *Discover*. To see {beatname_uc} data, make -sure the predefined +{beatname_lc}-*+ index pattern is selected. +sure the predefined +{beatname_lc}-*+ data view is selected. + -- TIP: If you don’t see data in {kib}, try changing the time filter to a larger diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 5b6dd6ffd00..a5fb79fb02c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-d2070ed2-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.15.0-d2070ed2-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.15.0-d2070ed2-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing"