From d51078113c28e7da5f29fe09a2a40b6a6f4a2fb6 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Tue, 8 Feb 2022 15:52:08 +0100 Subject: [PATCH 01/52] Removing GA workflow (#30263) Removing it as their is an issue while trying to run this github action on a fork PR, the secret token is not available to fork PR. --- .../workflows/elastic-agent-pull-requests.yml | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/elastic-agent-pull-requests.yml diff --git a/.github/workflows/elastic-agent-pull-requests.yml b/.github/workflows/elastic-agent-pull-requests.yml deleted file mode 100644 index 55c56614957a..000000000000 --- a/.github/workflows/elastic-agent-pull-requests.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Add to Elastic Agent Data Plane or Control Plane Board -on: - pull_request: - types: - - labeled -jobs: - add_to_data_plane-project: - runs-on: ubuntu-latest - if: | - github.event.label.name == 'Team:Elastic-Agent-Data-Plane' - steps: - - uses: octokit/graphql-action@v2.x - id: add_to_project - with: - query: | - mutation add_to_project($projectid:String!,$contentid:String!) { - updatePullRequest(input: {pullRequestId:$contentid, projectIds:$projectid}) { - clientMutationId - } - } - projectid: ${{ env.PROJECT_ID }} - contentid: ${{ github.event.pull_request.node_id }} - env: - PROJECT_ID: "PRO_kwDOAGc3Zs4AzG8z" - GITHUB_TOKEN: ${{ secrets.ELASTIC_AGENT_PROJECT_BOARD_TOKEN }} - add_to_control_plane-project: - runs-on: ubuntu-latest - if: | - github.event.label.name == 'Team:Elastic-Agent-Control-Plane' - steps: - - uses: octokit/graphql-action@v2.x - id: add_to_project - with: - query: | - mutation add_to_project($projectid:String!,$contentid:String!) { - updatePullRequest(input: {pullRequestId:$contentid, projectIds:$projectid}) { - clientMutationId - } - } - projectid: ${{ env.PROJECT_ID }} - contentid: ${{ github.event.pull_request.node_id }} - env: - PROJECT_ID: "PRO_kwDOAGc3Zs4AzG9E" - GITHUB_TOKEN: ${{ secrets.ELASTIC_AGENT_PROJECT_BOARD_TOKEN }} From b2c5516525f9c5329d149b68d0e01c66f91d36f2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 8 Feb 2022 18:48:59 +0000 Subject: [PATCH 02/52] mergify: auto-assign PR owner (#30282) --- .mergify.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 18ab7689c64f..4b1c6ea1eff9 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,6 +3,15 @@ queue_rules: conditions: - check-success=beats-ci/pr-merge pull_request_rules: + - name: self-assign PRs + conditions: + - -merged + - -closed + - "#assignee=0" + actions: + assign: + add_users: + - "{{ author }}" - name: forward-port patches to main branch conditions: - merged From 262bb683b704ca6137fd8c5fac3c47e929aaeeb2 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 8 Feb 2022 20:07:32 +0100 Subject: [PATCH 03/52] docs: add statement about config file encoding (#30182) Beats config files must be UTF-8, if they're not UTF-8 there will be problems with special characters. Closes 30053 --- libbeat/docs/config-file-format.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/docs/config-file-format.asciidoc b/libbeat/docs/config-file-format.asciidoc index c1b93772c883..32a6ff49913b 100644 --- a/libbeat/docs/config-file-format.asciidoc +++ b/libbeat/docs/config-file-format.asciidoc @@ -3,6 +3,7 @@ Beats config files are based on http://www.yaml.org[YAML], a file format that is easier to read and write than other common data formats like XML or JSON. +Config files must be encoded in UTF-8. In beats all YAML files start with a dictionary, an unordered collection of name/value pairs. In addition to dictionaries, YAML also supports lists, numbers, From cc634f1ecb3b17343e6a2235c707a6962ec65754 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 8 Feb 2022 20:08:11 +0100 Subject: [PATCH 04/52] filebeat/input/journald: fix field name translation (#30167) The field names from journald were not being translated to our format when sending the event to the output. This commit fixes it. Fixes 30031 --- filebeat/input/journald/conv.go | 58 ------------- filebeat/input/journald/input.go | 36 ++++++-- .../input/journald/input_filtering_test.go | 1 - filebeat/input/journald/input_test.go | 87 +++++++++++++++++++ 4 files changed, 115 insertions(+), 67 deletions(-) delete mode 100644 filebeat/input/journald/conv.go create mode 100644 filebeat/input/journald/input_test.go diff --git a/filebeat/input/journald/conv.go b/filebeat/input/journald/conv.go deleted file mode 100644 index a1bf52f30004..000000000000 --- a/filebeat/input/journald/conv.go +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux && cgo && withjournald -// +build linux,cgo,withjournald - -package journald - -import ( - "time" - - "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/logp" -) - -func eventFromFields( - log *logp.Logger, - timestamp uint64, - entryFields map[string]string, - saveRemoteHostname bool, -) beat.Event { - created := time.Now() - c := journalfield.NewConverter(log, nil) - fields := c.Convert(entryFields) - fields.Put("event.kind", "event") - - // if entry is coming from a remote journal, add_host_metadata overwrites the source hostname, so it - // has to be copied to a different field - if saveRemoteHostname { - remoteHostname, err := fields.GetValue("host.hostname") - if err == nil { - fields.Put("log.source.address", remoteHostname) - } - } - - fields.Put("event.created", created) - receivedByJournal := time.Unix(0, int64(timestamp)*1000) - - return beat.Event{ - Timestamp: receivedByJournal, - Fields: fields, - } -} diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index bf86aa59626a..b2dcdb1d36d7 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -145,7 +145,13 @@ func (inp *journald) Run( log.Error("Continue from current position. Seek failed with: %v", err) } - parser := inp.Parsers.Create(&readerAdapter{r: reader, canceler: ctx.Cancelation}) + parser := inp.Parsers.Create( + &readerAdapter{ + r: reader, + converter: journalfield.NewConverter(ctx.Logger, nil), + canceler: ctx.Cancelation, + saveRemoteHostname: inp.SaveRemoteHostname, + }) for { entry, err := parser.Next() @@ -231,11 +237,15 @@ func seekBy(log *logp.Logger, cp checkpoint, seek, defaultSeek journalread.SeekM return mode, cp.Position } -// readerAdapter is an adapter so journalread.Reader can -// behave like reader.Reader +// readerAdapter wraps journalread.Reader and adds two functionalities: +// - Allows it to behave like a reader.Reader +// - Translates the fields names from the journald format to something +// more human friendly type readerAdapter struct { - r *journalread.Reader - canceler input.Canceler + r *journalread.Reader + canceler input.Canceler + converter *journalfield.Converter + saveRemoteHostname bool } func (r *readerAdapter) Close() error { @@ -248,12 +258,22 @@ func (r *readerAdapter) Next() (reader.Message, error) { return reader.Message{}, err } + created := time.Now() + content := []byte(data.Fields["MESSAGE"]) delete(data.Fields, "MESSAGE") - fields := make(map[string]interface{}, len(data.Fields)) - for k, v := range data.Fields { - fields[k] = v + fields := r.converter.Convert(data.Fields) + fields.Put("event.kind", "event") + fields.Put("event.created", created) + + // if entry is coming from a remote journal, add_host_metadata overwrites + // the source hostname, so it has to be copied to a different field + if r.saveRemoteHostname { + remoteHostname, err := fields.GetValue("host.hostname") + if err == nil { + fields.Put("log.source.address", remoteHostname) + } } m := reader.Message{ diff --git a/filebeat/input/journald/input_filtering_test.go b/filebeat/input/journald/input_filtering_test.go index 75718b3e5861..f1be6c8b37e2 100644 --- a/filebeat/input/journald/input_filtering_test.go +++ b/filebeat/input/journald/input_filtering_test.go @@ -215,5 +215,4 @@ func TestInputIncludeMatches(t *testing.T) { } }) } - } diff --git a/filebeat/input/journald/input_test.go b/filebeat/input/journald/input_test.go new file mode 100644 index 000000000000..55d2e0112f96 --- /dev/null +++ b/filebeat/input/journald/input_test.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux && cgo && withjournald +// +build linux,cgo,withjournald + +package journald + +import ( + "context" + "fmt" + "path" + "testing" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestInputFieldsTranslation(t *testing.T) { + // A few random keys to verify + var keysToCheck = map[string]string{ + "systemd.user_unit": "log-service.service", + "process.pid": "2084785", + "systemd.transport": "stdout", + "host.hostname": "x-wing", + } + + testCases := map[string]struct { + saveRemoteHostname bool + }{ + "Save hostname enabled": {saveRemoteHostname: true}, + "Save hostname disabled": {saveRemoteHostname: true}, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + env := newInputTestingEnvironment(t) + + inp := env.mustCreateInput(common.MapStr{ + "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "include_matches.match": []string{"_SYSTEMD_USER_UNIT=log-service.service"}, + "save_remote_hostname": tc.saveRemoteHostname, + }) + + ctx, cancelInput := context.WithCancel(context.Background()) + env.startInput(ctx, inp) + env.waitUntilEventCount(6) + + for eventIdx, event := range env.pipeline.clients[0].GetEvents() { + for k, v := range keysToCheck { + got, err := event.Fields.GetValue(k) + if err == nil { + if got, want := fmt.Sprint(got), v; got != want { + t.Errorf("expecting key %q to have value '%#v', but got '%#v' instead", k, want, got) + } + } else { + t.Errorf("key %q not found on event %d", k, eventIdx) + } + } + if tc.saveRemoteHostname { + v, err := event.Fields.GetValue("log.source.address") + if err != nil { + t.Errorf("key 'log.source.address' not found on evet %d", eventIdx) + } + + if got, want := fmt.Sprint(v), "x-wing"; got != want { + t.Errorf("expecting key 'log.source.address' to have value '%#v', but got '%#v' instead", want, got) + } + } + } + cancelInput() + }) + } +} From adc386699b0aa2c056bf91c8cdeef75f69ddc0ec Mon Sep 17 00:00:00 2001 From: garethhumphriesgkc <62865047+garethhumphriesgkc@users.noreply.github.com> Date: Wed, 9 Feb 2022 14:58:15 +1300 Subject: [PATCH 05/52] Update doc with use of $ as escape char (#30041) * Update doc with use of $ as escape char Update doc to reflect that $ can used as a general escape char, not just to escape `$`. Co-authored-by: DeDe Morton --- libbeat/docs/shared-env-vars.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/docs/shared-env-vars.asciidoc b/libbeat/docs/shared-env-vars.asciidoc index 496f6aa480c7..e0285f8b9752 100644 --- a/libbeat/docs/shared-env-vars.asciidoc +++ b/libbeat/docs/shared-env-vars.asciidoc @@ -45,8 +45,7 @@ To specify custom error text, use: Where `error_text` is custom text that will be prepended to the error message if the environment variable cannot be expanded. -If you need to use a literal `${` in your configuration file then you can write -`$${` to escape the expansion. +If you need to use a special character in your configuration file, use `$` to escape the expansion. For example, you can escape `${` or `}` with `$${` or `$}`. After changing the value of an environment variable, you need to restart {beatname_uc} to pick up the new value. From 0bb2cfe2c939b26a9cb187028d7556071db7eb5d Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Tue, 8 Feb 2022 19:02:03 -0800 Subject: [PATCH 06/52] Update release docs for 8.0.0-rc1 (#30122) --- libbeat/docs/release-notes/7.0.0.asciidoc | 2 +- .../breaking/breaking-7.0.asciidoc | 70 --- .../breaking/breaking-7.1.asciidoc | 14 - .../breaking/breaking-7.10.asciidoc | 17 - .../breaking/breaking-7.11.asciidoc | 29 - .../breaking/breaking-7.12.asciidoc | 18 - .../breaking/breaking-7.13.asciidoc | 28 - .../breaking/breaking-7.14.asciidoc | 24 - .../breaking/breaking-7.15.asciidoc | 81 --- .../breaking/breaking-7.16.asciidoc | 42 -- .../breaking/breaking-7.2.asciidoc | 15 - .../breaking/breaking-7.3.asciidoc | 15 - .../breaking/breaking-7.4.asciidoc | 41 -- .../breaking/breaking-7.5.asciidoc | 15 - .../breaking/breaking-7.6.asciidoc | 42 -- .../breaking/breaking-7.7.asciidoc | 52 -- .../breaking/breaking-7.8.asciidoc | 31 - .../breaking/breaking-7.9.asciidoc | 23 - .../breaking/breaking-8.0.asciidoc | 7 +- .../release-notes/breaking/breaking.asciidoc | 81 +-- .../breaking/field-name-changes.asciidoc | 555 ------------------ libbeat/docs/upgrading.asciidoc | 5 + 22 files changed, 14 insertions(+), 1193 deletions(-) delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.0.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.1.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.10.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.11.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.12.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.13.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.14.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.15.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.16.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.2.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.3.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.4.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.5.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.7.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.8.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/breaking-7.9.asciidoc delete mode 100644 libbeat/docs/release-notes/breaking/field-name-changes.asciidoc diff --git a/libbeat/docs/release-notes/7.0.0.asciidoc b/libbeat/docs/release-notes/7.0.0.asciidoc index b0c7adc83702..284fca0b59d9 100644 --- a/libbeat/docs/release-notes/7.0.0.asciidoc +++ b/libbeat/docs/release-notes/7.0.0.asciidoc @@ -3,7 +3,7 @@ The list below covers the changes during the 7.0.0-alpha1, -alpha2, -beta1, -rc1 and -rc2 releases. -Also read <> for more detail about changes that affect +Also read <> for more detail about changes that affect upgrade. ==== Breaking changes diff --git a/libbeat/docs/release-notes/breaking/breaking-7.0.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.0.asciidoc deleted file mode 100644 index 443c4d60fa3f..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.0.asciidoc +++ /dev/null @@ -1,70 +0,0 @@ -[[breaking-changes-7.0]] - -=== Breaking changes in 7.0 -++++ -7.0 -++++ - -This section discusses the main changes that you need to be aware of to -migrate Beats to version 7.0. {see-relnotes} - -[float] -==== HTML escaping is disabled by default - -Starting with verion 7.0, embedded HTML or special symbols like `<` and `>` are -no longer escaped by default when publishing events. -To configure the old behavior of escaping HTML, set `escape_html: -true` in the output configuration. - -//tag::notable-breaking-changes[] -[float] -==== Filebeat registry - -Starting with version 7.0, Filebeat stores the registry in a sub-directory. -The directory is configured using the `filebeat.registry.path` setting. -If Filebeat finds an old registry file at the configured location, it will -automatically migrate the registry file to the new format. - -The settings `filebeat.registry_flush` and `filebeat.registry_file_permission` -have been renamed to `filebeat.registry.flush` and -`filebeat.registry.file_permission`. - -//end::notable-breaking-changes[] - -[float] -==== ILM support - -Support for Index Lifecycle Management is GA with Beats version 7.0. This -release moved most ILM settings from the `output.elasticsearch.ilm` namespace to -the `setup.ilm` namespace. - -//tag::notable-breaking-changes[] - -[float] -==== Filebeat apache2 module renamed - -The Filebeat `apache2` module is renamed to `apache` in 7.0. - -[float] -==== Field name changes - -include::./field-name-changes.asciidoc[] - -//end::notable-breaking-changes[] - -[float] -==== Auditbeat type changes - -The Auditbeat JSON data types produced by the output have been changed to align -with the data types used in the Elasticsearch index template. - -.Auditbeat Type Changes in 7.0 -[frame="topbot",options="header"] -|====================== -|Field|Old Type|New Type -|`file.gid` |number |string -|`file.uid` |number |string -|`process.pid` |string |number -|`process.ppid` |string |number -|====================== - diff --git a/libbeat/docs/release-notes/breaking/breaking-7.1.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.1.asciidoc deleted file mode 100644 index 5bc58d33b21a..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.1.asciidoc +++ /dev/null @@ -1,14 +0,0 @@ -[[breaking-changes-7.1]] - -=== Breaking changes in 7.1 -++++ -7.1 -++++ -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.10.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.10.asciidoc deleted file mode 100644 index 96b3032a6e5c..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.10.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[[breaking-changes-7.10]] - -=== Breaking changes in 7.10 -++++ -7.10 -++++ - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] -No breaking changes. -// end::notable-breaking-changes[] - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. - diff --git a/libbeat/docs/release-notes/breaking/breaking-7.11.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.11.asciidoc deleted file mode 100644 index 42aa3e693e46..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.11.asciidoc +++ /dev/null @@ -1,29 +0,0 @@ -[[breaking-changes-7.11]] - -=== Breaking changes in 7.11 -++++ -7.11 -++++ - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -[float] -==== Field changes - -The following field changes are potentially breaking for anything that relies -on these fields: - -* In {filebeat}, the `suricata.eve.timestamp` alias field has been removed from -the Suricata module. - -* In {auditbeat}, the file integrity dataset no longer includes a leading dot -in `file.extension` values. For example, it will report `png` instead of `.png` -to comply with Elastic Common Schema (ECS). - -// end::notable-breaking-changes[] - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. diff --git a/libbeat/docs/release-notes/breaking/breaking-7.12.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.12.asciidoc deleted file mode 100644 index a5ef7e4929eb..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.12.asciidoc +++ /dev/null @@ -1,18 +0,0 @@ -[[breaking-changes-7.12]] - -=== Breaking changes in 7.12 -++++ -7.12 -++++ - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -No breaking changes. - -// end::notable-breaking-changes[] - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. diff --git a/libbeat/docs/release-notes/breaking/breaking-7.13.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.13.asciidoc deleted file mode 100644 index d90cf44b3894..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.13.asciidoc +++ /dev/null @@ -1,28 +0,0 @@ -[[breaking-changes-7.13]] - -=== Breaking changes in 7.13 -++++ -7.13 -++++ - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -[discrete] -==== Beats may not be sending data to some distributions of {es} - -In this release, Elastic is enabling a licensing change that was broadly -communicated earlier in 2021 -(https://www.elastic.co/pricing/faq/licensing[Licensing FAQ]). This change would -imply that 7.13 instances of Beats would fail to connect to 7.10 or earlier open -source distributions of {es} and {kib}. - -This licensing change ensures that the {beats} modules are sending data to an -officially supported versions of {es} and {kib} where Elastic can attest to the -quality and scale of the products. -// end::notable-breaking-changes[] - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. diff --git a/libbeat/docs/release-notes/breaking/breaking-7.14.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.14.asciidoc deleted file mode 100644 index 6a028d62014a..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.14.asciidoc +++ /dev/null @@ -1,24 +0,0 @@ -[[breaking-changes-7.14]] - -=== Breaking changes in 7.14 -++++ -7.14 -++++ - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -[discrete] -==== {beats} central management has been removed - -Starting in version 7.14, Beats central management has been removed. If you're -currently using Beats central management, we recommend that you start using -{fleet} instead. For more information, refer to the -{fleet-guide}/index.html[{fleet} documentation]. - -// end::notable-breaking-changes[] - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. diff --git a/libbeat/docs/release-notes/breaking/breaking-7.15.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.15.asciidoc deleted file mode 100644 index 38af5a7eb3a7..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.15.asciidoc +++ /dev/null @@ -1,81 +0,0 @@ -[[breaking-changes-7.15]] - -=== Breaking changes in 7.15 -++++ -7.15 -++++ - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -[discrete] -==== Beats dashboard import and export requires Kibana 7.15 - -Loading Kibana assets (such as dashboards and index templates) relies on -the Saved Object API. Therefore, to provide a reliable service, Beats requires -{kib} 7.15 to import and export dashboards. - -[discrete] -==== Field changes - -* In Filebeat, the `log.path` field has been renamed to `log.file.path` in the -`filestream` input to be consistent with the `log` input and ECS. - -* In Filebeat, alias fields that were used to point to ECS fields from modules -are now removed. The following alias fields were removed from the Suricata and -Traefik modules: -+ --- -* `suricata.eve.fileinfo.filename` -* `suricata.eve.fileinfo.size` -* `suricata.eve.dest_port` -* `suricata.eve.src_port` -* `suricata.eve.proto` -* `suricata.eve.src_ip` -* `suricata.eve.dest_ip` -* `suricata.eve.http.status` -* `suricata.eve.http.http_user_agent` -* `suricata.eve.http.http_refer` -* `suricata.eve.http.url` -* `suricata.eve.http.hostname` -* `suricata.eve.http.http_refer` -* `suricata.eve.http.url` -* `suricata.eve.http.hostname` -* `suricata.eve.http.length` -* `suricata.eve.http.http_method` -* `suricata.eve.alert.severity` -* `suricata.eve.alert.action` -* `suricata.eve.flow.bytes_toclient` -* `suricata.eve.flow.start` -* `suricata.eve.flow.pkts_toclient` -* `suricata.eve.flow.bytes_toserver` -* `suricata.eve.flow.pkts_toserver` -* `suricata.eve.app_proto` -* `traefik.access.user_agent.device` --- - -* In Heartbeat, the `event.dataset` value is now set to the monitor type / -Fleet dataset to fix inconsistencies between Heartbeat and Fleet. - -[discrete] -==== Filebeat Crowdstrike ingest pipeline no longer flattens `process` fields - -In previous releases, the ingest pipeline used by the Crowdstrike module -flattened `process` fields instead of creating nested fields. The mix of -flattened and nested fields with similar names was confusing and led to errors -when running queries or automated processes that expect nested fields. To fix -this problem, the ingest pipeline no longer flattens `process` fields. - -[discrete] -==== Heartbeat `watch_poll` functionality has been removed - -The Heartbeat `watch_poll` functionality was deprecated a long time ago, and has -been completely removed in 7.15. - -// end::notable-breaking-changes[] - diff --git a/libbeat/docs/release-notes/breaking/breaking-7.16.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.16.asciidoc deleted file mode 100644 index 29e82e73337e..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.16.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[[breaking-changes-7.16]] - -=== Breaking changes in 7.16 -++++ -7.16 -++++ - -See the <> for a complete list of changes, -including changes to beta or experimental functionality. - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -// tag::notable-breaking-changes[] - -[discrete] -==== {journalbeat} is removed in 7.16 - -{journalbeat}, a lightweight shipper for collecting logs written by the Journald -system service, is removed in 7.16. This functionality is instead provided as -a {filebeat} input. If you're currently using {journalbeat}, you should -use the `journald` input in {filebeat} instead. For more information, refer to -the -{filebeat-ref}/filebeat-input-journald.html[Journald input] documentation. - -If you're using {agent} instead of {beats}, you can collect Journald logs by -adding the *Custom Journald logs* integration to your agent policy. For more -information, refer to -{fleet-guide}/add-integration-to-policy.html[Add an {agent} integration to a policy]. - -//TODO: Add pointer to the integrations docs for custom journald logs when -//available. - -[discrete] -==== Custom {beats} generator is deprecated in 7.16 - -The generator code for creating custom {beats} is deprecated in 7.16.0 and will -be removed in 8.0.0. You can continue to build custom {beats} using the -generators available in 7.16, or refer to existing {beats} as working examples. - -// end::notable-breaking-changes[] - diff --git a/libbeat/docs/release-notes/breaking/breaking-7.2.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.2.asciidoc deleted file mode 100644 index 620e70201a5f..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.2.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[breaking-changes-7.2]] - -=== Breaking changes in 7.2 -++++ -7.2 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.3.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.3.asciidoc deleted file mode 100644 index 2d1d4a845e53..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.3.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[breaking-changes-7.3]] - -=== Breaking changes in 7.3 -++++ -7.3 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.4.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.4.asciidoc deleted file mode 100644 index 544ce43755b4..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.4.asciidoc +++ /dev/null @@ -1,41 +0,0 @@ -[[breaking-changes-7.4]] - -=== Breaking changes in 7.4 -++++ -7.4 -++++ - -This section discusses the main changes that you need to be aware of to -migrate Beats to version 7.4. {see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -[float] -==== Golang version update - -The Beats platform has been updated to use Golang 1.12.9. - -[float] -==== {auditbeat} `system` module changes - -The `socket` dataset in the `system` module now uses Kprobes for finer-grained -monitoring and UDP support. For more information, see -{auditbeat-ref}/auditbeat-dataset-system-socket.html[System socket dataset]. - -[float] -==== {filebeat} field name changes - -Some field names exported by the `asa` fileset in the `cisco` module have -changed: - -[options="header"] -|==== -|Old field |New field -|`log.original` |`event.original` -|`cisco.asa.list_id` |`cisco.asa.rule_name` -|==== - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.5.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.5.asciidoc deleted file mode 100644 index 04bd85b27aa2..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.5.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[[breaking-changes-7.5]] - -=== Breaking changes in 7.5 -++++ -7.5 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc deleted file mode 100644 index e838710c95fc..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[[breaking-changes-7.6]] - -=== Breaking changes in 7.6 -++++ -7.6 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -[float] -==== Version removed from default ILM policy name - -To prevent custom ILM policies from breaking during upgrades, the default -ILM policy name no longer includes the {beats} version. - -If you want to include the version, use the `setup.ilm.policy_name` -configuration setting. For example: - -[source,yaml] ----- -setup.ilm.policy_name: "%{[agent.name]}-%{[agent.version]}" ----- - -[float] -==== Two Beat instances can no longer share the same data path - -To prevent accidental overwriting of internal state, two instances of the -same Beat running on the same host can no longer share the same data path. -To customize the data path for a Beat, use the `path.data` configuration -setting. For example: - -[source,yaml] ----- -path.data: ${path.home}/data-instance1 ----- - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.7.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.7.asciidoc deleted file mode 100644 index 651731b7bc17..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.7.asciidoc +++ /dev/null @@ -1,52 +0,0 @@ -[[breaking-changes-7.7]] - -=== Breaking changes in 7.7 -++++ -7.7 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] - -[float] -==== Environment variables can no longer reference other environment variables - -Environment variables are still supported in the {beats} configuration. -However, starting in Version 7.7.0, an environment variable cannot reference -another environment variable or event in the configuration. - -For example, if you have `export VAR1=${VAR2}` and use `var: ${VAR1}` in -the configuration, this results in `var: "${VAR2}"`, where -`${VAR2}` is not expanded. - -[float] -==== Docker and kubernetes processors no longer allowed in script processor - -Prior to this release, it was possible to use the `add_docker_metadata` and -`add_kubernetes_metadata` processors in the `script` processor. This was not a -good practice because it sometimes resulted in memory and file descriptor leaks. - -Starting in version 7.7.0, scripts that use these processors will fail. To -resolve this problem, define the processors in your configuration instead of the -script. - -[float] -==== Systemd unit file no longer overrides logging options - -Prior to this release, the systemd unit file set `BEAT_LOG_OPTS=-e`, which -caused {beats} to ignore the logging options specified in the -configuration file. - -The systemd unit file no longer sets this option, and the logging settings -specified under `logging` in the configuration now work. - -If you set `Environment="BEAT_LOG_OPTS=` in a previous release to work around -this problem, remove that workaround now, and use the `logging` options in the -configuration file to control logging behavior. For example, see -{filebeat-ref}/configuration-logging.html[Configure logging]. - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.8.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.8.asciidoc deleted file mode 100644 index f77c727fc1b8..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.8.asciidoc +++ /dev/null @@ -1,31 +0,0 @@ -[[breaking-changes-7.8]] - -=== Breaking changes in 7.8 -++++ -7.8 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] -[float] -==== Known issue with Kafka output - -The Kafka output fails to connect when using multiple TLS brokers. We advise -not to upgrade to {beats} 7.8.0 if you're using the Kafka output in this -configuration. - -[float] -==== APM Instrumentation - -Libbeat includes the Elastic APM Agent for instrumenting the publishing pipeline. -Currently the Elasticsearch output is instrumented. APM can be enabled simply with -setting the `ELASTIC_APM_ACTIVE` environment variable to `true` when starting the beat. -To make tracing possible, the `Publish` method of the `Client` interface takes a -`Context` object as first argument. That `Context` is intended for propagating -request-scoped values, not for cancellation. - -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-7.9.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.9.asciidoc deleted file mode 100644 index 19feb7c4d3bd..000000000000 --- a/libbeat/docs/release-notes/breaking/breaking-7.9.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -[[breaking-changes-7.9]] - -=== Breaking changes in 7.9 -++++ -7.9 -++++ - -{see-relnotes} - -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] -[float] -==== Some {filebeat} Okta module settings no longer accept JSON strings - -Prior to version 7.9, the OKTA module accepted JSON strings for some -configuration settings (`http_headers`, `http_request_body`, `pagination`, -`rate_limit`, and `ssl`). This was inconsistent with other {beats} settings, so -it has been removed. - -The affected settings now expect regular YAML objects for values. -// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc b/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc index 2134a405372b..c840e54d57d7 100644 --- a/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc +++ b/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc @@ -5,14 +5,11 @@ 8.0 ++++ -This section discusses the main changes that you should be aware of if you -upgrade the Beats to version 8.0. - -{see-relnotes} - //NOTE: The notable-breaking-changes tagged regions are re-used in the //Installation and Upgrade Guide //tag::notable-breaking-changes[] +Changes will be described in the GA release. + // end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking.asciidoc b/libbeat/docs/release-notes/breaking/breaking.asciidoc index 11e6b7321e55..bf69392dc5f4 100644 --- a/libbeat/docs/release-notes/breaking/breaking.asciidoc +++ b/libbeat/docs/release-notes/breaking/breaking.asciidoc @@ -4,82 +4,13 @@ == Breaking changes As a general rule, we strive to keep backwards compatibility between minor -versions (e.g. 7.x to 7.y) so you can upgrade without any configuration file -changes, but there are breaking changes between major versions (e.g. 6.x to -7.y). Migrating directly between non consecutive major versions (e.g. 5.x to -7.x) is not recommended. +versions (e.g. 8.x to 8.y) so you can upgrade without any configuration file +changes, but there are breaking changes between major versions (e.g. 7.x to +8.y). Migrating directly between non consecutive major versions (e.g. 6.x to +8.x) is not recommended. See the following topics for a description of breaking changes: -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -* <> - -include::breaking-7.17.asciidoc[] - -include::breaking-7.16.asciidoc[] - -include::breaking-7.15.asciidoc[] - -include::breaking-7.14.asciidoc[] - -include::breaking-7.13.asciidoc[] - -include::breaking-7.12.asciidoc[] - -include::breaking-7.11.asciidoc[] - -include::breaking-7.10.asciidoc[] - -include::breaking-7.9.asciidoc[] - -include::breaking-7.8.asciidoc[] - -include::breaking-7.7.asciidoc[] - -include::breaking-7.6.asciidoc[] - -include::breaking-7.5.asciidoc[] - -include::breaking-7.4.asciidoc[] - -include::breaking-7.3.asciidoc[] - -include::breaking-7.2.asciidoc[] - -include::breaking-7.1.asciidoc[] - -include::breaking-7.0.asciidoc[] +* <> +include::breaking-8.0.asciidoc[] diff --git a/libbeat/docs/release-notes/breaking/field-name-changes.asciidoc b/libbeat/docs/release-notes/breaking/field-name-changes.asciidoc deleted file mode 100644 index b8dac59f3fdc..000000000000 --- a/libbeat/docs/release-notes/breaking/field-name-changes.asciidoc +++ /dev/null @@ -1,555 +0,0 @@ -.Auditbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`auditd.messages` |`event.original` -|`auditd.warnings` |`error.message` -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`event.type` |`auditd.message_type` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`process.cwd` |`process.working_directory` -|`process.exe` |`process.executable` -|`source.hostname` |`source.domain` -|`user.auid` |`user.audit.id` -|`user.egid` |`user.effective.group.id` -|`user.euid` |`user.effective.id` -|`user.fsgid` |`user.filesystem.group.id` -|`user.fsuid` |`user.filesystem.id` -|`user.gid` |`user.group.id` -|`user.name_map.auid` |`user.audit.name` -|`user.name_map.egid` |`user.effective.group.name` -|`user.name_map.euid` |`user.effective.name` -|`user.name_map.fsgid` |`user.filesystem.group.name` -|`user.name_map.fsuid` |`user.filesystem.name` -|`user.name_map.gid` |`user.group.name` -|`user.name_map.sgid` |`user.saved.group.name` -|`user.name_map.suid` |`user.saved.name` -|`user.name_map.uid` |`user.name` -|`user.sgid` |`user.saved.group.id` -|`user.suid` |`user.saved.id` -|`user.uid` |`user.id` -|====================== - -.Filebeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`apache2.access.agent` |`user_agent.original` -|`apache2.access.body_sent.bytes` |`http.response.body.bytes` -|`apache2.access.geoip.city_name` |`source.geo.city_name` -|`apache2.access.geoip.continent_name` |`source.geo.continent_name` -|`apache2.access.geoip.country_iso_code` |`source.geo.country_iso_code` -|`apache2.access.geoip.location` |`source.geo.location` -|`apache2.access.geoip.region_iso_code` |`source.geo.region_iso_code` -|`apache2.access.geoip.region_name` |`source.geo.region_name` -|`apache2.access.http_version` |`http.version` -|`apache2.access.method` |`http.request.method` -|`apache2.access.referrer` |`http.request.referrer` -|`apache2.access.remote_ip` |`source.address` -|`apache2.access.response_code` |`http.response.status_code` -|`apache2.access.url` |`url.original` -|`apache2.access.user_agent.device` |`user_agent.device.name` -|`apache2.access.user_agent.major` |`user_agent.version` -|`apache2.access.user_agent.minor` |`user_agent.version` -|`apache2.access.user_agent.name` |`user_agent.name` -|`apache2.access.user_agent.original` |`user_agent.original` -|`apache2.access.user_agent.os` |`user_agent.os.full_name` -|`apache2.access.user_agent.os_major` |`user_agent.os.version` -|`apache2.access.user_agent.os_minor` |`user_agent.os.version` -|`apache2.access.user_agent.os_name` |`user_agent.os.name` -|`apache2.access.user_agent.os_patch` |`user_agent.os.version` -|`apache2.access.user_agent.patch` |`user_agent.version` -|`apache2.access.user_name` |`user.name` -|`apache2.error.client` |`source.address` -|`apache2.error.level` |`log.level` -|`apache2.error.message` |`message` -|`apache2.error.pid` |`process.pid` -|`apache2.error.tid` |`process.thread.id` -|`auditd.log.acct` |`user.name` -|`auditd.log.agid` |`user.audit.group.id` -|`auditd.log.arch` |`host.architecture` -|`auditd.log.auid` |`user.audit.id` -|`auditd.log.cmd` |`process.args` -|`auditd.log.comm` |`process.name` -|`auditd.log.dst` |`destination.address` -|`auditd.log.egid` |`user.effective.group.id` -|`auditd.log.euid` |`user.effective.id` -|`auditd.log.exe` |`process.executable` -|`auditd.log.fsgid` |`user.filesystem.group.id` -|`auditd.log.geoip.city_name` |`source.geo.city_name` -|`auditd.log.geoip.continent_name` |`source.geo.continent_name` -|`auditd.log.geoip.country_iso_code` |`source.geo.country_iso_code` -|`auditd.log.geoip.location` |`source.geo.location` -|`auditd.log.geoip.region_iso_code` |`source.geo.region_iso_code` -|`auditd.log.geoip.region_name` |`source.geo.region_name` -|`auditd.log.gid` |`user.group.id` -|`auditd.log.msg` |`message` -|`auditd.log.ogid` |`user.owner.group.id` -|`auditd.log.ouid` |`user.owner.id` -|`auditd.log.pid` |`process.pid` -|`auditd.log.ppid` |`process.ppid` -|`auditd.log.record_type` |`event.action` -|`auditd.log.res` |`event.outcome` -|`auditd.log.sgid` |`user.saved.group.id` -|`auditd.log.src` |`source.address` -|`auditd.log.suid` |`user.saved.id` -|`auditd.log.terminal` |`user.terminal` -|`auditd.log.uid` |`user.id` -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`elasticsearch.audit.origin_address` |`source.ip` -|`elasticsearch.audit.principal` |`user.name` -|`elasticsearch.audit.request_body` |`http.request.body.content` -|`elasticsearch.audit.uri` |`url.original` -|`elasticsearch.slowlog.took_millis` |`event.duration` -|`fileset.module` |`event.module` -|`haproxy.client.ip` |`source.address` -|`haproxy.client.port` |`source.port` -|`haproxy.destination.ip` |`destination.ip` -|`haproxy.destination.port` |`destination.port` -|`haproxy.geoip.city_name` |`source.geo.city_name` -|`haproxy.geoip.continent_name` |`source.geo.continent_name` -|`haproxy.geoip.country_iso_code` |`source.geo.country_iso_code` -|`haproxy.geoip.location` |`source.geo.location` -|`haproxy.geoip.region_iso_code` |`source.geo.region_iso_code` -|`haproxy.geoip.region_name` |`source.geo.region_name` -|`haproxy.http.request.time_active_ms` |`event.duration` -|`haproxy.http.response.status_code` |`http.response.status_code` -|`haproxy.pid` |`process.pid` -|`haproxy.process_name` |`process.name` -|`haproxy.total_waiting_time_ms` |`event.duration` -|`http.response.content_length` |`http.response.body.bytes` -|`http.response.elapsed_time` |`event.duration` -|`icinga.debug.message` |`message` -|`icinga.debug.severity` |`log.level` -|`icinga.main.message` |`message` -|`icinga.main.severity` |`log.level` -|`icinga.startup.message` |`message` -|`icinga.startup.severity` |`log.level` -|`iis.access.body_received.bytes` |`http.request.body.bytes` -|`iis.access.body_sent.bytes` |`http.response.body.bytes` -|`iis.access.geoip.city_name` |`source.geo.city_name` -|`iis.access.geoip.continent_name` |`source.geo.continent_name` -|`iis.access.geoip.country_iso_code` |`source.geo.country_iso_code` -|`iis.access.geoip.location` |`source.geo.location` -|`iis.access.geoip.region_iso_code` |`source.geo.region_iso_code` -|`iis.access.geoip.region_name` |`source.geo.region_name` -|`iis.access.hostname` |`destination.domain` -|`iis.access.method` |`http.request.method` -|`iis.access.port` |`destination.port` -|`iis.access.query_string` |`url.query` -|`iis.access.referrer` |`http.request.referrer` -|`iis.access.remote_ip` |`source.address` -|`iis.access.request_time_ms` |`event.duration` -|`iis.access.response_code` |`http.response.status_code` -|`iis.access.server_ip` |`destination.address` -|`iis.access.url` |`url.path` -|`iis.access.user_agent.device` |`user_agent.device.name` -|`iis.access.user_agent.major` |`user_agent.version` -|`iis.access.user_agent.minor` |`user_agent.version` -|`iis.access.user_agent.name` |`user_agent.name` -|`iis.access.user_agent.original` |`user_agent.original` -|`iis.access.user_agent.os` |`user_agent.os.full_name` -|`iis.access.user_agent.os_major` |`user_agent.os.version` -|`iis.access.user_agent.os_minor` |`user_agent.os.version` -|`iis.access.user_agent.os_name` |`user_agent.os.name` -|`iis.access.user_agent.os_patch` |`user_agent.os.version` -|`iis.access.user_agent.patch` |`user_agent.version` -|`iis.access.user_name` |`user.name` -|`iis.error.geoip.city_name` |`source.geo.city_name` -|`iis.error.geoip.continent_name` |`source.geo.continent_name` -|`iis.error.geoip.country_iso_code` |`source.geo.country_iso_code` -|`iis.error.geoip.location` |`source.geo.location` -|`iis.error.geoip.region_iso_code` |`source.geo.region_iso_code` -|`iis.error.geoip.region_name` |`source.geo.region_name` -|`iis.error.http_version` |`http.version` -|`iis.error.method` |`http.request.method` -|`iis.error.remote_ip` |`source.address` -|`iis.error.remote_port` |`source.port` -|`iis.error.response_code` |`http.response.status_code` -|`iis.error.server_ip` |`destination.address` -|`iis.error.server_port` |`destination.port` -|`iis.error.url` |`url.original` -|`kafka.log.level` |`log.level` -|`kafka.log.message` |`message` -|`kibana.log.meta.meta.statusCode` |`http.response.status_code` -|`kibana.log.meta.method` |`http.request.method` -|`kibana.log.meta.req.headers.referer` |`http.request.referrer` -|`kibana.log.meta.req.headers.user-agent` |`user_agent.original` -|`kibana.log.meta.req.referer` |`http.request.referrer` -|`kibana.log.meta.req.remoteAddress` |`source.address` -|`kibana.log.meta.req.url` |`url.original` -|`logstash.log.level` |`log.level` -|`logstash.log.message` |`message` -|`logstash.slowlog.level` |`log.level` -|`logstash.slowlog.took_in_nanos` |`event.duration` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`mongodb.log.message` |`message` -|`mongodb.log.severity` |`log.level` -|`mysql.error.level` |`log.level` -|`mysql.error.message` |`message` -|`mysql.error.thread_id` |`mysql.thread_id` -|`mysql.slowlog.host` |`source.domain` -|`mysql.slowlog.id` |`mysql.thread_id` -|`mysql.slowlog.ip` |`source.ip` -|`mysql.slowlog.query_time.sec` |`event.duration` -|`mysql.slowlog.user` |`user.name` -|`nginx.access.agent` |`user_agent.original` -|`nginx.access.body_sent.bytes` |`http.response.body.bytes` -|`nginx.access.geoip.city_name` |`source.geo.city_name` -|`nginx.access.geoip.continent_name` |`source.geo.continent_name` -|`nginx.access.geoip.country_iso_code` |`source.geo.country_iso_code` -|`nginx.access.geoip.location` |`source.geo.location` -|`nginx.access.geoip.region_iso_code` |`source.geo.region_iso_code` -|`nginx.access.geoip.region_name` |`source.geo.region_name` -|`nginx.access.http_version` |`http.version` -|`nginx.access.method` |`http.request.method` -|`nginx.access.referrer` |`http.request.referrer` -|`nginx.access.remote_ip` |`source.address` -|`nginx.access.response_code` |`http.response.status_code` -|`nginx.access.url` |`url.original` -|`nginx.access.user_agent.device` |`user_agent.device.name` -|`nginx.access.user_agent.major` |`user_agent.version` -|`nginx.access.user_agent.minor` |`user_agent.version` -|`nginx.access.user_agent.name` |`user_agent.name` -|`nginx.access.user_agent.os` |`user_agent.os.full_name` -|`nginx.access.user_agent.os_major` |`user_agent.os.version` -|`nginx.access.user_agent.os_minor` |`user_agent.os.version` -|`nginx.access.user_agent.os_name` |`user_agent.os.name` -|`nginx.access.user_agent.os_patch` |`user_agent.os.version` -|`nginx.access.user_agent.patch` |`user_agent.version` -|`nginx.access.user_name` |`user.name` -|`nginx.error.level` |`log.level` -|`nginx.error.message` |`message` -|`nginx.error.pid` |`process.pid` -|`nginx.error.tid` |`process.thread.id` -|`offset` |`log.offset` -|`postgresql.log.duration` |`event.duration` -|`postgresql.log.level` |`log.level` -|`postgresql.log.message` |`message` -|`postgresql.log.thread_id` |`process.pid` -|`postgresql.log.timezone` |`event.timezone` -|`postgresql.log.user` |`user.name` -|`process.exe` |`process.executable` -|`read_timestamp` |`event.created` -|`redis.log.level` |`log.level` -|`redis.log.message` |`message` -|`redis.log.pid` |`process.pid` -|`source_ecs.geo.city_name` |`source.geo.city_name` -|`source_ecs.geo.continent_name` |`source.geo.continent_name` -|`source_ecs.geo.country_iso_code` |`source.geo.country_iso_code` -|`source_ecs.geo.location` |`source.geo.location` -|`source_ecs.geo.region_iso_code` |`source.geo.region_iso_code` -|`source_ecs.geo.region_name` |`source.geo.region_name` -|`source_ecs.ip` |`source.ip` -|`source_ecs.port` |`source.port` -|`suricata.eve.alert.action` |`event.outcome` -|`suricata.eve.alert.severity` |`event.severity` -|`suricata.eve.app_proto` |`network.protocol` -|`suricata.eve.dest_ip` |`destination.ip` -|`suricata.eve.dest_port` |`destination.port` -|`suricata.eve.fileinfo.filename` |`file.path` -|`suricata.eve.fileinfo.size` |`file.size` -|`suricata.eve.flow.bytes_toclient` |`destination.bytes` -|`suricata.eve.flow.bytes_toserver` |`source.bytes` -|`suricata.eve.flow.pkts_toclient` |`destination.packets` -|`suricata.eve.flow.pkts_toserver` |`source.packets` -|`suricata.eve.flow.start` |`event.start` -|`suricata.eve.http.hostname` |`url.domain` -|`suricata.eve.http.http_method` |`http.request.method` -|`suricata.eve.http.http_refer` |`http.request.referrer` -|`suricata.eve.http.http_user_agent` |`user_agent.original` -|`suricata.eve.http.length` |`http.response.body.bytes` -|`suricata.eve.http.status` |`http.response.status_code` -|`suricata.eve.http.url` |`url.original` -|`suricata.eve.proto` |`network.transport` -|`suricata.eve.src_ip` |`source.ip` -|`suricata.eve.src_port` |`source.port` -|`suricata.eve.timestamp` |`@timestamp` -|`system.auth.groupadd.gid` |`group.id` -|`system.auth.groupadd.name` |`group.name` -|`system.auth.hostname` |`host.hostname` -|`system.auth.message` |`message` -|`system.auth.pid` |`process.pid` -|`system.auth.program` |`process.name` -|`system.auth.ssh.geoip.city_name` |`source.geo.city_name` -|`system.auth.ssh.geoip.continent_name` |`source.geo.continent_name` -|`system.auth.ssh.geoip.country_iso_code` |`source.geo.country_iso_code` -|`system.auth.ssh.geoip.location` |`source.geo.location` -|`system.auth.ssh.geoip.region_iso_code` |`source.geo.region_iso_code` -|`system.auth.ssh.geoip.region_name` |`source.geo.region_name` -|`system.auth.ssh.ip` |`source.ip` -|`system.auth.ssh.port` |`source.port` -|`system.auth.timestamp` |`@timestamp` -|`system.auth.user` |`user.name` -|`system.auth.useradd.gid` |`group.id` -|`system.auth.useradd.name` |`user.name` -|`system.auth.useradd.uid` |`user.id` -|`system.syslog.hostname` |`host.hostname` -|`system.syslog.message` |`message` -|`system.syslog.pid` |`process.pid` -|`system.syslog.program` |`process.name` -|`traefik.access.agent` |`user_agent.original` -|`traefik.access.body_sent.bytes` |`http.response.body.bytes` -|`traefik.access.duration` |`event.duration` -|`traefik.access.geoip.city_name` |`source.geo.city_name` -|`traefik.access.geoip.continent_name` |`source.geo.continent_name` -|`traefik.access.geoip.country_iso_code` |`source.geo.country_iso_code` -|`traefik.access.geoip.location` |`source.geo.location` -|`traefik.access.geoip.region_iso_code` |`source.geo.region_iso_code` -|`traefik.access.geoip.region_name` |`source.geo.region_name` -|`traefik.access.http_version` |`http.version` -|`traefik.access.method` |`http.request.method` -|`traefik.access.referrer` |`http.request.referrer` -|`traefik.access.remote_ip` |`source.address` -|`traefik.access.response_code` |`http.response.status_code` -|`traefik.access.url` |`url.original` -|`traefik.access.user_agent.device` |`user_agent.device.name` -|`traefik.access.user_agent.major` |`user_agent.version` -|`traefik.access.user_agent.minor` |`user_agent.version` -|`traefik.access.user_agent.name` |`user_agent.name` -|`traefik.access.user_agent.original` |`user_agent.original` -|`traefik.access.user_agent.os` |`user_agent.os.full_name` -|`traefik.access.user_agent.os_major` |`user_agent.os.version` -|`traefik.access.user_agent.os_minor` |`user_agent.os.version` -|`traefik.access.user_agent.os_name` |`user_agent.os.name` -|`traefik.access.user_agent.os_patch` |`user_agent.os.version` -|`traefik.access.user_agent.patch` |`user_agent.version` -|`traefik.access.user_name` |`user.name` -|====================== - -.Heartbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`http.url` |`url.full` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`monitor.host` |`url.domain` -|`monitor.scheme` |`url.scheme` -|`process.exe` |`process.executable` -|`resolve.host` |`url.domain` -|`tcp.port` |`url.port` -|====================== - -.Journalbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`host.name` |`host.hostname` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`process.exe` |`process.executable` -|`read_timestamp` |`event.created` -|====================== - -.Metricbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`haproxy.info.pid` |`process.pid` -|`haproxy.stat.process_id` |`process.pid` -|`http.request.body` |`http.request.body.content` -|`kibana.stats.transport_address` |`service.address` -|`kibana.stats.uuid` |`service.id` -|`kibana.stats.version` |`service.version` -|`kibana.status.uuid` |`service.id` -|`kibana.status.version.number` |`service.version` -|`logstash.node.host` |`service.hostname` -|`logstash.node.jvm.pid` |`process.pid` -|`logstash.node.version` |`service.version` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`metricset.host` |`service.address` -|`metricset.module` |`event.module` -|`metricset.namespace` |`event.dataset` -|`metricset.rrt` |`event.duration` -|`mongodb.status.process` |`process.name` -|`mongodb.status.version` |`service.version` -|`php_fpm.status.content_length` |`http.response.body.bytes` -|`php_fpm.status.pid` |`process.pid` -|`php_fpm.status.request_method` |`http.request.method` -|`php_fpm.status.request_uri` |`url.original` -|`php_fpm.status.user` |`http.response.user.name` -|`process.exe` |`process.executable` -|`rabbitmq.connection.node` |`rabbitmq.node.name` -|`rabbitmq.connection.user` |`user.name` -|`rabbitmq.connection.vhost` |`rabbitmq.vhost` -|`rabbitmq.exchange.user` |`user.name` -|`rabbitmq.exchange.vhost` |`rabbitmq.vhost` -|`rabbitmq.queue.node` |`rabbitmq.node.name` -|`rabbitmq.queue.vhost` |`rabbitmq.vhost` -|`redis.info.server.os` |`os.full` -|`redis.info.server.process_id` |`process.pid` -|`redis.info.server.version` |`service.version` -|`system.process.cwd` |`process.working_directory` -|`system.process.name` |`process.name` -|`system.process.pgid` |`process.pgid` -|`system.process.pid` |`process.pid` -|`system.process.ppid` |`process.ppid` -|`system.process.username` |`user.name` -|`system.socket.direction` |`network.direction` -|`system.socket.family` |`network.type` -|`system.socket.process.command` |`process.name` -|`system.socket.process.exe` |`process.executable` -|`system.socket.process.pid` |`process.pid` -|`system.socket.user.id` |`user.id` -|`system.socket.user.name` |`user.full_name` -|`zookeeper.mntr.version` |`service.version` -|====================== - -.Packetbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`bytes_in` |`source.bytes` -|`bytes_out` |`destination.bytes` -|`dest.stats.net_bytes_total` |`destination.bytes` -|`dest.stats.net_packets_total` |`destination.packets` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`final` |`flow.final` -|`flow_id` |`flow.id` -|`http.request.body` |`http.request.body.content` -|`http.request.params` |`url.query` -|`http.response.body` |`http.response.body.content` -|`http.response.code` |`http.response.status_code` -|`http.response.phrase` |`http.response.status_phrase` -|`last_time` |`event.end` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`method` |`http.request.method` -|`mysql.iserror` |`status` -|`no_request` |`cassandra.no_request` -|`notes` |`error.message` -|`path` |`url.path` -|`process.exe` |`process.executable` -|`real_ip` |`network.forwarded_ip` -|`responsetime` |`event.duration` -|`rpc.call_size` |`source.bytes` -|`rpc.reply_size` |`destination.bytes` -|`rpc.time` |`event.duration` -|`source.stats.net_bytes_total` |`source.bytes` -|`source.stats.net_packets_total` |`source.packets` -|`start_time` |`event.start` -|`transport` |`network.transport` -|====================== - -.Winlogbeat renamed fields in 7.0 -[frame="topbot",options="header"] -|====================== -|Old Field|New Field -|`activity_id` |`winlog.activity_id` -|`beat.hostname` |`agent.hostname` -|`beat.name` |`host.name` -|`beat.timezone` |`event.timezone` -|`beat.version` |`agent.version` -|`computer_name` |`winlog.computer_name` -|`docker.container.id` |`container.id` -|`docker.container.image` |`container.image.name` -|`docker.container.labels` |`container.labels` -|`docker.container.name` |`container.name` -|`event_id` |`winlog.event_id` -|`keywords` |`winlog.keywords` -|`level` |`log.level` -|`log_name` |`winlog.channel` -|`message_error` |`error.message` -|`meta.cloud.availability_zone` |`cloud.availability_zone` -|`meta.cloud.instance_id` |`cloud.instance.id` -|`meta.cloud.instance_name` |`cloud.instance.name` -|`meta.cloud.machine_type` |`cloud.machine.type` -|`meta.cloud.project_id` |`cloud.project.id` -|`meta.cloud.provider` |`cloud.provider` -|`meta.cloud.region` |`cloud.region` -|`opcode` |`winlog.opcode` -|`process.exe` |`process.executable` -|`process_id` |`winlog.process.pid` -|`provider_guid` |`winlog.provider_guid` -|`record_number` |`winlog.record_id` -|`related_activity_id` |`winlog.related_activity_id` -|`source_name` |`winlog.provider_name` -|`task` |`winlog.task` -|`thread_id` |`winlog.process.thread.id` -|`type` |`winlog.api` -|`user.domain` |`winlog.user.domain` -|`user.identifier` |`winlog.user.identifier` -|`user.type` |`winlog.user.type` -|`version` |`winlog.version` -|`xml` |`event.original` -|====================== - diff --git a/libbeat/docs/upgrading.asciidoc b/libbeat/docs/upgrading.asciidoc index b183312fc40e..6cdea344104d 100644 --- a/libbeat/docs/upgrading.asciidoc +++ b/libbeat/docs/upgrading.asciidoc @@ -1,6 +1,9 @@ [[upgrading]] == Upgrade +coming[8.0.0] + +///// This section gives general recommendations for upgrading {beats} shippers: * <> @@ -441,3 +444,5 @@ PUT /metricbeat-6.6.2-2019.04.09/_settings } -------------------------------------------------- // CONSOLE + +///// \ No newline at end of file From e208c22507890eb3633d55f1ba7d388f3076b2d6 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 9 Feb 2022 10:37:26 +0100 Subject: [PATCH 07/52] libbeat/common/transport: fix log message about TLS (#30063) This commit fixes the log message issued by the `test output` command. Our current TLS verification relies on more than the value of `tlsConfig.InsecureSkipVerify`, so the previous implementation would log that TLS was disabled when it was not. This commit fixes it by checking the value of `config.Verification`. Co-authored-by: Craig MacKenzie --- CHANGELOG.next.asciidoc | 1 + libbeat/common/transport/tls.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 285cfce9026e..c3186a25a28a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -38,6 +38,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif *Affecting all Beats* +- Fix a logging bug when `ssl.verification_mode` was set to `full` or `certificate`, the command `test output` incorrectly logged that TLS was disabled. - Fix field names with `add_network_direction` processor. {issue}29747[29747] {pull}29751[29751] *Auditbeat* diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index 5f8ade67012c..98ebe846d982 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -154,7 +154,14 @@ func tlsDialWith( } } - if tlsConfig.InsecureSkipVerify { + // We only check the status of config.Verification (`ssl.verification_mode` + // in the configuration file) because we have a custom verification logic + // implemented by setting tlsConfig.VerifyConnection that runs regardless of + // the status of tlsConfig.InsecureSkipVerify. + // For verification modes VerifyFull and VerifyCeritifcate we set + // tlsConfig.InsecureSkipVerify to true, hence it's not an indicator of + // whether TLS verification is enabled or not. + if config.Verification == tlscommon.VerifyNone { d.Warn("security", "server's certificate chain verification is disabled") } else { d.Info("security", "server's certificate chain verification is enabled") From ffa1387ab96cb68fa3781c28c98c9ca7c78ae341 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Wed, 9 Feb 2022 08:42:16 -0500 Subject: [PATCH 08/52] Fix the orphan child processes on Windows (#30254) --- libbeat/common/proc/job_unix.go | 52 +++++++++ libbeat/common/proc/job_windows.go | 101 ++++++++++++++++++ x-pack/elastic-agent/main.go | 8 ++ .../pkg/core/plugin/process/start.go | 11 ++ x-pack/osquerybeat/internal/osqd/osqueryd.go | 9 ++ x-pack/osquerybeat/main.go | 9 ++ 6 files changed, 190 insertions(+) create mode 100644 libbeat/common/proc/job_unix.go create mode 100644 libbeat/common/proc/job_windows.go diff --git a/libbeat/common/proc/job_unix.go b/libbeat/common/proc/job_unix.go new file mode 100644 index 000000000000..1ff62b4af305 --- /dev/null +++ b/libbeat/common/proc/job_unix.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows +// +build !windows + +package proc + +import ( + "os" +) + +// Job is noop on unix +type Job int + +var ( + // Public global JobObject, 0 value on linux + JobObject Job +) + +func CreateJobObject() (pj Job, err error) { + return +} + +// NewJob is noop on unix +func NewJob() (Job, error) { + return 0, nil +} + +// Close is noop on unix +func (job Job) Close() error { + return nil +} + +// Assign is noop on unix +func (job Job) Assign(p *os.Process) error { + return nil +} diff --git a/libbeat/common/proc/job_windows.go b/libbeat/common/proc/job_windows.go new file mode 100644 index 000000000000..c131846aa4c7 --- /dev/null +++ b/libbeat/common/proc/job_windows.go @@ -0,0 +1,101 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows +// +build windows + +package proc + +import ( + "os" + "unsafe" + + "golang.org/x/sys/windows" +) + +// Job is wrapper for windows JobObject +// https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects +// This helper guarantees a clean process tree kill on job handler close +type Job windows.Handle + +var ( + // Public global JobObject should be initialized once in main + JobObject Job +) + +// CreateJobObject creates JobObject on Windows, global per process +// Should only be initialized once in main function +func CreateJobObject() (pj Job, err error) { + if pj, err = NewJob(); err != nil { + return pj, err + } + JobObject = pj + return pj, nil +} + +// NewJob creates a instance of the JobObject +func NewJob() (Job, error) { + h, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, err + } + + // From https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects + // ... if the job has the JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag specified, + // closing the last job object handle terminates all associated processes + // and then destroys the job object itself. + // If a nested job has the JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag specified, + // closing the last job object handle terminates all processes associated + // with the job and its child jobs in the hierarchy. + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject( + h, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info))); err != nil { + return 0, err + } + + return Job(h), nil +} + +// Close closes job handler +func (job Job) Close() error { + if job == 0 { + return nil + } + return windows.CloseHandle(windows.Handle(job)) +} + +// Assign assigns the process to the JobObject +func (job Job) Assign(p *os.Process) error { + if job == 0 || p == nil { + return nil + } + return windows.AssignProcessToJobObject( + windows.Handle(job), + windows.Handle((*process)(unsafe.Pointer(p)).Handle)) +} + +type process struct { + Pid int + Handle uintptr +} diff --git a/x-pack/elastic-agent/main.go b/x-pack/elastic-agent/main.go index 170a77811f0b..82313532eaee 100644 --- a/x-pack/elastic-agent/main.go +++ b/x-pack/elastic-agent/main.go @@ -11,6 +11,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/cmd/platformcheck" + "github.com/elastic/beats/v7/libbeat/common/proc" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/cmd" ) @@ -21,6 +22,13 @@ func main() { os.Exit(1) } + pj, err := proc.CreateJobObject() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to initialize process job object: %v\n", err) + os.Exit(1) + } + defer pj.Close() + rand.Seed(time.Now().UnixNano()) command := cmd.NewCommand() if err := command.Execute(); err != nil { diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index 60792649da0d..8a20b00ebf3d 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -13,6 +13,8 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/beats/v7/libbeat/common/proc" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/app" @@ -134,6 +136,15 @@ func (a *Application) start(ctx context.Context, t app.Taggable, cfg map[string] if err != nil { return err } + + // Hook to JobObject on windows, noop on other platforms. + // This ties the application processes lifespan to the agent's. + // Fixes the orphaned beats processes left behind situation + // after the agent process gets killed. + if err := proc.JobObject.Assign(a.state.ProcessInfo.Process); err != nil { + a.logger.Errorf("application process failed job assign: %v", err) + } + // write connect info to stdin go a.writeToStdin(a.srvState, a.state.ProcessInfo.Stdin) diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd.go b/x-pack/osquerybeat/internal/osqd/osqueryd.go index 206846f985b8..0f38d28e082f 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd.go @@ -21,7 +21,9 @@ import ( "github.com/dolmen-go/contextio" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/proc" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fileutil" ) @@ -182,6 +184,13 @@ func (q *OSQueryD) Run(ctx context.Context, flags Flags) error { return err } + // Assign osqueryd process to the JobObject on windows + // in order to assure no orphan process is left behind + // after osquerybeat process is killed. + if err := proc.JobObject.Assign(cmd.Process); err != nil { + q.log.Errorf("osqueryd process failed job assign: %v", err) + } + var ( errbuf strings.Builder ) diff --git a/x-pack/osquerybeat/main.go b/x-pack/osquerybeat/main.go index 5889eb3ddbe2..53348ce8f564 100644 --- a/x-pack/osquerybeat/main.go +++ b/x-pack/osquerybeat/main.go @@ -5,14 +5,23 @@ package main import ( + "fmt" "os" "github.com/elastic/beats/v7/x-pack/osquerybeat/cmd" + "github.com/elastic/beats/v7/libbeat/common/proc" + _ "github.com/elastic/beats/v7/x-pack/osquerybeat/include" ) func main() { + pj, err := proc.CreateJobObject() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create process JobObject: %v\n", err) + os.Exit(1) + } + defer pj.Close() if err := cmd.RootCmd.Execute(); err != nil { os.Exit(1) } From b3bf12eed38eb372e59205952269b573c351139c Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 9 Feb 2022 09:26:20 -0500 Subject: [PATCH 09/52] [Automation] Update elastic stack version to 8.2.0-ff5ac1e4 for testing (#30299) Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot-oss.yml | 6 +++--- testing/environments/snapshot.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testing/environments/snapshot-oss.yml b/testing/environments/snapshot-oss.yml index 7be99e14b767..9893632c72da 100644 --- a/testing/environments/snapshot-oss.yml +++ b/testing/environments/snapshot-oss.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0-1cb1f3ad-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0-ff5ac1e4-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -21,7 +21,7 @@ services: - "script.context.template.cache_max_size=2000" logstash: - image: docker.elastic.co/logstash/logstash-oss:8.1.0-1cb1f3ad-SNAPSHOT + image: docker.elastic.co/logstash/logstash-oss:8.2.0-ff5ac1e4-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -31,7 +31,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.1.0-1cb1f3ad-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.2.0-ff5ac1e4-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 39891e1fe0fa..873d80ca619d 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0-1cb1f3ad-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0-ff5ac1e4-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.1.0-1cb1f3ad-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.2.0-ff5ac1e4-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From c2f51de2a14c1036d6079f332b6f5364bb4d18da Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Wed, 9 Feb 2022 15:45:40 +0100 Subject: [PATCH 10/52] Fix Kafka input for filebeat (#30277) The Kafka input was broken and had 2 issues: A serialization error on filebeat output: Looks like the ack function was put in the `reader.Message.Meta` map by mistake in 20d6038b8dd192c5239c7f78ffc076b18efe878b `Meta` is a `MapStr` type that does not support function values, therefore fails to serialize itself when requested later on output. `ack` was not called for incoming messages: The ack function was never used because it was supposed to be a part of the `beat.Event.Private` and it was not put in there. --- CHANGELOG.next.asciidoc | 1 + filebeat/input/kafka/input.go | 5 ++- .../input/kafka/kafka_integration_test.go | 39 ++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c3186a25a28a..a0c341eb49dd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - tcp/unix input: Stop accepting connections after socket is closed. {pull}29712[29712] - Fix using log_group_name_prefix in aws-cloudwatch input. {pull}29695[29695] - aws-s3: fix race condition in states used by s3-poller. {issue}30123[30123] {pull}30131[30131] +- Fix broken Kafka input {issue}29746[29746] {pull}30277[30277] *Heartbeat* diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 9c872f1a14b8..6440dcf9d2a2 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -318,6 +318,7 @@ func (h *groupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim s Timestamp: message.Ts, Meta: message.Meta, Fields: message.Fields, + Private: message.Private, }) } return nil @@ -458,8 +459,8 @@ func composeMessage(timestamp time.Time, content []byte, kafkaFields common.MapS "kafka": kafkaFields, "message": string(content), }, - Meta: common.MapStr{ - "ackHandler": ackHandler, + Private: eventMeta{ + ackHandler: ackHandler, }, } } diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index cb3f3526c99b..58b5c89ec2f2 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -62,6 +62,7 @@ func recordHeader(key, value string) sarama.RecordHeader { func TestInput(t *testing.T) { testTopic := createTestTopicName() + groupID := "filebeat" // Send test messages to the topic for the input to read. messages := []testMessage{ @@ -88,7 +89,7 @@ func TestInput(t *testing.T) { config := common.MustNewConfigFrom(common.MapStr{ "hosts": getTestKafkaHost(), "topics": []string{testTopic}, - "group_id": "filebeat", + "group_id": groupID, "wait_close": 0, }) @@ -113,6 +114,13 @@ func TestInput(t *testing.T) { assert.Equal(t, text, msg.message) checkMatchingHeaders(t, event, msg.headers) + + // emulating the pipeline (kafkaInput.Run) + meta, ok := event.Private.(eventMeta) + if !ok { + t.Fatal("could not get eventMeta and ack the message") + } + meta.ackHandler() case <-timeout: t.Fatal("timeout waiting for incoming events") } @@ -132,6 +140,8 @@ func TestInput(t *testing.T) { t.Fatal("timeout waiting for beat to shut down") case <-didClose: } + + assertOffset(t, groupID, testTopic, int64(len(messages))) } func TestInputWithMultipleEvents(t *testing.T) { @@ -420,6 +430,33 @@ func getTestKafkaHost() string { ) } +func assertOffset(t *testing.T, groupID, topic string, expected int64) { + client, err := sarama.NewClient([]string{getTestKafkaHost()}, nil) + assert.NoError(t, err) + defer client.Close() + + ofm, err := sarama.NewOffsetManagerFromClient(groupID, client) + assert.NoError(t, err) + defer ofm.Close() + + partitions, err := client.Partitions(topic) + assert.NoError(t, err) + + var offsetSum int64 + + for _, partitionID := range partitions { + pom, err := ofm.ManagePartition(topic, partitionID) + assert.NoError(t, err) + + offset, _ := pom.NextOffset() + offsetSum += offset + + pom.Close() + } + + assert.Equal(t, expected, offsetSum, "offset does not match, perhaps messages were not acknowledged") +} + func writeToKafkaTopic( t *testing.T, topic string, message string, headers []sarama.RecordHeader, timeout time.Duration, From c8c49ba1751d0f89840dbcc963168abdc44bc71e Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 9 Feb 2022 09:49:11 -0500 Subject: [PATCH 11/52] Remove unused code for fakeweb (#30198) --- .../dev-tools/cmd/fakewebapi/README.md | 13 -- .../cmd/fakewebapi/action_example.json | 38 ---- .../dev-tools/cmd/fakewebapi/checkin.json | 4 - .../dev-tools/cmd/fakewebapi/fetch.sh | 5 - .../dev-tools/cmd/fakewebapi/main.go | 163 ------------------ .../dev-tools/cmd/fakewebapi/push.sh | 3 - 6 files changed, 226 deletions(-) delete mode 100644 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/README.md delete mode 100644 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json delete mode 100644 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/checkin.json delete mode 100755 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/fetch.sh delete mode 100644 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go delete mode 100755 x-pack/elastic-agent/dev-tools/cmd/fakewebapi/push.sh diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/README.md b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/README.md deleted file mode 100644 index 43def1b4cc7c..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Fakewebapi is a simple test only Webserver - -The server implements the minimal calls and response to do high level testing of the elastic-agent: - -- Enroll successfully an Agent. -- Allow an Agent to periodically check in. - - -By default the server will return an empty list of actions, it's possible at runtime to change the returned -data by using the `push.sh` script. The script will POST a JSON document to return on the next request. - -Read the code of `push.sh` and the `fetch.sh` script for the usage information. - diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json deleted file mode 100644 index f08e98942ea1..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "action": "checkin", - "actions": [ - { - "type": "POLICY_CHANGE", - "data": { - "policy": { - "id": "default", - "outputs": { - "default": { - "id": "default", - "name": "default", - "type": "elasticsearch", - "hosts": "http://localhost:9200", - "ingest_pipeline": "default" - } - }, - "streams": [ - { - "type": "metric/nginx", - "metricsets": [ - "stubstatus" - ], - "period": "10s", - "enabled": true, - "hosts": "http://127.0.0.1", - "id": "stubstatus", - "output": { - "use_output": "default" - } - } - ] - } - }, - "id": "6d22f55a-d6e8-4e52-bcaa-16dde6091c5c" - } - ] -} diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/checkin.json b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/checkin.json deleted file mode 100644 index 7a7ecd4e2d1a..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/checkin.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "events": [] -} - diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/fetch.sh b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/fetch.sh deleted file mode 100755 index c84d305081c4..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/fetch.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -APIKEY=${1:-"abc123"} -AGENTID=${2:-"agent007"} -FILE=${3:-"checkin.json"} -curl -H "Authorization: ApiKey $APIKEY" -X POST --data "@$FILE" http://localhost:8080/api/fleet/agents/$AGENTID/checkin diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go deleted file mode 100644 index 11d90d13773e..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package main - -import ( - "bytes" - "encoding/json" - "flag" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "regexp" - "strings" - "sync" - "time" - - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" -) - -var ( - host string - apiKey string - mutex sync.Mutex - - pathCheckin = regexp.MustCompile(`^/api/fleet/agents/(.+)/checkin`) - checkinResponse = response{Actions: make([]action, 0)} -) - -type response struct { - Actions []action `json:"actions"` -} - -type action interface{} - -func init() { - flag.StringVar(&apiKey, "apikey", "abc123", "API Key to authenticate") - flag.StringVar(&host, "host", "localhost:8080", "The IP and port to use for the webserver") -} - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/api/fleet/agents/enroll", handlerEnroll) - mux.HandleFunc("/admin/actions", handlerAction) - mux.HandleFunc("/", handlerRoot) - - log.Printf("Starting webserver and listening on %s", host) - - listener, err := net.Listen("tcp", host) - if err != nil { - panic(err) - } - defer listener.Close() - - http.Serve(listener, mux) -} - -func handlerRoot(w http.ResponseWriter, r *http.Request) { - if pathCheckin.MatchString(r.URL.Path) { - authHandler(handlerCheckin, apiKey)(w, r) - return - } - - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{ "message": "Hello!"}`)) - log.Println("Root hello!") - log.Println("Path: ", r.URL.Path) -} - -func handlerEnroll(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } - - response := &fleetapi.EnrollResponse{ - Action: "created", - Item: fleetapi.EnrollItemResponse{ - ID: "a4937110-e53e-11e9-934f-47a8e38a522c", - Active: true, - PolicyID: "default", - Type: fleetapi.PermanentEnroll, - EnrolledAt: time.Now(), - UserProvidedMetadata: make(map[string]interface{}), - LocalMetadata: make(map[string]interface{}), - AccessAPIKey: apiKey, - }, - } - - b, err := json.Marshal(response) - if err != nil { - log.Printf("failed to enroll: %+v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusOK) - w.Write(b) - log.Println("Enroll response:", string(b)) -} - -func handlerCheckin(w http.ResponseWriter, r *http.Request) { - mutex.Lock() - defer mutex.Unlock() - - b, err := json.Marshal(checkinResponse) - if err != nil { - log.Printf("Failed to checkin, error: %+v", err) - http.Error(w, "Internal Server error", http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) - w.Write(b) - log.Println("Checkin response: ", string(b)) -} - -func handlerAction(w http.ResponseWriter, r *http.Request) { - mutex.Lock() - defer mutex.Unlock() - if r.Method != "POST" { - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } - - resp := response{} - - var buf bytes.Buffer - tee := io.TeeReader(r.Body, &buf) - - c, err := ioutil.ReadAll(tee) - if err != nil { - log.Printf("Fails to update the actions") - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - decoder := json.NewDecoder(&buf) - err = decoder.Decode(&resp) - if err != nil { - log.Printf("Fails to update the actions") - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - checkinResponse = resp - w.WriteHeader(http.StatusCreated) - log.Println("Action request: ", string(c)) -} - -func authHandler(handler http.HandlerFunc, apiKey string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - const key = "Authorization" - const prefix = "ApiKey " - - v := strings.TrimPrefix(r.Header.Get(key), prefix) - if v != apiKey { - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - handler(w, r) - } -} diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/push.sh b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/push.sh deleted file mode 100755 index 602df720cb11..000000000000 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/push.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -FILE=${1:-"action_example.json"} -curl -X POST --data "@$FILE" http://localhost:8080/admin/actions From c0a76b46a116e82213b4da35b26fae34510e4cfc Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:49:47 +0100 Subject: [PATCH 12/52] Add github action for automating QA labels (#30303) --- .github/workflows/qa-labels.yml | 82 +++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/qa-labels.yml diff --git a/.github/workflows/qa-labels.yml b/.github/workflows/qa-labels.yml new file mode 100644 index 000000000000..dc49c8e520e0 --- /dev/null +++ b/.github/workflows/qa-labels.yml @@ -0,0 +1,82 @@ +name: Add QA labels to Elastic Agent issues +on: + pull_request: + types: + - closed + +jobs: + fetch_issues_to_label: + runs-on: ubuntu-latest + # Only run on PRs that were merged for the Elastic Agent teams + if: | + github.event.pull_request.merged_at && + ( + contains(github.event.pull_request.labels.*.name, 'Team:Elastic-Agent') || + contains(github.event.pull_request.labels.*.name, 'Team:Elastic-Agent-Data-Plane') || + contains(github.event.pull_request.labels.*.name, 'Team:Elastic-Agent-Control-Plane') + ) + outputs: + matrix: ${{ steps.issues_to_label.outputs.value }} + label_ids: ${{ steps.label_ids.outputs.value }} + steps: + - uses: octokit/graphql-action@v2.x + id: closing_issues + with: + query: | + query closingIssueNumbersQuery($prnumber: Int!) { + repository(owner: "elastic", name: "beats") { + pullRequest(number: $prnumber) { + closingIssuesReferences(first: 10) { + nodes { + id + labels(first: 20) { + nodes { + id + name + } + } + } + } + } + } + } + prnumber: ${{ github.event.number }} + token: ${{ secrets.GITHUB_TOKEN }} + - uses: sergeysova/jq-action@v2 + id: issues_to_label + with: + # Map to the issues' node id + cmd: echo $CLOSING_ISSUES | jq -c '.repository.pullRequest.closingIssuesReferences.nodes | map(.id)' + multiline: true + env: + CLOSING_ISSUES: ${{ steps.closing_issues.outputs.data }} + - uses: sergeysova/jq-action@v2 + id: label_ids + with: + # Get list of version labels on pull request and map to label's node id, append 'QA:Ready For Testing' id ("LA_kwDOAPya887jWb9x") + cmd: echo $PR_LABELS | jq -c 'map(select(.name | test("v[0-9]+\\.[0-9]+\\.[0-9]+")) | .node_id) + ["LA_kwDOAPya887jWb9x"]' + multiline: true + env: + PR_LABELS: ${{ toJSON(github.event.pull_request.labels) }} + + label_issues: + needs: fetch_issues_to_label + runs-on: ubuntu-latest + # For each issue closed by the PR run this job + strategy: + matrix: + issueNodeId: ${{ fromJSON(needs.fetch_issues_to_label.outputs.matrix) }} + name: Label issue ${{ matrix.issueNodeId }} + steps: + - uses: octokit/graphql-action@v2.x + id: add_labels_to_closed_issue + with: + query: | + mutation add_label($issueid:String!, $labelids:[String!]!) { + addLabelsToLabelable(input: {labelableId: $issueid, labelIds: $labelids}) { + clientMutationId + } + } + issueid: ${{ matrix.issueNodeId }} + labelids: ${{ needs.fetch_issues_to_label.outputs.label_ids }} + token: ${{ secrets.GITHUB_TOKEN }} From a1bf996c72bc51ab4fba22a40e2acaffd21f6176 Mon Sep 17 00:00:00 2001 From: Florian Lehner Date: Wed, 9 Feb 2022 19:20:02 +0100 Subject: [PATCH 13/52] mage: add -trimpath flag to build process (#30141) --- dev-tools/mage/build.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 54d1b9f5c181..e93547d51c07 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -70,6 +70,8 @@ func DefaultBuildArgs() BuildArgs { } else { // Strip all debug symbols from binary (does not affect Go stack traces). args.LDFlags = append(args.LDFlags, "-s") + // Remove all file system paths from the compiled executable, to improve build reproducibility + args.ExtraFlags = append(args.ExtraFlags, "-trimpath") } return args From 5bf9a93b59a923b933f2168f28c033f96c695e5a Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 9 Feb 2022 20:26:16 +0100 Subject: [PATCH 14/52] filebeat/input/httpjson: drain and close response.Body (#30243) httpClient.do now reads the whole response.Body into memory and replaces it with the in-memory copy. This brings some advantages: * We can easily ensure the response.Body is always closed * We read the response as quick as possible, releasing the connection to be reused and avoiding possible timeouts or other errors caused by keeping the connection open and blocked. It also replaces `ioutil.ReadAll` by `io.ReadAll` as `ioutil` has been deprecated. --- x-pack/filebeat/input/httpjson/request.go | 38 +++++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 0c5fbc39fa2f..a15f9eb03c1d 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -8,7 +8,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" "net/http" "net/url" @@ -37,9 +37,17 @@ func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *ht if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) } + defer resp.Body.Close() + + // Read the whole resp.Body so we can release the conneciton. + // This implementaion is inspired by httputil.DumpResponse + resp.Body, err = drainBody(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + if resp.StatusCode > 399 { - body, _ := ioutil.ReadAll(resp.Body) - resp.Body.Close() + body, _ := io.ReadAll(resp.Body) return nil, fmt.Errorf("server responded with status code %d: %s", resp.StatusCode, string(body)) } return resp, nil @@ -211,3 +219,27 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p return nil } + +// drainBody reads all of b to memory and then returns a equivalent +// ReadCloser yielding the same bytes. +// +// It returns an error if the initial slurp of all bytes fails. It does not attempt +// to make the returned ReadCloser have identical error-matching behavior. +// +// This function is a modified version of drainBody from the http/httputil package. +func drainBody(b io.ReadCloser) (r1 io.ReadCloser, err error) { + if b == nil || b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, nil + } + + var buf bytes.Buffer + if _, err = buf.ReadFrom(b); err != nil { + return b, err + } + if err = b.Close(); err != nil { + return b, err + } + + return io.NopCloser(&buf), nil +} From 4b480bd04028b084986acd5bdea4919305c86076 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Wed, 9 Feb 2022 13:25:13 -0800 Subject: [PATCH 15/52] Remove support for Homebrew from the docs (#30311) --- .../docs/howto/load-index-templates.asciidoc | 12 ----- libbeat/docs/shared-brew-install.asciidoc | 21 -------- libbeat/docs/shared-brew-run.asciidoc | 53 ------------------- libbeat/docs/shared-directory-layout.asciidoc | 13 ----- .../enable-modules-widget.asciidoc | 18 ------- .../docs/tab-widgets/enable-modules.asciidoc | 7 --- .../install-widget-filebeat.asciidoc | 18 ------- .../install-widget-heartbeat.asciidoc | 18 ------- .../install-widget-metricbeat.asciidoc | 18 ------- .../docs/tab-widgets/install-widget.asciidoc | 18 ------- libbeat/docs/tab-widgets/install.asciidoc | 22 -------- .../tab-widgets/list-modules-widget.asciidoc | 18 ------- .../docs/tab-widgets/list-modules.asciidoc | 7 --- .../load-dashboards-logstash-widget.asciidoc | 18 ------- .../load-dashboards-logstash.asciidoc | 12 ----- .../load-dashboards-widget.asciidoc | 18 ------- .../docs/tab-widgets/load-dashboards.asciidoc | 7 --- .../docs/tab-widgets/setup-widget.asciidoc | 18 ------- libbeat/docs/tab-widgets/setup.asciidoc | 7 --- .../docs/tab-widgets/start-filebeat.asciidoc | 19 ------- .../start-widget-filebeat.asciidoc | 18 ------- .../start-widget-heartbeat.asciidoc | 18 ------- .../start-widget-metricbeat.asciidoc | 18 ------- .../docs/tab-widgets/start-widget.asciidoc | 18 ------- libbeat/docs/tab-widgets/start.asciidoc | 52 ------------------ packetbeat/docs/gettingstarted.asciidoc | 4 -- .../docs/tab-widgets/devices-widget.asciidoc | 18 ------- packetbeat/docs/tab-widgets/devices.asciidoc | 7 --- .../install-libpcap-widget.asciidoc | 18 ------- .../docs/tab-widgets/install-libpcap.asciidoc | 4 -- 30 files changed, 517 deletions(-) delete mode 100644 libbeat/docs/shared-brew-install.asciidoc delete mode 100644 libbeat/docs/shared-brew-run.asciidoc diff --git a/libbeat/docs/howto/load-index-templates.asciidoc b/libbeat/docs/howto/load-index-templates.asciidoc index 5225e228f910..ef7e0d1cf81f 100644 --- a/libbeat/docs/howto/load-index-templates.asciidoc +++ b/libbeat/docs/howto/load-index-templates.asciidoc @@ -133,12 +133,6 @@ ifdef::mac_os[] ./{beatname_lc} setup --index-management{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---- -*brew:* - -["source","sh",subs="attributes"] ----- -{beatname_lc} setup --index-management{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ----- endif::mac_os[] ifdef::linux_os[] @@ -257,12 +251,6 @@ ifdef::mac_os[] ./{beatname_lc} export template > {beatname_lc}.template.json ---- -*brew:* - -["source","sh",subs="attributes"] ----- -{beatname_lc} export template > {beatname_lc}.template.json ----- endif::mac_os[] ifdef::linux_os[] diff --git a/libbeat/docs/shared-brew-install.asciidoc b/libbeat/docs/shared-brew-install.asciidoc deleted file mode 100644 index 0b11bb854f7c..000000000000 --- a/libbeat/docs/shared-brew-install.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -[[brew]] -*brew:* - -ifeval::["{release-state}"=="unreleased"] - -Version {version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -["source","sh",subs="attributes"] -------------------------- -brew tap elastic/tap -brew install elastic/tap/{beatname_lc}-full -------------------------- - -This installs the most recently released default distribution of {beatname_uc}. -To install the OSS distribution, specify +elastic/tap/{beatname_lc}-oss+. - -endif::[] diff --git a/libbeat/docs/shared-brew-run.asciidoc b/libbeat/docs/shared-brew-run.asciidoc deleted file mode 100644 index 92b686d74174..000000000000 --- a/libbeat/docs/shared-brew-run.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -*brew:* - -To have launchd start +elastic/tap/{beatname_lc}+ and then restart it at login, -run: - -["source","sh",subs="attributes"] ------ -brew services start elastic/tap/{beatname_lc}-full ------ - -The launchd service is configured with `Umask=0027` which means the most permissive mask allowed for files created by {beatname_uc} is `0640`. All configured file permissions higher than `0640` will be ignored. Please edit the service file manually in case you need to change that. - -ifndef::requires-sudo[] -To run {beatname_uc} in the foreground instead of running it as a background -service, run: - -["source","sh",subs="attributes"] ------ -{beatname_lc} -e ------ -endif::[] - -ifdef::requires-sudo[] -To run {beatname_uc} in the foreground instead of running it as a background -service, run: - -ifndef::has_modules_command[] -["source","sh",subs="attributes"] ------ -sudo chown root /usr/local/etc/{beatname_lc}/{beatname_lc}.yml <1> -sudo {beatname_lc} -e ------ -<1> You'll be running {beatname_uc} as root, so you need to change ownership -of the configuration file, or run {beatname_uc} with `--strict.perms=false` -specified. See -{beats-ref}/config-file-permissions.html[Config File Ownership and Permissions]. -endif::has_modules_command[] - -ifdef::has_modules_command[] -["source","sh",subs="attributes,callouts"] ----------------------------------------------------------------------- -sudo chown root /usr/local/etc/{beatname_lc}/{beatname_lc}.yml <1> -sudo chown root /usr/local/etc/{beatname_lc}/modules.d/system.yml <1> -sudo {beatname_lc} -e ----------------------------------------------------------------------- -<1> You'll be running {beatname_uc} as root, so you need to change ownership of the -configuration file and any configurations enabled in the `modules.d` directory, -or run {beatname_uc} with `--strict.perms=false` specified. See -{beats-ref}/config-file-permissions.html[Config File Ownership and Permissions]. - -endif::has_modules_command[] - -endif::requires-sudo[] diff --git a/libbeat/docs/shared-directory-layout.asciidoc b/libbeat/docs/shared-directory-layout.asciidoc index 83b5c44cdde1..e9375833213c 100644 --- a/libbeat/docs/shared-directory-layout.asciidoc +++ b/libbeat/docs/shared-directory-layout.asciidoc @@ -99,19 +99,6 @@ ifndef::win_only[] endif::win_only[] -ifdef::mac_os[] -===== brew -[cols=" MacOS - - - - - - - - - - - - - - -