diff --git a/internal/common/helpers.go b/internal/common/helpers.go index 5d5b16351..023d527cd 100644 --- a/internal/common/helpers.go +++ b/internal/common/helpers.go @@ -22,3 +22,17 @@ func StringSliceContains(slice []string, s string) bool { } return false } + +// StringSlicesUnion joins multiple slices and returns an slice with the distinct +// elements of all of them. +func StringSlicesUnion(slices ...[]string) (result []string) { + for _, slice := range slices { + for _, elem := range slice { + if StringSliceContains(result, elem) { + continue + } + result = append(result, elem) + } + } + return +} diff --git a/internal/common/helpers_test.go b/internal/common/helpers_test.go index dbcb1553d..1474419b0 100644 --- a/internal/common/helpers_test.go +++ b/internal/common/helpers_test.go @@ -39,3 +39,22 @@ func TestStringSliceContains(t *testing.T) { assert.Equalf(t, c.expected, found, "checking if slice %v contains '%s'", c.slice, c.s) } } + +func TestStringSlicesUnion(t *testing.T) { + cases := []struct { + slices [][]string + expected []string + }{ + {nil, nil}, + {[][]string{{"foo", "bar"}, nil}, []string{"foo", "bar"}}, + {[][]string{nil, {"foo", "bar"}}, []string{"foo", "bar"}}, + {[][]string{{"foo", "bar"}, {"foo", "bar"}}, []string{"foo", "bar"}}, + {[][]string{{"foo", "baz"}, {"foo", "bar"}}, []string{"foo", "bar", "baz"}}, + {[][]string{{"foo", "bar"}, {"foo", "baz"}}, []string{"foo", "bar", "baz"}}, + } + + for _, c := range cases { + result := StringSlicesUnion(c.slices...) + assert.ElementsMatch(t, c.expected, result) + } +} diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 934733578..42831fa12 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -38,7 +38,7 @@ var ( dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent") ) -//LocationManager maintains an instance of a config path location +// LocationManager maintains an instance of a config path location type LocationManager struct { stackPath string } diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index 14e856497..16a945200 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -262,6 +262,10 @@ func transformImportedField(fd FieldDefinition) common.MapStr { m["doc_values"] = *fd.DocValues } + if len(fd.Normalize) > 0 { + m["normalize"] = fd.Normalize + } + if len(fd.MultiFields) > 0 { var t []common.MapStr for _, f := range fd.MultiFields { diff --git a/internal/fields/dependency_manager_test.go b/internal/fields/dependency_manager_test.go index 52c42a6b3..80cf91392 100644 --- a/internal/fields/dependency_manager_test.go +++ b/internal/fields/dependency_manager_test.go @@ -200,6 +200,51 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) { changed: true, valid: true, }, + { + title: "array field", + defs: []common.MapStr{ + { + "name": "host.ip", + "external": "test", + }, + }, + result: []common.MapStr{ + { + "name": "host.ip", + "type": "ip", + "description": "Host ip addresses.", + "normalize": []string{ + "array", + }, + }, + }, + changed: true, + valid: true, + }, + { + title: "array field override", + defs: []common.MapStr{ + { + "name": "container.id", + "external": "test", + "normalize": []string{ + "array", + }, + }, + }, + result: []common.MapStr{ + { + "name": "container.id", + "type": "keyword", + "description": "Container identifier.", + "normalize": []string{ + "array", + }, + }, + }, + changed: true, + valid: true, + }, { title: "unknown field", defs: []common.MapStr{ @@ -335,6 +380,14 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) { Index: &indexFalse, DocValues: &indexFalse, }, + { + Name: "host.ip", + Description: "Host ip addresses.", + Type: "ip", + Normalize: []string{ + "array", + }, + }, { Name: "source.mac", Description: "MAC address of the source.", diff --git a/internal/fields/model.go b/internal/fields/model.go index cedc3b54a..20823224a 100644 --- a/internal/fields/model.go +++ b/internal/fields/model.go @@ -28,6 +28,7 @@ type FieldDefinition struct { External string `yaml:"external"` Index *bool `yaml:"index"` DocValues *bool `yaml:"doc_values"` + Normalize []string `yaml:"normalize,omitempty"` Fields FieldDefinitions `yaml:"fields,omitempty"` MultiFields []FieldDefinition `yaml:"multi_fields,omitempty"` } @@ -73,6 +74,10 @@ func (orig *FieldDefinition) Update(fd FieldDefinition) { orig.DocValues = fd.DocValues } + if len(fd.Normalize) > 0 { + orig.Normalize = common.StringSlicesUnion(orig.Normalize, fd.Normalize) + } + if len(fd.Fields) > 0 { orig.Fields = updateFields(orig.Fields, fd.Fields) } diff --git a/internal/fields/testdata/fields/fields.yml b/internal/fields/testdata/fields/fields.yml index 77f5d66e3..b78ddc655 100644 --- a/internal/fields/testdata/fields/fields.yml +++ b/internal/fields/testdata/fields/fields.yml @@ -13,3 +13,7 @@ value: correct - name: ip_address type: ip +- name: container.image.tag + type: keyword + normalize: + - array diff --git a/internal/fields/testdata/invalid-array-normalization.json b/internal/fields/testdata/invalid-array-normalization.json new file mode 100644 index 000000000..853599529 --- /dev/null +++ b/internal/fields/testdata/invalid-array-normalization.json @@ -0,0 +1,3 @@ +{ + "container.image.tag": "sometag" +} diff --git a/internal/fields/testdata/valid-array-normalization.json b/internal/fields/testdata/valid-array-normalization.json new file mode 100644 index 000000000..5d0fb4336 --- /dev/null +++ b/internal/fields/testdata/valid-array-normalization.json @@ -0,0 +1,3 @@ +{ + "container.image.tag": ["sometag"] +} diff --git a/internal/fields/validate.go b/internal/fields/validate.go index f672b6603..97d7c769a 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -15,6 +15,7 @@ import ( "regexp" "strings" + "github.com/Masterminds/semver" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -32,6 +33,9 @@ type Validator struct { // FieldDependencyManager resolves references to external fields FieldDependencyManager *DependencyManager + // SpecVersion contains the version of the spec used by the package. + specVersion semver.Version + defaultNumericConversion bool numericKeywordFields map[string]struct{} @@ -44,6 +48,18 @@ type Validator struct { // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDirectory. type ValidatorOption func(*Validator) error +// WithSpecVersion enables validation dependant of the spec version used by the package. +func WithSpecVersion(version string) ValidatorOption { + return func(v *Validator) error { + sv, err := semver.NewVersion(version) + if err != nil { + return fmt.Errorf("invalid version %q: %v", version, err) + } + v.specVersion = *sv + return nil + } +} + // WithDefaultNumericConversion configures the validator to accept defined keyword (or constant_keyword) fields as numeric-type. func WithDefaultNumericConversion() ValidatorOption { return func(v *Validator) error { @@ -255,7 +271,12 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error { val = fmt.Sprintf("%q", val) } - err := v.parseElementValue(key, *definition, val) + err := v.validateExpectedNormalization(*definition, val) + if err != nil { + return errors.Wrapf(err, "field %q is not normalized as expected", key) + } + + err = v.parseElementValue(key, *definition, val) if err != nil { return errors.Wrap(err, "parsing field value failed") } @@ -361,6 +382,22 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool { return false } +func (v *Validator) validateExpectedNormalization(definition FieldDefinition, val interface{}) error { + // Validate expected normalization starting with packages following spec v2 format. + if v.specVersion.LessThan(semver.MustParse("2.0.0")) { + return nil + } + for _, normalize := range definition.Normalize { + switch normalize { + case "array": + if _, isArray := val.([]interface{}); val != nil && !isArray { + return fmt.Errorf("expected array, found %q (%T)", val, val) + } + } + } + return nil +} + // validSubField checks if the extra part that didn't match with any field definition, // matches with the possible sub field of complex fields like geo_point or histogram. func validSubField(def FieldDefinition, extraPart string) bool { diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index dc3a53978..b1daa2327 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -91,6 +91,28 @@ func TestValidate_ipAddress(t *testing.T) { require.Empty(t, errs) } +func TestValidate_WithSpecVersion(t *testing.T) { + validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0")) + require.NoError(t, err) + + e := readSampleEvent(t, "testdata/invalid-array-normalization.json") + errs := validator.ValidateDocumentBody(e) + require.Len(t, errs, 1) + require.Contains(t, errs[0].Error(), `field "container.image.tag" is not normalized as expected`) + + e = readSampleEvent(t, "testdata/valid-array-normalization.json") + errs = validator.ValidateDocumentBody(e) + require.Empty(t, errs) + + // Check now that this validation was only enabled for 2.0.0. + validator, err = CreateValidatorForDirectory("testdata", WithSpecVersion("1.99.99")) + require.NoError(t, err) + + e = readSampleEvent(t, "testdata/invalid-array-normalization.json") + errs = validator.ValidateDocumentBody(e) + require.Empty(t, errs) +} + func Test_parseElementValue(t *testing.T) { for _, test := range []struct { key string diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 86831da5e..ae5ecf032 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -111,6 +111,7 @@ type Owner struct { // PackageManifest represents the basic structure of a package's manifest type PackageManifest struct { + SpecVersion string `config:"format_version" json:"format_version" yaml:"format_version"` Name string `config:"name" json:"name" yaml:"name"` Title string `config:"title" json:"title" yaml:"title"` Type string `config:"type" json:"type" yaml:"type"` diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index cdd5045f8..e3bb72d90 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -95,6 +95,11 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil, errors.Wrap(err, "installing ingest pipelines failed") } + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return nil, errors.Wrap(err, "failed to read manifest") + } + results := make([]testrunner.TestResult, 0) for _, testCaseFile := range testCaseFiles { tr := testrunner.TestResult{ @@ -135,6 +140,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { tr.TimeElapsed = time.Since(startTime) fieldsValidator, err := fields.CreateValidatorForDirectory(dataStreamPath, + fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithNumericKeywordFields(tc.config.NumericKeywordFields), // explicitly enabled for pipeline tests only // since system tests can have dynamic public IPs diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index 418c152ba..3361bd2a3 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/elastic-package/internal/fields" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -64,12 +65,17 @@ func (r runner) run() ([]testrunner.TestResult, error) { return result.WithSkip(testConfig.Skip) } + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return result.WithError(errors.Wrap(err, "failed to read manifest")) + } + var results []testrunner.TestResult - results = append(results, r.verifySampleEvent()...) + results = append(results, r.verifySampleEvent(pkgManifest)...) return results, nil } -func (r runner) verifySampleEvent() []testrunner.TestResult { +func (r runner) verifySampleEvent(pkgManifest *packages.PackageManifest) []testrunner.TestResult { dataStreamPath := filepath.Join(r.options.PackageRootPath, "data_stream", r.options.TestFolder.DataStream) sampleEventPath := filepath.Join(dataStreamPath, sampleEventJSON) _, err := os.Stat(sampleEventPath) @@ -89,8 +95,8 @@ func (r runner) verifySampleEvent() []testrunner.TestResult { return results } - fieldsValidator, err := fields.CreateValidatorForDirectory( - dataStreamPath, + fieldsValidator, err := fields.CreateValidatorForDirectory(dataStreamPath, + fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithDefaultNumericConversion()) if err != nil { results, _ := resultComposer.WithError(errors.Wrap(err, "creating fields validator for data stream failed")) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 27356a0cc..f271de8a7 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -448,6 +448,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // Validate fields in docs fieldsValidator, err := fields.CreateValidatorForDirectory(serviceOptions.DataStreamRootPath, + fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithNumericKeywordFields(config.NumericKeywordFields)) if err != nil { return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath)) diff --git a/test/packages/parallel/apache/_dev/build/build.yml b/test/packages/parallel/apache/_dev/build/build.yml index 08d85edcf..6b5cc3fbc 100644 --- a/test/packages/parallel/apache/_dev/build/build.yml +++ b/test/packages/parallel/apache/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.12 + reference: git@8.1 diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 12b25eb19..80604eeeb 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -1,114 +1,128 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614845014Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T10:30:19.171259100Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614847398Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:19.171272300Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, - "url": { - "path": "/hello", - "original": "/hello" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614848507Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 @@ -118,143 +132,149 @@ "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T10:30:19.171276600Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": {} }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614849470Z", + "kind": "event", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "172.17.0.1", "ip": "172.17.0.1" }, - "event": { - "ingested": "2021-12-14T10:30:19.171281Z", - "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/stringpatch", + "path": "/stringpatch" }, "user": { "name": "-" }, - "url": { - "path": "/stringpatch", - "original": "/stringpatch" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": {} }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614850424Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, - "event": { - "ingested": "2021-12-14T10:30:19.171285300Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/status", + "path": "/status" }, "user": { "name": "-" }, - "url": { - "path": "/status", - "original": "/status" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { "access": {} }, - "@timestamp": "2019-02-02T04:38:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614854678Z", + "kind": "event", + "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "referrer": "-" @@ -270,85 +290,79 @@ "address": "127.0.0.1", "ip": "127.0.0.1" }, - "event": { - "ingested": "2021-12-14T10:30:19.171289700Z", - "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, + "tags": [ + "preserve_original_event" + ], "user": { "name": "-" }, "user_agent": { - "name": "Other", "device": { "name": "Other" }, + "name": "Other", "original": "-" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": {} }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.614855729Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, - "event": { - "ingested": "2021-12-14T10:30:19.171328400Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" }, "user": { "name": "-" }, - "url": { - "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", - "extension": "mp4", - "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 1094c79d0..173c06e28 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -1,100 +1,114 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T14:16:28.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659642334Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T10:30:20.126148600Z", - "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659644823Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T10:30:20.126162400Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659645835Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 @@ -104,203 +118,201 @@ "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T10:30:20.126170700Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:23:35.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659646801Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T10:30:20.126178600Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:23:41.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659647759Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 206 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T10:30:20.126186500Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/notfound", + "path": "/notfound" }, "user": { "name": "-" - }, - "url": { - "path": "/notfound", - "original": "/notfound" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:23:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.659648717Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 201 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T10:30:20.126194300Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hmm", + "path": "/hmm" }, "user": { "name": "-" - }, - "url": { - "path": "/hmm", - "original": "/hmm" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 3f8442649..4b56cde97 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -1,6 +1,7 @@ { "expected": [ { + "@timestamp": "2018-08-10T07:45:56.000Z", "apache": { "access": { "ssl": { @@ -9,48 +10,50 @@ } } }, - "@timestamp": "2018-08-10T07:45:56.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.704516601Z", + "kind": "event", + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 1375 } - } - }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" + }, + "version": "1.1" }, "source": { "address": "172.30.0.119", "ip": "172.30.0.119" }, - "event": { - "ingested": "2021-12-14T10:30:20.756861200Z", - "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/nagiosxi/ajaxhelper.php", "extension": "php", "original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "path": "/nagiosxi/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-10-16T09:53:47.000Z", "apache": { "access": { "ssl": { @@ -59,59 +62,60 @@ } } }, - "@timestamp": "2019-10-16T09:53:47.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.704520327Z", + "kind": "event", + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" + }, "http": { "request": { "method": "GET" }, "version": "1.1" }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" - }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T10:30:20.756875500Z", - "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/appl/ajaxhelper.php", "extension": "php", "original": "/appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "path": "/appl/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 77a81ce00..d1f392cd6 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -1,518 +1,536 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:18:09.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762449415Z", + "kind": "event", + "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 491 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081782700Z", - "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, - "url": { - "path": "/", - "original": "/" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Wget", "original": "Wget/1.13.4 (linux-gnu)", "os": { "name": "Linux" }, - "device": { - "name": "Other" - }, "version": "1.13.4" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762453903Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081797300Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, - "url": { - "path": "/", - "original": "/" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762455848Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "http://192.168.33.72/" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081805900Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762457571Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081814Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, - "url": { - "path": "/", - "original": "/" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762459219Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081822Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762460903Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081829900Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:10.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762462605Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 498 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081837900Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/test", + "path": "/test" }, "user": { "name": "-" }, - "url": { - "path": "/test", - "original": "/test" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762464295Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081845900Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, - "url": { - "path": "/hello", - "original": "/hello" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:17.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.762465914Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:21.081853800Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/crap", + "path": "/crap" }, "user": { "name": "-" }, - "url": { - "path": "/crap", - "original": "/crap" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 3f778d516..120247e44 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -1,61 +1,63 @@ { "expected": [ { - "destination": { - "domain": "vhost1.domaine.fr" - }, - "source": { - "ip": "192.168.33.2" - }, - "url": { - "path": "/hello", - "original": "/hello", - "domain": "vhost1.domaine.fr" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:14.000Z", "apache": { "access": {} }, - "@timestamp": "2016-12-26T16:22:14.000Z", + "destination": { + "domain": "vhost1.domaine.fr" + }, "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-09-06T07:11:46.823078522Z", + "kind": "event", + "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-14T10:30:22.626765300Z", - "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "ip": "192.168.33.2" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "vhost1.domaine.fr", + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } diff --git a/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index dadfb3a49..38d5c041d 100644 --- a/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -46,7 +46,7 @@ processors: value: event - set: field: event.category - value: web + value: ["web"] - set: field: event.outcome value: success diff --git a/test/packages/parallel/apache/data_stream/access/fields/agent.yml b/test/packages/parallel/apache/data_stream/access/fields/agent.yml index e313ec828..3c8ad89f0 100644 --- a/test/packages/parallel/apache/data_stream/access/fields/agent.yml +++ b/test/packages/parallel/apache/data_stream/access/fields/agent.yml @@ -121,10 +121,6 @@ As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - name: mac level: core type: keyword diff --git a/test/packages/parallel/apache/data_stream/access/fields/ecs.yml b/test/packages/parallel/apache/data_stream/access/fields/ecs.yml index a0e827823..db3a3a47c 100644 --- a/test/packages/parallel/apache/data_stream/access/fields/ecs.yml +++ b/test/packages/parallel/apache/data_stream/access/fields/ecs.yml @@ -98,3 +98,5 @@ name: user_agent.os.version - external: ecs name: user_agent.version +- external: ecs + name: host.ip diff --git a/test/packages/parallel/apache/data_stream/access/sample_event.json b/test/packages/parallel/apache/data_stream/access/sample_event.json index faf5bb50a..7885d8d9d 100644 --- a/test/packages/parallel/apache/data_stream/access/sample_event.json +++ b/test/packages/parallel/apache/data_stream/access/sample_event.json @@ -1,84 +1,91 @@ { + "@timestamp": "2022-09-06T07:48:09.000Z", "agent": { - "hostname": "4942ef7a8cfc", - "name": "4942ef7a8cfc", - "id": "73de002e-d848-49c7-829d-e903959d0d44", + "ephemeral_id": "357ad51a-c816-46b4-a094-8da1fe2b47af", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", "type": "filebeat", - "ephemeral_id": "e8970288-5c73-40e7-8626-8d297104f4eb", - "version": "7.11.0" - }, - "log": { - "file": { - "path": "/tmp/service_logs/access.log" - }, - "offset": 0 - }, - "elastic_agent": { - "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", - "version": "7.11.0", - "snapshot": true - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "original": "/" - }, - "input": { - "type": "log" + "version": "8.3.3" }, "apache": { "access": {} }, - "@timestamp": "2020-12-03T16:25:36.000Z", - "ecs": { - "version": "1.5.0" - }, "data_stream": { + "dataset": "apache.access", "namespace": "ep", - "type": "logs", - "dataset": "apache.access" + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-09-06T07:48:30.073Z", + "dataset": "apache.access", + "ingested": "2022-09-06T07:48:31Z", + "kind": "event", + "outcome": "success" }, "host": { - "hostname": "4942ef7a8cfc", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, - "containerized": true, + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", "ip": [ - "192.168.0.4" + "172.21.0.7" ], - "name": "4942ef7a8cfc", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:00:04" + "02:42:ac:15:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "http": { "request": { "method": "GET" }, "response": { - "status_code": 200, "body": { "bytes": 45 - } + }, + "status_code": 200 }, "version": "1.1" }, - "event": { - "kind": "event", - "created": "2020-12-03T16:25:53.907Z", - "category": "web", - "dataset": "apache.access", - "outcome": "success" + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "apache-access" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" diff --git a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index d3a1f6021..3672b5ad2 100644 --- a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -1,170 +1,186 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154511517Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.084440800Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T10:30:23.084454100Z", + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154513602Z", + "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", - "category": "web", - "type": "info", "timezone": "GMT+2", - "kind": "event" + "type": [ + "info" + ] + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 35708, - "thread": { - "id": 4328636416 - } - }, + "@timestamp": "2011-09-09T10:42:29.902+02:00", "apache": { "error": { "module": "core" } }, - "file": { - "path": "/usr/local/apache2/htdocs/favicon.ico" - }, - "@timestamp": "2011-09-09T10:42:29.902+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154514592Z", + "kind": "event", + "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "process": { + "pid": 35708, + "thread": { + "id": 4328636416 + } + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T10:30:23.084463800Z", - "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 15934 - }, + "@timestamp": "2019-06-27T06:58:09.169+02:00", "apache": { "error": { "module": "include" } }, - "@timestamp": "2019-06-27T06:58:09.169+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.154515567Z", + "kind": "event", + "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, "log": { "level": "warn" }, + "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "process": { + "pid": 15934 + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", - "port": 12345, - "ip": "89.160.20.156" - }, - "event": { - "ingested": "2021-12-14T10:30:23.084472500Z", - "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 12345 }, - "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "tags": [ "preserve_original_event" ] diff --git a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index e7dec80ec..732c38fc7 100644 --- a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -1,59 +1,67 @@ { "expected": [ { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "mpm_prefork" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T10:30:23.334072200Z", + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.194317231Z", + "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", - "category": "web", - "type": "info", "timezone": "GMT+2", - "kind": "event" + "type": [ + "info" + ] + }, + "log": { + "level": "notice" }, "message": "AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T10:30:23.334081Z", + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.194320072Z", + "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", - "category": "web", - "type": "info", "timezone": "GMT+2", - "kind": "event" + "type": [ + "info" + ] + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] diff --git a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index eb72ece49..ea82b6718 100644 --- a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -1,33 +1,37 @@ { "expected": [ { - "process": { - "pid": 121591, - "thread": { - "id": 140413273032448 - } - }, + "@timestamp": "2021-10-20T19:20:59.121+02:00", "apache": { "error": { "module": "rewrite" } }, - "@timestamp": "2021-10-20T19:20:59.121+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "trace3" - }, "event": { - "ingested": "2021-12-14T10:30:23.413830300Z", + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.227400182Z", + "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", - "category": "web", - "type": "info", "timezone": "GMT+2", - "kind": "event" + "type": [ + "info" + ] + }, + "log": { + "level": "trace3" }, "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "process": { + "pid": 121591, + "thread": { + "id": 140413273032448 + } + }, "tags": [ "preserve_original_event" ] diff --git a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 2a8a76e17..d3274d559 100644 --- a/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/test/packages/parallel/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -1,23 +1,27 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:17:53.000+02:00", "apache": { "error": {} }, - "@timestamp": "2016-12-26T16:17:53.000+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T10:30:23.471847700Z", + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259657304Z", + "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", - "category": "web", - "type": "info", "timezone": "GMT+2", - "kind": "event" + "type": [ + "info" + ] + }, + "log": { + "level": "notice" }, "message": "Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "tags": [ @@ -25,192 +29,216 @@ ] }, { + "@timestamp": "2016-12-26T16:22:00.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:00.000+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "error" + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259659782Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" }, "http": { "request": { "referrer": "http://192.168.33.72/" } }, + "log": { + "level": "error" + }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471861500Z", - "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259660793Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471870100Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259661763Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471878Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:10.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/test" - }, - "@timestamp": "2016-12-26T16:22:10.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259662714Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/test" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/test", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471885700Z", - "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/test", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:13.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/hello" - }, - "@timestamp": "2016-12-26T16:22:13.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259663668Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/hello" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/hello", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471889200Z", - "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/hello", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:17.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/crap" - }, - "@timestamp": "2016-12-26T16:22:17.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": [ + "web" + ], + "ingested": "2022-09-06T07:11:47.259664611Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", + "timezone": "GMT+2", + "type": [ + "error" + ] + }, + "file": { + "path": "/var/www/crap" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/crap", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T10:30:23.471894500Z", - "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/crap", "tags": [ "preserve_original_event" ] diff --git a/test/packages/parallel/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml index 6c4bba6c7..92037c49d 100644 --- a/test/packages/parallel/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/parallel/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -64,16 +64,16 @@ processors: value: event - set: field: event.category - value: web + value: ["web"] - script: if: "ctx?.log?.level != null" lang: painless source: >- def err_levels = ["emerg", "alert", "crit", "error", "warn"]; if (err_levels.contains(ctx.log.level)) { - ctx.event.type = "error"; + ctx.event.type = ["error"]; } else { - ctx.event.type = "info"; + ctx.event.type = ["info"]; } - grok: diff --git a/test/packages/parallel/apache/data_stream/error/sample_event.json b/test/packages/parallel/apache/data_stream/error/sample_event.json index 6ddd60ff9..afe50f526 100644 --- a/test/packages/parallel/apache/data_stream/error/sample_event.json +++ b/test/packages/parallel/apache/data_stream/error/sample_event.json @@ -1,74 +1,82 @@ { + "@timestamp": "2022-09-06T07:48:48.030Z", "agent": { - "hostname": "4942ef7a8cfc", - "name": "4942ef7a8cfc", - "id": "73de002e-d848-49c7-829d-e903959d0d44", - "ephemeral_id": "e8970288-5c73-40e7-8626-8d297104f4eb", + "ephemeral_id": "7d905f37-6778-4228-95af-98b28a0fcb12", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", "type": "filebeat", - "version": "7.11.0" + "version": "8.3.3" }, - "process": { - "pid": 1, - "thread": { - "id": 140503592395904 - } - }, - "log": { - "file": { - "path": "/tmp/service_logs/error.log" - }, - "offset": 0, - "level": "notice" - }, - "elastic_agent": { - "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", - "version": "7.11.0", - "snapshot": true - }, - "message": "AH00489: Apache/2.4.46 (Unix) configured -- resuming normal operations", - "input": { - "type": "log" - }, - "@timestamp": "2020-12-03T16:28:16.376Z", "apache": { "error": { "module": "mpm_event" } }, - "ecs": { - "version": "1.5.0" - }, "data_stream": { + "dataset": "apache.error", "namespace": "ep", - "type": "logs", - "dataset": "apache.error" + "type": "logs" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "apache.error", + "ingested": "2022-09-06T07:49:06Z", + "kind": "event", + "timezone": "+00:00", + "type": [ + "info" + ] }, "host": { - "hostname": "4942ef7a8cfc", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, - "containerized": true, + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", "ip": [ - "192.168.0.4" + "172.21.0.7" ], - "name": "4942ef7a8cfc", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:00:04" + "02:42:ac:15:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, - "event": { - "timezone": "+00:00", - "kind": "event", - "category": "web", - "type": "info", - "dataset": "apache.error" - } + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "level": "notice", + "offset": 0 + }, + "message": "AH00489: Apache/2.4.46 (Unix) configured -- resuming normal operations", + "process": { + "pid": 1, + "thread": { + "id": 140493350368384 + } + }, + "tags": [ + "apache-error" + ] } \ No newline at end of file diff --git a/test/packages/parallel/apache/data_stream/status/sample_event.json b/test/packages/parallel/apache/data_stream/status/sample_event.json index 2d2326729..fe5807825 100644 --- a/test/packages/parallel/apache/data_stream/status/sample_event.json +++ b/test/packages/parallel/apache/data_stream/status/sample_event.json @@ -1,101 +1,110 @@ { - "@timestamp": "2020-12-03T16:31:04.445Z", - "data_stream": { - "type": "metrics", - "dataset": "apache.status", - "namespace": "ep" - }, - "elastic_agent": { - "version": "7.11.0", - "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", - "snapshot": true - }, - "host": { - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.9.184-linuxkit", - "codename": "Core" - }, - "id": "06c26569966fd125c15acac5d7feffb6", - "name": "4942ef7a8cfc", - "containerized": true, - "ip": [ - "192.168.0.4" - ], - "mac": [ - "02:42:c0:a8:00:04" - ], - "hostname": "4942ef7a8cfc", - "architecture": "x86_64" - }, + "@timestamp": "2022-09-06T07:49:38.359Z", "agent": { - "hostname": "4942ef7a8cfc", - "ephemeral_id": "8371d3a3-5321-4436-9fd5-cafcabfe4c57", - "id": "af6f66ef-d7d0-4784-b9bb-3fddbcc151b5", - "name": "4942ef7a8cfc", + "ephemeral_id": "8d98054a-3077-4bb5-81b8-89c4da73f566", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", "type": "metricbeat", - "version": "7.11.0" - }, - "metricset": { - "name": "status", - "period": 30000 - }, - "service": { - "address": "http://elastic-package-service_apache_1:80/server-status?auto=", - "type": "apache" + "version": "8.3.3" }, "apache": { "status": { - "load": { - "5": 1.89, - "15": 1.07, - "1": 1.53 - }, - "total_accesses": 11, + "bytes_per_request": 0, + "bytes_per_sec": 0, "connections": { - "total": 0, "async": { "closing": 0, - "writing": 0, - "keep_alive": 0 - } + "keep_alive": 0, + "writing": 0 + }, + "total": 0 }, - "requests_per_sec": 0.916667, + "cpu": { + "children_system": 0, + "children_user": 0, + "load": 0.230769, + "system": 0.02, + "user": 0.01 + }, + "load": { + "1": 2.68, + "15": 2.79, + "5": 3.48 + }, + "requests_per_sec": 0.923077, "scoreboard": { - "starting_up": 0, + "closing_connection": 0, + "dns_lookup": 0, + "gracefully_finishing": 0, + "idle_cleanup": 0, "keepalive": 0, - "sending_reply": 1, "logging": 0, - "gracefully_finishing": 0, - "dns_lookup": 0, - "closing_connection": 0, "open_slot": 325, + "reading_request": 0, + "sending_reply": 1, + "starting_up": 0, "total": 400, - "idle_cleanup": 0, - "waiting_for_connection": 74, - "reading_request": 0 + "waiting_for_connection": 74 }, - "bytes_per_sec": 0, - "bytes_per_request": 0, + "total_accesses": 12, + "total_bytes": 0, "uptime": { - "server_uptime": 12, - "uptime": 12 + "server_uptime": 13, + "uptime": 13 }, - "total_bytes": 0, "workers": { "busy": 1, "idle": 74 - }, - "cpu": { - "load": 0.583333, - "user": 0.03, - "system": 0.04, - "children_user": 0, - "children_system": 0 } } + }, + "data_stream": { + "dataset": "apache.status", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "dataset": "apache.status", + "duration": 1049700, + "ingested": "2022-09-06T07:49:38Z", + "module": "apache" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "status", + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service-apache-1:80/server-status?auto=", + "type": "apache" } } \ No newline at end of file diff --git a/test/packages/parallel/apache/docs/README.md b/test/packages/parallel/apache/docs/README.md index f94910d4d..411b02c50 100644 --- a/test/packages/parallel/apache/docs/README.md +++ b/test/packages/parallel/apache/docs/README.md @@ -37,7 +37,7 @@ Access logs collects the Apache access logs. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | Destination domain. | keyword | +| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | error.message | Error message. | match_only_text | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | @@ -66,7 +66,7 @@ Access logs collects the Apache access logs. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | @@ -84,7 +84,7 @@ Access logs collects the Apache access logs. | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.domain | Source domain. | keyword | +| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | source.geo.city_name | City name. | keyword | | source.geo.continent_name | Name of the continent. | keyword | | source.geo.country_iso_code | Country ISO code. | keyword | @@ -170,7 +170,7 @@ Error logs collects the Apache error logs. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | @@ -223,104 +223,113 @@ An example event for `status` looks as following: ```json { - "@timestamp": "2020-12-03T16:31:04.445Z", - "data_stream": { - "type": "metrics", - "dataset": "apache.status", - "namespace": "ep" - }, - "elastic_agent": { - "version": "7.11.0", - "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", - "snapshot": true - }, - "host": { - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.9.184-linuxkit", - "codename": "Core" - }, - "id": "06c26569966fd125c15acac5d7feffb6", - "name": "4942ef7a8cfc", - "containerized": true, - "ip": [ - "192.168.0.4" - ], - "mac": [ - "02:42:c0:a8:00:04" - ], - "hostname": "4942ef7a8cfc", - "architecture": "x86_64" - }, + "@timestamp": "2022-09-06T07:49:38.359Z", "agent": { - "hostname": "4942ef7a8cfc", - "ephemeral_id": "8371d3a3-5321-4436-9fd5-cafcabfe4c57", - "id": "af6f66ef-d7d0-4784-b9bb-3fddbcc151b5", - "name": "4942ef7a8cfc", + "ephemeral_id": "8d98054a-3077-4bb5-81b8-89c4da73f566", + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "name": "docker-fleet-agent", "type": "metricbeat", - "version": "7.11.0" - }, - "metricset": { - "name": "status", - "period": 30000 - }, - "service": { - "address": "http://elastic-package-service_apache_1:80/server-status?auto=", - "type": "apache" + "version": "8.3.3" }, "apache": { "status": { - "load": { - "5": 1.89, - "15": 1.07, - "1": 1.53 - }, - "total_accesses": 11, + "bytes_per_request": 0, + "bytes_per_sec": 0, "connections": { - "total": 0, "async": { "closing": 0, - "writing": 0, - "keep_alive": 0 - } + "keep_alive": 0, + "writing": 0 + }, + "total": 0 }, - "requests_per_sec": 0.916667, + "cpu": { + "children_system": 0, + "children_user": 0, + "load": 0.230769, + "system": 0.02, + "user": 0.01 + }, + "load": { + "1": 2.68, + "15": 2.79, + "5": 3.48 + }, + "requests_per_sec": 0.923077, "scoreboard": { - "starting_up": 0, + "closing_connection": 0, + "dns_lookup": 0, + "gracefully_finishing": 0, + "idle_cleanup": 0, "keepalive": 0, - "sending_reply": 1, "logging": 0, - "gracefully_finishing": 0, - "dns_lookup": 0, - "closing_connection": 0, "open_slot": 325, + "reading_request": 0, + "sending_reply": 1, + "starting_up": 0, "total": 400, - "idle_cleanup": 0, - "waiting_for_connection": 74, - "reading_request": 0 + "waiting_for_connection": 74 }, - "bytes_per_sec": 0, - "bytes_per_request": 0, + "total_accesses": 12, + "total_bytes": 0, "uptime": { - "server_uptime": 12, - "uptime": 12 + "server_uptime": 13, + "uptime": 13 }, - "total_bytes": 0, "workers": { "busy": 1, "idle": 74 - }, - "cpu": { - "load": 0.583333, - "user": 0.03, - "system": 0.04, - "children_user": 0, - "children_system": 0 } } + }, + "data_stream": { + "dataset": "apache.status", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "snapshot": false, + "version": "8.3.3" + }, + "event": { + "agent_id_status": "verified", + "dataset": "apache.status", + "duration": 1049700, + "ingested": "2022-09-06T07:49:38Z", + "module": "apache" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.0-43-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "status", + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service-apache-1:80/server-status?auto=", + "type": "apache" } } ``` diff --git a/test/packages/parallel/apache/manifest.yml b/test/packages/parallel/apache/manifest.yml index b04065d21..01ddfe7e2 100644 --- a/test/packages/parallel/apache/manifest.yml +++ b/test/packages/parallel/apache/manifest.yml @@ -1,11 +1,10 @@ -format_version: 1.0.0 +format_version: 2.0.0 name: apache title: Apache HTTP Server # version is set to something very large to so this test package can # be installed in the package registry regardless of the version of # the actual apache package in the registry at any given time. version: 999.999.999 -license: basic description: Collect logs and metrics from Apache servers with Elastic Agent. type: integration categories: @@ -13,6 +12,7 @@ categories: release: ga conditions: kibana.version: "^7.14.0 || ^8.0.0" + elastic.subscription: basic screenshots: - src: /img/apache-metrics-overview.png title: Apache metrics overview diff --git a/test/packages/parallel/nginx/_dev/build/build.yml b/test/packages/parallel/nginx/_dev/build/build.yml index 08d85edcf..6b5cc3fbc 100644 --- a/test/packages/parallel/nginx/_dev/build/build.yml +++ b/test/packages/parallel/nginx/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.12 + reference: git@8.1 diff --git a/test/packages/parallel/nginx/docs/README.md b/test/packages/parallel/nginx/docs/README.md index b23e8ec4b..8d7edbc10 100644 --- a/test/packages/parallel/nginx/docs/README.md +++ b/test/packages/parallel/nginx/docs/README.md @@ -158,7 +158,7 @@ An example event for `access` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | Destination domain. | keyword | +| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.port | Port of the destination. | long | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | @@ -182,7 +182,7 @@ An example event for `access` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long |