From 05e80f8682c1d26f1dd400c9450440a0b213ae5f Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Mon, 21 Oct 2019 14:28:28 -0500 Subject: [PATCH 1/4] Replace mapval with go-lookslike This cleans up some tech debt since the mapval package has been renamed and moved to https://github.com/elastic/go-lookslike , we should stop using the now older/outdated mapval package. --- libbeat/common/mapval/compiled_schema.go | 44 -- libbeat/common/mapval/core.go | 151 ------- libbeat/common/mapval/core_test.go | 377 ------------------ libbeat/common/mapval/doc.go | 23 -- libbeat/common/mapval/doc_test.go | 206 ---------- libbeat/common/mapval/is_defs.go | 334 ---------------- libbeat/common/mapval/is_defs_test.go | 210 ---------- libbeat/common/mapval/path.go | 194 --------- libbeat/common/mapval/path_test.go | 324 --------------- libbeat/common/mapval/reflect_tools.go | 62 --- libbeat/common/mapval/results.go | 139 ------- libbeat/common/mapval/results_test.go | 48 --- libbeat/common/mapval/testing.go | 46 --- libbeat/common/mapval/value.go | 87 ---- libbeat/common/mapval/walk.go | 95 ----- packetbeat/flows/worker_test.go | 27 +- .../module/cisco/ios/pipeline_test.go | 46 ++- 17 files changed, 40 insertions(+), 2373 deletions(-) delete mode 100644 libbeat/common/mapval/compiled_schema.go delete mode 100644 libbeat/common/mapval/core.go delete mode 100644 libbeat/common/mapval/core_test.go delete mode 100644 libbeat/common/mapval/doc.go delete mode 100644 libbeat/common/mapval/doc_test.go delete mode 100644 libbeat/common/mapval/is_defs.go delete mode 100644 libbeat/common/mapval/is_defs_test.go delete mode 100644 libbeat/common/mapval/path.go delete mode 100644 libbeat/common/mapval/path_test.go delete mode 100644 libbeat/common/mapval/reflect_tools.go delete mode 100644 libbeat/common/mapval/results.go delete mode 100644 libbeat/common/mapval/results_test.go delete mode 100644 libbeat/common/mapval/testing.go delete mode 100644 libbeat/common/mapval/value.go delete mode 100644 libbeat/common/mapval/walk.go diff --git a/libbeat/common/mapval/compiled_schema.go b/libbeat/common/mapval/compiled_schema.go deleted file mode 100644 index 054565550d68..000000000000 --- a/libbeat/common/mapval/compiled_schema.go +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import "github.com/elastic/beats/libbeat/common" - -type flatValidator struct { - path path - isDef IsDef -} - -// CompiledSchema represents a compiled definition for driving a Validator. -type CompiledSchema []flatValidator - -// Check executes the the checks within the CompiledSchema -func (cs CompiledSchema) Check(actual common.MapStr) *Results { - results := NewResults() - for _, pv := range cs { - actualV, actualKeyExists := pv.path.getFrom(actual) - - if !pv.isDef.optional || pv.isDef.optional && actualKeyExists { - var checkRes *Results - checkRes = pv.isDef.check(pv.path, actualV, actualKeyExists) - results.merge(checkRes) - } - } - - return results -} diff --git a/libbeat/common/mapval/core.go b/libbeat/common/mapval/core.go deleted file mode 100644 index 7359d52d1e73..000000000000 --- a/libbeat/common/mapval/core.go +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "reflect" - "sort" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -// Is creates a named IsDef with the given checker. -func Is(name string, checker ValueValidator) IsDef { - return IsDef{name: name, checker: checker} -} - -// Optional wraps an IsDef to mark the field's presence as optional. -func Optional(id IsDef) IsDef { - id.name = "optional " + id.name - id.optional = true - return id -} - -// Map is the type used to define schema definitions for Compile and to represent an arbitrary -// map of values of any type. -type Map map[string]interface{} - -// Slice is a convenience []interface{} used to declare schema defs. You would typically nest this inside -// a Map as a value, and it would be able to match against any type of non-empty slice. -type Slice []interface{} - -// Validator is the result of Compile and is run against the map you'd like to test. -type Validator func(common.MapStr) *Results - -// Compose combines multiple SchemaValidators into a single one. -func Compose(validators ...Validator) Validator { - return func(actual common.MapStr) *Results { - results := make([]*Results, len(validators)) - for idx, validator := range validators { - results[idx] = validator(actual) - } - - combined := NewResults() - for _, r := range results { - r.EachResult(func(path path, vr ValueResult) bool { - combined.record(path, vr) - return true - }) - } - return combined - } -} - -// Strict is used when you want any unspecified keys that are encountered to be considered errors. -func Strict(laxValidator Validator) Validator { - return func(actual common.MapStr) *Results { - results := laxValidator(actual) - - // The inner workings of this are a little weird - // We use a hash of dotted paths to track the results - // We can check if a key had a test associated with it by looking up the laxValidator - // result data - // What's trickier is intermediate maps, maps don't usually have explicit tests, they usually just have - // their properties tested. - // This method counts an intermediate map as tested if a subkey is tested. - // Since the datastructure we have to search is a flattened hashmap of the original map we take that hashmap - // and turn it into a sorted string array, then do a binary prefix search to determine if a subkey was tested. - // It's a little weird, but is fairly efficient. We could stop using the flattened map as a datastructure, but - // that would add complexity elsewhere. Probably a good refactor at some point, but not worth it now. - validatedPaths := []string{} - for k := range results.Fields { - validatedPaths = append(validatedPaths, k) - } - sort.Strings(validatedPaths) - - walk(actual, false, func(woi walkObserverInfo) error { - _, validatedExactly := results.Fields[woi.path.String()] - if validatedExactly { - return nil // This key was tested, passes strict test - } - - // Search returns the point just before an actual match (since we ruled out an exact match with the cheaper - // hash check above. We have to validate the actual match with a prefix check as well - matchIdx := sort.SearchStrings(validatedPaths, woi.path.String()) - if matchIdx < len(validatedPaths) && strings.HasPrefix(validatedPaths[matchIdx], woi.path.String()) { - return nil - } - - results.merge(StrictFailureResult(woi.path)) - - return nil - }) - - return results - } -} - -// Compile takes the given map, validates the paths within it, and returns -// a Validator that can check real data. -func Compile(in Map) (validator Validator, err error) { - compiled := make(CompiledSchema, 0) - err = walk(common.MapStr(in), true, func(current walkObserverInfo) error { - - // Determine whether we should test this value - // We want to test all values except collections that contain a value - // If a collection contains a value, we check those 'leaf' values instead - rv := reflect.ValueOf(current.value) - kind := rv.Kind() - isCollection := kind == reflect.Map || kind == reflect.Slice - isNonEmptyCollection := isCollection && rv.Len() > 0 - - if !isNonEmptyCollection { - isDef, isIsDef := current.value.(IsDef) - if !isIsDef { - isDef = IsEqual(current.value) - } - - compiled = append(compiled, flatValidator{current.path, isDef}) - } - return nil - }) - - return func(actual common.MapStr) *Results { - return compiled.Check(actual) - }, err -} - -// MustCompile compiles the given map, panic-ing if that map is invalid. -func MustCompile(in Map) Validator { - compiled, err := Compile(in) - if err != nil { - panic(err) - } - return compiled -} diff --git a/libbeat/common/mapval/core_test.go b/libbeat/common/mapval/core_test.go deleted file mode 100644 index d97fdb994fb0..000000000000 --- a/libbeat/common/mapval/core_test.go +++ /dev/null @@ -1,377 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" -) - -func assertValidator(t *testing.T, validator Validator, input common.MapStr) { - res := validator(input) - assertResults(t, res) -} - -// assertResults validates the schema passed successfully. -func assertResults(t *testing.T, r *Results) *Results { - for _, err := range r.Errors() { - assert.NoError(t, err) - } - return r -} - -func TestFlat(t *testing.T) { - m := common.MapStr{ - "foo": "bar", - "baz": 1, - } - - results := MustCompile(Map{ - "foo": "bar", - "baz": IsIntGt(0), - })(m) - - assertResults(t, results) -} - -func TestBadFlat(t *testing.T) { - m := common.MapStr{} - - fakeT := new(testing.T) - - results := MustCompile(Map{ - "notafield": IsDuration, - })(m) - - assertResults(fakeT, results) - - assert.True(t, fakeT.Failed()) - - result := results.Fields["notafield"][0] - assert.False(t, result.Valid) - assert.Equal(t, result, KeyMissingVR) -} - -func TestNested(t *testing.T) { - m := common.MapStr{ - "foo": common.MapStr{ - "bar": "baz", - "dur": time.Duration(100), - }, - } - - results := MustCompile(Map{ - "foo": Map{ - "bar": "baz", - }, - "foo.dur": IsDuration, - })(m) - - assertResults(t, results) - - assert.Len(t, results.Fields, 2, "One result per matcher") -} - -func TestComposition(t *testing.T) { - m := common.MapStr{ - "foo": "bar", - "baz": "bot", - } - - fooValidator := MustCompile(Map{"foo": "bar"}) - bazValidator := MustCompile(Map{"baz": "bot"}) - composed := Compose(fooValidator, bazValidator) - - // Test that the validators work individually - assertValidator(t, fooValidator, m) - assertValidator(t, bazValidator, m) - - // Test that the composition of them works - assertValidator(t, composed, m) - - composedRes := composed(m) - assert.Len(t, composedRes.Fields, 2) - - badValidator := MustCompile(Map{"notakey": "blah"}) - badComposed := Compose(badValidator, composed) - - fakeT := new(testing.T) - assertValidator(fakeT, badComposed, m) - badComposedRes := badComposed(m) - - assert.Len(t, badComposedRes.Fields, 3) - assert.True(t, fakeT.Failed()) -} - -func TestStrictFunc(t *testing.T) { - m := common.MapStr{ - "foo": "bar", - "baz": "bot", - "nest": common.MapStr{ - "very": common.MapStr{ - "deep": "true", - }, - }, - } - - validValidator := MustCompile(Map{ - "foo": "bar", - "baz": "bot", - "nest": Map{ - "very": Map{ - "deep": "true", - }, - }, - }) - - assertValidator(t, validValidator, m) - - partialValidator := MustCompile(Map{ - "foo": "bar", - }) - - // Should pass, since this is not a strict check - assertValidator(t, partialValidator, m) - - res := Strict(partialValidator)(m) - - assert.Equal(t, []ValueResult{StrictFailureVR}, res.DetailedErrors().Fields["baz"]) - assert.Equal(t, []ValueResult{StrictFailureVR}, res.DetailedErrors().Fields["nest.very.deep"]) - assert.Nil(t, res.DetailedErrors().Fields["bar"]) - assert.False(t, res.Valid) -} - -func TestOptional(t *testing.T) { - m := common.MapStr{ - "foo": "bar", - } - - validator := MustCompile(Map{ - "non": Optional(IsEqual("foo")), - }) - - assertValidator(t, validator, m) -} - -func TestExistence(t *testing.T) { - m := common.MapStr{ - "exists": "foo", - } - - validator := MustCompile(Map{ - "exists": KeyPresent, - "non": KeyMissing, - }) - - assertValidator(t, validator, m) -} - -func TestComplex(t *testing.T) { - m := common.MapStr{ - "foo": "bar", - "hash": common.MapStr{ - "baz": 1, - "bot": 2, - "deep_hash": common.MapStr{ - "qux": "quark", - }, - }, - "slice": []string{"pizza", "pasta", "and more"}, - "empty": nil, - "arr": []common.MapStr{{"foo": "bar"}, {"foo": "baz"}}, - } - - validator := MustCompile(Map{ - "foo": "bar", - "hash": Map{ - "baz": 1, - "bot": 2, - "deep_hash": Map{ - "qux": "quark", - }, - }, - "slice": []string{"pizza", "pasta", "and more"}, - "empty": KeyPresent, - "doesNotExist": KeyMissing, - "arr": IsArrayOf(MustCompile(Map{"foo": IsStringContaining("a")})), - }) - - assertValidator(t, validator, m) -} - -func TestLiteralArray(t *testing.T) { - m := common.MapStr{ - "a": []interface{}{ - []interface{}{1, 2, 3}, - []interface{}{"foo", "bar"}, - "hello", - }, - } - - validator := MustCompile(Map{ - "a": []interface{}{ - []interface{}{1, 2, 3}, - []interface{}{"foo", "bar"}, - "hello", - }, - }) - - goodRes := validator(m) - - assertResults(t, goodRes) - // We evaluate multidimensional slice as a single field for now - // This is kind of easier, but maybe we should do our own traversal later. - assert.Len(t, goodRes.Fields, 6) -} - -func TestStringSlice(t *testing.T) { - m := common.MapStr{ - "a": []string{"a", "b"}, - } - - validator := MustCompile(Map{ - "a": []string{"a", "b"}, - }) - - goodRes := validator(m) - - assertResults(t, goodRes) - // We evaluate multidimensional slices as a single field for now - // This is kind of easier, but maybe we should do our own traversal later. - assert.Len(t, goodRes.Fields, 2) -} - -func TestEmptySlice(t *testing.T) { - // In the case of an empty Slice, the validator will compare slice type - // In this case we're treating the slice as a value and doing a literal comparison - // Users should use an IsDef testing for an empty slice (that can use reflection) - // if they need something else. - m := common.MapStr{ - "a": []interface{}{}, - "b": []string{}, - } - - validator := MustCompile(Map{ - "a": []interface{}{}, - "b": []string{}, - }) - - goodRes := validator(m) - - assertResults(t, goodRes) - assert.Len(t, goodRes.Fields, 2) -} - -func TestLiteralMdSlice(t *testing.T) { - m := common.MapStr{ - "a": [][]int{ - {1, 2, 3}, - {4, 5, 6}, - }, - } - - validator := MustCompile(Map{ - "a": [][]int{ - {1, 2, 3}, - {4, 5, 6}, - }, - }) - - goodRes := validator(m) - - assertResults(t, goodRes) - // We evaluate multidimensional slices as a single field for now - // This is kind of easier, but maybe we should do our own traversal later. - assert.Len(t, goodRes.Fields, 6) - - badValidator := Strict(MustCompile(Map{ - "a": [][]int{ - {1, 2, 3}, - }, - })) - - badRes := badValidator(m) - - assert.False(t, badRes.Valid) - assert.Len(t, badRes.Fields, 7) - // The reason the len is 4 is that there is 1 extra slice + 4 values. - assert.Len(t, badRes.Errors(), 4) -} - -func TestSliceOfIsDefs(t *testing.T) { - m := common.MapStr{ - "a": []int{1, 2, 3}, - "b": []interface{}{"foo", "bar", 3}, - } - - goodV := MustCompile(Map{ - "a": []interface{}{IsIntGt(0), IsIntGt(1), 3}, - "b": []interface{}{IsStringContaining("o"), "bar", IsIntGt(2)}, - }) - - assertValidator(t, goodV, m) - - badV := MustCompile(Map{ - "a": []interface{}{IsIntGt(100), IsIntGt(1), 3}, - "b": []interface{}{IsStringContaining("X"), "bar", IsIntGt(2)}, - }) - badRes := badV(m) - - assert.False(t, badRes.Valid) - assert.Len(t, badRes.Errors(), 2) -} - -func TestMatchArrayAsValue(t *testing.T) { - m := common.MapStr{ - "a": []int{1, 2, 3}, - "b": []interface{}{"foo", "bar", 3}, - } - - goodV := MustCompile(Map{ - "a": []int{1, 2, 3}, - "b": []interface{}{"foo", "bar", 3}, - }) - - assertValidator(t, goodV, m) - - badV := MustCompile(Map{ - "a": "robot", - "b": []interface{}{"foo", "bar", 3}, - }) - - badRes := badV(m) - - assert.False(t, badRes.Valid) - assert.False(t, badRes.Fields["a"][0].Valid) - for _, f := range badRes.Fields["b"] { - assert.True(t, f.Valid) - } -} - -func TestInvalidPathIsdef(t *testing.T) { - badPath := "foo...bar" - _, err := Compile(Map{ - badPath: "invalid", - }) - - assert.Equal(t, InvalidPathString(badPath), err) -} diff --git a/libbeat/common/mapval/doc.go b/libbeat/common/mapval/doc.go deleted file mode 100644 index 5fdf652f16ec..000000000000 --- a/libbeat/common/mapval/doc.go +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/* -Package mapval is used to validate JSON-like nested map data-structure against a set of expectations. Its key features are allowing custom, function defined validators for values, and allowing the composition of multiple validation specs. - -See the example below for more details. Most key functions include detailed examples of their use within this documentation. -*/ -package mapval diff --git a/libbeat/common/mapval/doc_test.go b/libbeat/common/mapval/doc_test.go deleted file mode 100644 index 7b2e655b62b1..000000000000 --- a/libbeat/common/mapval/doc_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "fmt" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -func Example() { - // Let's say we want to validate this map - data := common.MapStr{"foo": "bar", "baz": "bot", "count": 1} - - // We can validate the data by creating a mapval.Validator - // Validators are functions created by compiling the special mapval.Map - // type. This is a map[string]interface{} that can be compiled - // into a series of checks. - // - // Literal values in a mapval.Map are checked for equality. - // More complex checks can be done using values of the mapval.IsDef - // type. In this case, we're using an IsDef to see if the "foo" key - // contains the string "a", and we're using a literal to check that the - // "baz" key contains the exact value "bot". - validator := MustCompile(Map{ - "foo": IsStringContaining("a"), - "baz": "bot", - }) - - // When being used in test-suites, you should use mapvaltest.Test to execute the validator - // This produces easy to read test output, and outputs one failed assertion per failed matcher - // See the docs for mapvaltest for more info - // mapvaltest.Test(t, validator, data) - - // If you need more control than mapvaltest.Test provides, you can use the results directly - results := validator(data) - - // The Results.Valid property indicates if the validator passed - fmt.Printf("Results.Valid: %t\n", results.Valid) - - // Results.Errors() returns one error per failed match - fmt.Printf("There were %d errors\n", len(results.Errors())) - - // Results.Fields is a map of paths defined in the input mapval.Map to the result of their validation - // This is useful if you need more control - fmt.Printf("Over %d fields\n", len(results.Fields)) - - // You may be thinking that the validation above should have failed since there was an - // extra key, 'count', defined that was encountered. By default mapval does not - // consider extra data to be an error. To change that behavior, wrap the validator - // in mapval.Strict() - strictResults := Strict(validator)(data) - - fmt.Printf("Strict Results.Valid: %t\n", strictResults.Valid) - - // You can check an exact field for an error - fmt.Printf("For the count field specifically .Valid is: %t\n", strictResults.Fields["count"][0].Valid) - - // And get error objects for each error - for _, err := range strictResults.Errors() { - fmt.Println(err) - } - - // And even get a new Results object with only invalid fields included - strictResults.DetailedErrors() -} - -func ExampleCompose() { - // Composition is useful when you need to share common validation logic between validators. - // Let's imagine that we want to validate maps describing pets. - - pets := []common.MapStr{ - {"name": "rover", "barks": "often", "fur_length": "long"}, - {"name": "lucky", "barks": "rarely", "fur_length": "short"}, - {"name": "pounce", "meows": "often", "fur_length": "short"}, - {"name": "peanut", "meows": "rarely", "fur_length": "long"}, - } - - // We can see that all pets have the "fur_length" property, but that only cats meow, and dogs bark. - // We can concisely encode this in mapval using mapval.Compose. - // We can also see that both "meows" and "barks" contain the same enums of values. - // We'll start by creating a composed IsDef using the IsAny composition, which creates a new IsDef that is - // a logical 'or' of its IsDef arguments - - isFrequency := IsAny(IsEqual("often"), IsEqual("rarely")) - - petValidator := MustCompile(Map{ - "name": IsNonEmptyString, - "fur_length": IsAny(IsEqual("long"), IsEqual("short")), - }) - dogValidator := Compose( - petValidator, - MustCompile(Map{"barks": isFrequency}), - ) - catValidator := Compose( - petValidator, - MustCompile(Map{"meows": isFrequency}), - ) - - for _, pet := range pets { - var petType string - if dogValidator(pet).Valid { - petType = "dog" - } else if catValidator(pet).Valid { - petType = "cat" - } - fmt.Printf("%s is a %s\n", pet["name"], petType) - } - - // Output: - // rover is a dog - // lucky is a dog - // pounce is a cat - // peanut is a cat -} - -func ExampleOptional() { - dataNoError := common.MapStr{"foo": "bar"} - dataError := common.MapStr{"foo": "bar", "error": true} - - validator := MustCompile(Map{"foo": "bar", "error": Optional(IsEqual(true))}) - - // Both inputs pass - fmt.Printf("Validator classifies both maps as true: %t", validator(dataNoError).Valid && validator(dataError).Valid) - - // Output: - // Validator classifies both maps as true: true -} - -func ExampleIs() { - // More advanced validations can be used with built-in and custom functions. - // These are represented with the IfDef type - - data := common.MapStr{"foo": "bar", "count": 1} - - // Values can also be tested programatically if a mapval.IsDef is used as a value - // Here we'll define a custom IsDef using the mapval DSL, then validate it. - // The Is() function is the preferred way to costruct IsDef objects. - startsWithB := Is("starts with b", func(path path, v interface{}) *Results { - vStr, ok := v.(string) - if !ok { - return SimpleResult(path, false, "Expected a string, got a %t", v) - } - - if strings.HasPrefix(vStr, "b") { - return ValidResult(path) - } - - return SimpleResult(path, false, "Expected string to start with b, got %v", vStr) - }) - - funcValidator := MustCompile(Map{"foo": startsWithB}) - - funcValidatorResult := funcValidator(data) - - fmt.Printf("Valid: %t", funcValidatorResult.Valid) - - // Output: - // Valid: true -} - -func ExampleMap() { - v := MustCompile(Map{ - "foo": IsStringContaining("a"), - "baz": "bot", - }) - - data := common.MapStr{ - "foo": "bar", - "baz": "bot", - } - - fmt.Printf("Result is %t", v(data).Valid) - - // Output: - // Result is true -} - -func ExampleSlice() { - v := MustCompile(Map{ - "foo": Slice{"foo", IsNonEmptyString}, - }) - - data := common.MapStr{"foo": []string{"foo", "something"}} - - fmt.Printf("Result is %t", v(data).Valid) - - // Output: - // Result is true -} diff --git a/libbeat/common/mapval/is_defs.go b/libbeat/common/mapval/is_defs.go deleted file mode 100644 index 457e85343dcf..000000000000 --- a/libbeat/common/mapval/is_defs.go +++ /dev/null @@ -1,334 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "fmt" - "reflect" - "regexp" - "strings" - "time" - - "github.com/elastic/beats/libbeat/common" -) - -// KeyPresent checks that the given key is in the map, even if it has a nil value. -var KeyPresent = IsDef{name: "check key present"} - -// KeyMissing checks that the given key is not present defined. -var KeyMissing = IsDef{name: "check key not present", checkKeyMissing: true} - -func init() { - MustRegisterEqual(IsEqualToTime) -} - -// InvalidEqualFnError is the error type returned by RegisterEqual when -// there is an issue with the given function. -type InvalidEqualFnError struct{ msg string } - -func (e InvalidEqualFnError) Error() string { - return fmt.Sprintf("Function is not a valid equal function: %s", e.msg) -} - -// MustRegisterEqual is the panic-ing equivalent of RegisterEqual. -func MustRegisterEqual(fn interface{}) { - if err := RegisterEqual(fn); err != nil { - panic(fmt.Sprintf("Could not register fn as equal! %v", err)) - } -} - -var equalChecks = map[reflect.Type]reflect.Value{} - -// RegisterEqual takes a function of the form fn(v someType) IsDef -// and registers it to check equality for that type. -func RegisterEqual(fn interface{}) error { - fnV := reflect.ValueOf(fn) - fnT := fnV.Type() - - if fnT.Kind() != reflect.Func { - return InvalidEqualFnError{"Provided value is not a function"} - } - if fnT.NumIn() != 1 { - return InvalidEqualFnError{"Equal FN should take one argument"} - } - if fnT.NumOut() != 1 { - return InvalidEqualFnError{"Equal FN should return one value"} - } - if fnT.Out(0) != reflect.TypeOf(IsDef{}) { - return InvalidEqualFnError{"Equal FN should return an IsDef"} - } - - inT := fnT.In(0) - if _, ok := equalChecks[inT]; ok { - return InvalidEqualFnError{fmt.Sprintf("Duplicate Equal FN for type %v encountered!", inT)} - } - - equalChecks[inT] = fnV - - return nil -} - -// IsEqual tests that the given object is equal to the actual object. -func IsEqual(to interface{}) IsDef { - toV := reflect.ValueOf(to) - isDefFactory, ok := equalChecks[toV.Type()] - - // If there are no handlers declared explicitly for this type we perform a deep equality check - if !ok { - return IsDeepEqual(to) - } - - // We know this is an isdef due to the Register check previously - checker := isDefFactory.Call([]reflect.Value{toV})[0].Interface().(IsDef).checker - - return Is("equals", func(path path, v interface{}) *Results { - return checker(path, v) - }) -} - -// IsEqualToTime ensures that the actual value is the given time, regardless of zone. -func IsEqualToTime(to time.Time) IsDef { - return Is("equal to time", func(path path, v interface{}) *Results { - actualTime, ok := v.(time.Time) - if !ok { - return SimpleResult(path, false, "Value %t was not a time.Time", v) - } - - if actualTime.Equal(to) { - return ValidResult(path) - } - - return SimpleResult(path, false, "actual(%v) != expected(%v)", actualTime, to) - }) -} - -// IsDeepEqual checks equality using reflect.DeepEqual. -func IsDeepEqual(to interface{}) IsDef { - return Is("equals", func(path path, v interface{}) *Results { - if reflect.DeepEqual(v, to) { - return ValidResult(path) - } - return SimpleResult( - path, - false, - fmt.Sprintf("objects not equal: actual(%T(%v)) != expected(%T(%v))", v, v, to, to), - ) - }) -} - -// IsArrayOf validates that the array at the given key is an array of objects all validatable -// via the given Validator. -func IsArrayOf(validator Validator) IsDef { - return Is("array of maps", func(path path, v interface{}) *Results { - vArr, isArr := v.([]common.MapStr) - if !isArr { - return SimpleResult(path, false, "Expected array at given path") - } - - results := NewResults() - - for idx, curMap := range vArr { - var validatorRes *Results - validatorRes = validator(curMap) - results.mergeUnderPrefix(path.extendSlice(idx), validatorRes) - } - - return results - }) -} - -// IsAny takes a variable number of IsDef's and combines them with a logical OR. If any single definition -// matches the key will be marked as valid. -func IsAny(of ...IsDef) IsDef { - names := make([]string, len(of)) - for i, def := range of { - names[i] = def.name - } - isName := fmt.Sprintf("either %#v", names) - - return Is(isName, func(path path, v interface{}) *Results { - for _, def := range of { - vr := def.check(path, v, true) - if vr.Valid { - return vr - } - } - - return SimpleResult( - path, - false, - fmt.Sprintf("Value was none of %#v, actual value was %#v", names, v), - ) - }) -} - -// IsUnique instances are used in multiple spots, flagging a value as being in error if it's seen across invocations. -// To use it, assign IsUnique to a variable, then use that variable multiple times in a Map. -func IsUnique() IsDef { - return ScopedIsUnique().IsUniqueTo("") -} - -// UniqScopeTracker is represents the tracking data for invoking IsUniqueTo. -type UniqScopeTracker map[interface{}]string - -// IsUniqueTo validates that the given value is only ever seen within a single namespace. -func (ust UniqScopeTracker) IsUniqueTo(namespace string) IsDef { - return Is("unique", func(path path, v interface{}) *Results { - for trackerK, trackerNs := range ust { - hasNamespace := len(namespace) > 0 - if reflect.DeepEqual(trackerK, v) && (!hasNamespace || namespace != trackerNs) { - return SimpleResult(path, false, "Value '%v' is repeated", v) - } - } - - ust[v] = namespace - return ValidResult(path) - }) -} - -func ScopedIsUnique() UniqScopeTracker { - return UniqScopeTracker{} -} - -// isStrCheck is a helper for IsDefs that must assert that the value is a string first. -func isStrCheck(path path, v interface{}) (str string, errorResults *Results) { - strV, ok := v.(string) - - if !ok { - return "", SimpleResult( - path, - false, - fmt.Sprintf("Unable to convert '%v' to string", v), - ) - } - - return strV, nil -} - -// IsString checks that the given value is a string. -var IsString = Is("is a string", func(path path, v interface{}) *Results { - _, errorResults := isStrCheck(path, v) - if errorResults != nil { - return errorResults - } - - return ValidResult(path) -}) - -// IsNonEmptyString checks that the given value is a string and has a length > 1. -var IsNonEmptyString = Is("is a non-empty string", func(path path, v interface{}) *Results { - strV, errorResults := isStrCheck(path, v) - if errorResults != nil { - return errorResults - } - - if len(strV) == 0 { - return SimpleResult(path, false, "String '%s' should not be empty", strV) - } - - return ValidResult(path) -}) - -// IsStringMatching checks whether a value matches the given regexp. -func IsStringMatching(regexp *regexp.Regexp) IsDef { - return Is("is string matching regexp", func(path path, v interface{}) *Results { - strV, errorResults := isStrCheck(path, v) - if errorResults != nil { - return errorResults - } - - if !regexp.MatchString(strV) { - return SimpleResult( - path, - false, - fmt.Sprintf("String '%s' did not match regexp %s", strV, regexp.String()), - ) - } - - return ValidResult(path) - }) -} - -// IsStringContaining validates that the the actual value contains the specified substring. -func IsStringContaining(needle string) IsDef { - return Is("is string containing", func(path path, v interface{}) *Results { - strV, errorResults := isStrCheck(path, v) - if errorResults != nil { - return errorResults - } - - if !strings.Contains(strV, needle) { - return SimpleResult( - path, - false, - fmt.Sprintf("String '%s' did not contain substring '%s'", strV, needle), - ) - } - - return ValidResult(path) - }) -} - -// IsDuration tests that the given value is a duration. -var IsDuration = Is("is a duration", func(path path, v interface{}) *Results { - if _, ok := v.(time.Duration); ok { - return ValidResult(path) - } - return SimpleResult( - path, - false, - fmt.Sprintf("Expected a time.duration, got '%v' which is a %T", v, v), - ) -}) - -// IsNil tests that a value is nil. -var IsNil = Is("is nil", func(path path, v interface{}) *Results { - if v == nil { - return ValidResult(path) - } - return SimpleResult( - path, - false, - fmt.Sprintf("Value %v is not nil", v), - ) -}) - -func intGtChecker(than int) ValueValidator { - return func(path path, v interface{}) *Results { - n, ok := v.(int) - if !ok { - msg := fmt.Sprintf("%v is a %T, but was expecting an int!", v, v) - return SimpleResult(path, false, msg) - } - - if n > than { - return ValidResult(path) - } - - return SimpleResult( - path, - false, - fmt.Sprintf("%v is not greater than %v", n, than), - ) - } -} - -// IsIntGt tests that a value is an int greater than. -func IsIntGt(than int) IsDef { - return Is("greater than", intGtChecker(than)) -} diff --git a/libbeat/common/mapval/is_defs_test.go b/libbeat/common/mapval/is_defs_test.go deleted file mode 100644 index f50250595fdc..000000000000 --- a/libbeat/common/mapval/is_defs_test.go +++ /dev/null @@ -1,210 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "regexp" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/libbeat/common" -) - -func assertIsDefValid(t *testing.T, id IsDef, value interface{}) *Results { - res := id.check(mustParsePath("p"), value, true) - - if !res.Valid { - assert.Fail( - t, - "Expected Valid IsDef", - "Isdef %#v was not valid for value %#v with error: ", id, value, res.Errors(), - ) - } - return res -} - -func assertIsDefInvalid(t *testing.T, id IsDef, value interface{}) *Results { - res := id.check(mustParsePath("p"), value, true) - - if res.Valid { - assert.Fail( - t, - "Expected invalid IsDef", - "Isdef %#v was should not have been valid for value %#v", - id, - value, - ) - } - return res -} - -func TestIsArrayOf(t *testing.T) { - validator := MustCompile(Map{"foo": "bar"}) - - id := IsArrayOf(validator) - - goodMap := common.MapStr{"foo": "bar"} - goodMapArr := []common.MapStr{goodMap, goodMap} - - goodRes := assertIsDefValid(t, id, goodMapArr) - goodFields := goodRes.Fields - assert.Len(t, goodFields, 2) - assert.Contains(t, goodFields, "p.[0].foo") - assert.Contains(t, goodFields, "p.[1].foo") - - badMap := common.MapStr{"foo": "bot"} - badMapArr := []common.MapStr{badMap} - - badRes := assertIsDefInvalid(t, id, badMapArr) - badFields := badRes.Fields - assert.Len(t, badFields, 1) - assert.Contains(t, badFields, "p.[0].foo") -} - -func TestIsAny(t *testing.T) { - id := IsAny(IsEqual("foo"), IsEqual("bar")) - - assertIsDefValid(t, id, "foo") - assertIsDefValid(t, id, "bar") - assertIsDefInvalid(t, id, "basta") -} - -func TestIsEqual(t *testing.T) { - id := IsEqual("foo") - - assertIsDefValid(t, id, "foo") - assertIsDefInvalid(t, id, "bar") -} - -func TestRegisteredIsEqual(t *testing.T) { - // Time equality comes from a registered function - // so this is a quick way to test registered functions - now := time.Now() - id := IsEqual(now) - - assertIsDefValid(t, id, now) - assertIsDefInvalid(t, id, now.Add(100)) -} - -func TestIsString(t *testing.T) { - assertIsDefValid(t, IsString, "abc") - assertIsDefValid(t, IsString, "a") - assertIsDefInvalid(t, IsString, 123) -} - -func TestIsNonEmptyString(t *testing.T) { - assertIsDefValid(t, IsNonEmptyString, "abc") - assertIsDefValid(t, IsNonEmptyString, "a") - assertIsDefInvalid(t, IsNonEmptyString, "") - assertIsDefInvalid(t, IsString, 123) -} - -func TestIsStringMatching(t *testing.T) { - id := IsStringMatching(regexp.MustCompile(`^f`)) - - assertIsDefValid(t, id, "fall") - assertIsDefInvalid(t, id, "potato") - assertIsDefInvalid(t, IsString, 123) -} - -func TestIsStringContaining(t *testing.T) { - id := IsStringContaining("foo") - - assertIsDefValid(t, id, "foo") - assertIsDefValid(t, id, "a foo b") - assertIsDefInvalid(t, id, "a bar b") - assertIsDefInvalid(t, IsString, 123) -} - -func TestIsDuration(t *testing.T) { - id := IsDuration - - assertIsDefValid(t, id, time.Duration(1)) - assertIsDefInvalid(t, id, "foo") -} - -func TestIsIntGt(t *testing.T) { - id := IsIntGt(100) - - assertIsDefValid(t, id, 101) - assertIsDefInvalid(t, id, 100) - assertIsDefInvalid(t, id, 99) -} - -func TestIsNil(t *testing.T) { - assertIsDefValid(t, IsNil, nil) - assertIsDefInvalid(t, IsNil, "foo") -} - -func TestIsUnique(t *testing.T) { - tests := []struct { - name string - validator func() Validator - data common.MapStr - isValid bool - }{ - { - "IsUnique find dupes", - func() Validator { - v := IsUnique() - return MustCompile(Map{"a": v, "b": v}) - }, - common.MapStr{"a": 1, "b": 1}, - false, - }, - { - "IsUnique separate instances don't care about dupes", - func() Validator { return MustCompile(Map{"a": IsUnique(), "b": IsUnique()}) }, - common.MapStr{"a": 1, "b": 1}, - true, - }, - { - "IsUniqueTo duplicates across namespaces fail", - func() Validator { - s := ScopedIsUnique() - return MustCompile(Map{"a": s.IsUniqueTo("test"), "b": s.IsUniqueTo("test2")}) - }, - common.MapStr{"a": 1, "b": 1}, - false, - }, - - { - "IsUniqueTo duplicates within a namespace succeeds", - func() Validator { - s := ScopedIsUnique() - return MustCompile(Map{"a": s.IsUniqueTo("test"), "b": s.IsUniqueTo("test")}) - }, - common.MapStr{"a": 1, "b": 1}, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - if tt.isValid { - Test(t, tt.validator(), tt.data) - } else { - result := tt.validator()(tt.data) - require.False(t, result.Valid) - } - }) - } -} diff --git a/libbeat/common/mapval/path.go b/libbeat/common/mapval/path.go deleted file mode 100644 index 4ae5fbb2724a..000000000000 --- a/libbeat/common/mapval/path.go +++ /dev/null @@ -1,194 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "fmt" - "reflect" - "regexp" - "strconv" - "strings" - - "github.com/elastic/beats/libbeat/common" -) - -// pathComponentType indicates the type of pathComponent. -type pathComponentType int - -const ( - // pcMapKey is the Type for map keys. - pcMapKey pathComponentType = 1 + iota - // pcSliceIdx is the Type for slice indices. - pcSliceIdx -) - -func (pct pathComponentType) String() string { - if pct == pcMapKey { - return "map" - } else if pct == pcSliceIdx { - return "slice" - } else { - // This should never happen, but we don't want to return an - // error since that would unnecessarily complicate the fluid API - return "" - } -} - -// pathComponent structs represent one breadcrumb in a path. -type pathComponent struct { - Type pathComponentType // One of pcMapKey or pcSliceIdx - Key string // Populated for maps - Index int // Populated for slices -} - -func (pc pathComponent) String() string { - if pc.Type == pcSliceIdx { - return fmt.Sprintf("[%d]", pc.Index) - } - return pc.Key -} - -// path represents the path within a nested set of maps. -type path []pathComponent - -// extendSlice is used to add a new pathComponent of the pcSliceIdx type. -func (p path) extendSlice(index int) path { - return p.extend( - pathComponent{pcSliceIdx, "", index}, - ) -} - -// extendMap adds a new pathComponent of the pcMapKey type. -func (p path) extendMap(key string) path { - return p.extend( - pathComponent{pcMapKey, key, -1}, - ) -} - -func (p path) extend(pc pathComponent) path { - out := make(path, len(p)+1) - copy(out, p) - out[len(p)] = pc - return out -} - -// concat combines two paths into a new path without modifying any existing paths. -func (p path) concat(other path) path { - out := make(path, 0, len(p)+len(other)) - out = append(out, p...) - return append(out, other...) -} - -func (p path) String() string { - out := make([]string, len(p)) - for idx, pc := range p { - out[idx] = pc.String() - } - return strings.Join(out, ".") -} - -// last returns a pointer to the last pathComponent in this path. If the path empty, -// a nil pointer is returned. -func (p path) last() *pathComponent { - idx := len(p) - 1 - if idx < 0 { - return nil - } - return &p[len(p)-1] -} - -// getFrom takes a map and fetches the given path from it. -func (p path) getFrom(m common.MapStr) (value interface{}, exists bool) { - value = m - exists = true - for _, pc := range p { - rt := reflect.TypeOf(value) - switch rt.Kind() { - case reflect.Map: - converted := interfaceToMapStr(value) - value, exists = converted[pc.Key] - case reflect.Slice: - converted := sliceToSliceOfInterfaces(value) - if pc.Index < len(converted) { - exists = true - value = converted[pc.Index] - } else { - exists = false - value = nil - } - default: - // If this case has been reached this means the expected type, say a map, - // is actually something else, like a string or an array. In this case we - // simply say the value doesn't exist. From a practical perspective this is - // the right behavior since it will cause validation to fail. - return nil, false - } - - if exists == false { - return nil, exists - } - } - - return value, exists -} - -var arrMatcher = regexp.MustCompile("\\[(\\d+)\\]") - -// InvalidPathString is the error type returned from unparseable paths. -type InvalidPathString string - -func (ps InvalidPathString) Error() string { - return fmt.Sprintf("Invalid path path: %#v", ps) -} - -// parsePath parses a path of form key.[0].otherKey.[1] into a path object. -func parsePath(in string) (p path, err error) { - keyParts := strings.Split(in, ".") - - p = make(path, len(keyParts)) - for idx, part := range keyParts { - r := arrMatcher.FindStringSubmatch(part) - pc := pathComponent{Index: -1} - if len(r) > 0 { - pc.Type = pcSliceIdx - // Cannot fail, validated by regexp already - pc.Index, err = strconv.Atoi(r[1]) - if err != nil { - return p, err - } - } else if len(part) > 0 { - pc.Type = pcMapKey - pc.Key = part - } else { - return p, InvalidPathString(in) - } - - p[idx] = pc - } - - return p, nil -} - -// mustParsePath is a convenience method for parsing paths that have been previously validated -func mustParsePath(in string) path { - out, err := parsePath(in) - if err != nil { - panic(err) - } - return out -} diff --git a/libbeat/common/mapval/path_test.go b/libbeat/common/mapval/path_test.go deleted file mode 100644 index 8cb5949a419d..000000000000 --- a/libbeat/common/mapval/path_test.go +++ /dev/null @@ -1,324 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "reflect" - "testing" - - "github.com/elastic/beats/libbeat/common" -) - -func TestPathComponentType_String(t *testing.T) { - tests := []struct { - name string - pct pathComponentType - want string - }{ - { - "Should return the correct type", - pcMapKey, - "map", - }, - { - "Should return the correct type", - pcSliceIdx, - "slice", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.pct.String(); got != tt.want { - t.Errorf("pathComponentType.String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPathComponent_String(t *testing.T) { - type fields struct { - Type pathComponentType - Key string - Index int - } - tests := []struct { - name string - fields fields - want string - }{ - { - "Map key should return a literal", - fields{pcMapKey, "foo", 0}, - "foo", - }, - { - "Array index should return a bracketed number", - fields{pcSliceIdx, "", 123}, - "[123]", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pc := pathComponent{ - Type: tt.fields.Type, - Key: tt.fields.Key, - Index: tt.fields.Index, - } - if got := pc.String(); got != tt.want { - t.Errorf("pathComponent.String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_ExtendSlice(t *testing.T) { - type args struct { - index int - } - tests := []struct { - name string - p path - args args - want path - }{ - { - "Extending an empty slice", - path{}, - args{123}, - path{pathComponent{pcSliceIdx, "", 123}}, - }, - { - "Extending a non-empty slice", - path{pathComponent{pcMapKey, "foo", -1}}, - args{123}, - path{pathComponent{pcMapKey, "foo", -1}, pathComponent{pcSliceIdx, "", 123}}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.p.extendSlice(tt.args.index); !reflect.DeepEqual(got, tt.want) { - t.Errorf("path.extendSlice() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_ExtendMap(t *testing.T) { - type args struct { - key string - } - tests := []struct { - name string - p path - args args - want path - }{ - { - "Extending an empty slice", - path{}, - args{"foo"}, - path{pathComponent{pcMapKey, "foo", -1}}, - }, - { - "Extending a non-empty slice", - path{}.extendMap("foo"), - args{"bar"}, - path{pathComponent{pcMapKey, "foo", -1}, pathComponent{pcMapKey, "bar", -1}}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.p.extendMap(tt.args.key); !reflect.DeepEqual(got, tt.want) { - t.Errorf("path.extendMap() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_Concat(t *testing.T) { - tests := []struct { - name string - p path - arg path - want path - }{ - { - "simple", - path{}.extendMap("foo"), - path{}.extendSlice(123), - path{}.extendMap("foo").extendSlice(123), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.p.concat(tt.arg); !reflect.DeepEqual(got, tt.want) { - t.Errorf("path.concat() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_String(t *testing.T) { - tests := []struct { - name string - p path - want string - }{ - { - "empty", - path{}, - "", - }, - { - "one element", - path{}.extendMap("foo"), - "foo", - }, - { - "complex", - path{}.extendMap("foo").extendSlice(123).extendMap("bar"), - "foo.[123].bar", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.p.String(); got != tt.want { - t.Errorf("path.String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_Last(t *testing.T) { - tests := []struct { - name string - p path - want *pathComponent - }{ - { - "empty path", - path{}, - nil, - }, - { - "one element", - path{}.extendMap("foo"), - &pathComponent{pcMapKey, "foo", -1}, - }, - { - "many elements", - path{}.extendMap("foo").extendMap("bar").extendSlice(123), - &pathComponent{pcSliceIdx, "", 123}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.p.last(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("path.last() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPath_GetFrom(t *testing.T) { - fooPath := path{}.extendMap("foo") - complexPath := path{}.extendMap("foo").extendSlice(0).extendMap("bar").extendSlice(1) - tests := []struct { - name string - p path - arg common.MapStr - wantValue interface{} - wantExists bool - }{ - { - "simple present", - fooPath, - common.MapStr{"foo": "bar"}, - "bar", - true, - }, - { - "simple missing", - fooPath, - common.MapStr{}, - nil, - false, - }, - { - "complex present", - complexPath, - common.MapStr{"foo": []interface{}{common.MapStr{"bar": []string{"bad", "good"}}}}, - "good", - true, - }, - { - "complex missing", - complexPath, - common.MapStr{}, - nil, - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotValue, gotExists := tt.p.getFrom(tt.arg) - if !reflect.DeepEqual(gotValue, tt.wantValue) { - t.Errorf("path.getFrom() gotValue = %v, want %v", gotValue, tt.wantValue) - } - if gotExists != tt.wantExists { - t.Errorf("path.getFrom() gotExists = %v, want %v", gotExists, tt.wantExists) - } - }) - } -} - -func TestParsePath(t *testing.T) { - tests := []struct { - name string - arg string - wantP path - wantErr bool - }{ - { - "simple", - "foo", - path{}.extendMap("foo"), - false, - }, - { - "complex", - "foo.[0].bar.[1].baz", - path{}.extendMap("foo").extendSlice(0).extendMap("bar").extendSlice(1).extendMap("baz"), - false, - }, - // TODO: The validation and testing for this needs to be better - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotP, err := parsePath(tt.arg) - if (err != nil) != tt.wantErr { - t.Errorf("parsePath() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(gotP, tt.wantP) { - t.Errorf("parsePath() = %v, want %v", gotP, tt.wantP) - } - }) - } -} diff --git a/libbeat/common/mapval/reflect_tools.go b/libbeat/common/mapval/reflect_tools.go deleted file mode 100644 index c97c4b650dcf..000000000000 --- a/libbeat/common/mapval/reflect_tools.go +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "reflect" - - "github.com/elastic/beats/libbeat/common" -) - -func interfaceToMapStr(o interface{}) common.MapStr { - newMap := common.MapStr{} - rv := reflect.ValueOf(o) - - for _, key := range rv.MapKeys() { - mapV := rv.MapIndex(key) - keyStr := key.Interface().(string) - var value interface{} - - if !mapV.IsNil() { - value = mapV.Interface().(interface{}) - } - - newMap[keyStr] = value - } - return newMap -} - -func sliceToSliceOfInterfaces(o interface{}) []interface{} { - rv := reflect.ValueOf(o) - converted := make([]interface{}, rv.Len()) - for i := 0; i < rv.Len(); i++ { - var indexV = rv.Index(i) - var convertedValue interface{} - if indexV.Type().Kind() == reflect.Interface { - if !indexV.IsNil() { - convertedValue = indexV.Interface().(interface{}) - } else { - convertedValue = nil - } - } else { - convertedValue = indexV.Interface().(interface{}) - } - converted[i] = convertedValue - } - return converted -} diff --git a/libbeat/common/mapval/results.go b/libbeat/common/mapval/results.go deleted file mode 100644 index 07c8a37b8afb..000000000000 --- a/libbeat/common/mapval/results.go +++ /dev/null @@ -1,139 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import "fmt" - -// Results the results of executing a schema. -// They are a flattened map (using dotted paths) of all the values []ValueResult representing the results -// of the IsDefs. -type Results struct { - Fields map[string][]ValueResult - Valid bool -} - -// NewResults creates a new Results object. -func NewResults() *Results { - return &Results{ - Fields: make(map[string][]ValueResult), - Valid: true, - } -} - -// SimpleResult provides a convenient and simple method for creating a *Results object for a single validation. -// It's a very common way for validators to return a *Results object, and is generally simpler than -// using SingleResult. -func SimpleResult(path path, valid bool, msg string, args ...interface{}) *Results { - vr := ValueResult{valid, fmt.Sprintf(msg, args...)} - return SingleResult(path, vr) -} - -// SingleResult returns a *Results object with a single validated value at the given path -// using the provided ValueResult as its sole validation. -func SingleResult(path path, result ValueResult) *Results { - r := NewResults() - r.record(path, result) - return r -} - -func (r *Results) merge(other *Results) { - for path, valueResults := range other.Fields { - for _, valueResult := range valueResults { - r.record(mustParsePath(path), valueResult) - } - } -} - -func (r *Results) mergeUnderPrefix(prefix path, other *Results) { - if len(prefix) == 0 { - // If the prefix is empty, just use standard merge - // No need to add the dots - r.merge(other) - return - } - - for path, valueResults := range other.Fields { - for _, valueResult := range valueResults { - parsed := mustParsePath(path) - r.record(prefix.concat(parsed), valueResult) - } - } -} - -func (r *Results) record(path path, result ValueResult) { - if r.Fields[path.String()] == nil { - r.Fields[path.String()] = []ValueResult{result} - } else { - r.Fields[path.String()] = append(r.Fields[path.String()], result) - } - - if !result.Valid { - r.Valid = false - } -} - -// EachResult executes the given callback once per Value result. -// The provided callback can return true to keep iterating, or false -// to stop. -func (r Results) EachResult(f func(path, ValueResult) bool) { - for path, pathResults := range r.Fields { - for _, result := range pathResults { - if !f(mustParsePath(path), result) { - return - } - } - } -} - -// DetailedErrors returns a new Results object consisting only of error data. -func (r *Results) DetailedErrors() *Results { - errors := NewResults() - r.EachResult(func(path path, vr ValueResult) bool { - if !vr.Valid { - errors.record(path, vr) - } - - return true - }) - return errors -} - -// ValueResultError is used to represent an error validating an individual value. -type ValueResultError struct { - path path - valueResult ValueResult -} - -// Error returns the error that occurred during validation with its context included. -func (vre ValueResultError) Error() string { - return fmt.Sprintf("@path '%s': %s", vre.path, vre.valueResult.Message) -} - -// Errors returns a list of error objects, one per failed value validation. -func (r Results) Errors() []error { - errors := make([]error, 0) - - r.EachResult(func(path path, vr ValueResult) bool { - if !vr.Valid { - errors = append(errors, ValueResultError{path, vr}) - } - return true - }) - - return errors -} diff --git a/libbeat/common/mapval/results_test.go b/libbeat/common/mapval/results_test.go deleted file mode 100644 index 8930d8b15da9..000000000000 --- a/libbeat/common/mapval/results_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestEmpty(t *testing.T) { - r := NewResults() - assert.True(t, r.Valid) - assert.Empty(t, r.DetailedErrors().Fields) - assert.Empty(t, r.Errors()) -} - -func TestWithError(t *testing.T) { - r := NewResults() - r.record(mustParsePath("foo"), KeyMissingVR) - r.record(mustParsePath("bar"), ValidVR) - - assert.False(t, r.Valid) - - assert.Equal(t, KeyMissingVR, r.Fields["foo"][0]) - assert.Equal(t, ValidVR, r.Fields["bar"][0]) - - assert.Equal(t, KeyMissingVR, r.DetailedErrors().Fields["foo"][0]) - assert.NotContains(t, r.DetailedErrors().Fields, "bar") - - assert.False(t, r.DetailedErrors().Valid) - assert.NotEmpty(t, r.Errors()) -} diff --git a/libbeat/common/mapval/testing.go b/libbeat/common/mapval/testing.go deleted file mode 100644 index e318b5d5c8ac..000000000000 --- a/libbeat/common/mapval/testing.go +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" -) - -// Test takes the output from a Validator invocation and runs test assertions on the result. -// If you are using this library for testing you will probably want to run Test(t, Compile(Map{...}), actual) as a pattern. -func Test(t *testing.T, v Validator, m common.MapStr) *Results { - r := v(m) - - if !r.Valid { - assert.Fail( - t, - "mapval could not validate map", - "%d errors validating source: \n%s", len(r.Errors()), spew.Sdump(m), - ) - } - - for _, err := range r.Errors() { - assert.NoError(t, err) - } - return r -} diff --git a/libbeat/common/mapval/value.go b/libbeat/common/mapval/value.go deleted file mode 100644 index 73d066582f7b..000000000000 --- a/libbeat/common/mapval/value.go +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -// ValueResult represents the result of checking a leaf value. -type ValueResult struct { - Valid bool - Message string // Reason this is invalid -} - -// A ValueValidator is used to validate a value in a Map. -type ValueValidator func(path path, v interface{}) *Results - -// An IsDef defines the type of check to do. -// Generally only name and checker are set. optional and checkKeyMissing are -// needed for weird checks like key presence. -type IsDef struct { - name string - checker ValueValidator - optional bool - checkKeyMissing bool -} - -func (id IsDef) check(path path, v interface{}, keyExists bool) *Results { - if id.checkKeyMissing { - if !keyExists { - return ValidResult(path) - } - - return SimpleResult(path, false, "this key should not exist") - } - - if !id.optional && !keyExists { - return KeyMissingResult(path) - } - - if id.checker != nil { - return id.checker(path, v) - } - - return ValidResult(path) -} - -// ValidResult is a convenience value for Valid results. -func ValidResult(path path) *Results { - return SimpleResult(path, true, "is valid") -} - -// ValidVR is a convenience value for Valid results. -var ValidVR = ValueResult{true, "is valid"} - -// KeyMissingResult is emitted when a key was expected, but was not present. -func KeyMissingResult(path path) *Results { - return SingleResult(path, KeyMissingVR) -} - -// KeyMissingVR is emitted when a key was expected, but was not present. -var KeyMissingVR = ValueResult{ - false, - "expected this key to be present", -} - -// StrictFailureResult is emitted when Strict() is used, and an unexpected field is found. -func StrictFailureResult(path path) *Results { - return SingleResult(path, StrictFailureVR) -} - -// StrictFailureVR is emitted when Strict() is used, and an unexpected field is found. -var StrictFailureVR = ValueResult{ - false, - "unexpected field encountered during strict validation", -} diff --git a/libbeat/common/mapval/walk.go b/libbeat/common/mapval/walk.go deleted file mode 100644 index 2ec278b5ff0c..000000000000 --- a/libbeat/common/mapval/walk.go +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mapval - -import ( - "reflect" - - "github.com/elastic/beats/libbeat/common" -) - -type walkObserverInfo struct { - key pathComponent - value interface{} - rootMap common.MapStr - path path -} - -// walkObserver functions run once per object in the tree. -type walkObserver func(info walkObserverInfo) error - -// walk is a shorthand way to walk a tree. -func walk(m common.MapStr, expandPaths bool, wo walkObserver) error { - return walkFullMap(m, m, path{}, expandPaths, wo) -} - -func walkFull(o interface{}, root common.MapStr, path path, expandPaths bool, wo walkObserver) (err error) { - lastPathComponent := path.last() - if lastPathComponent == nil { - panic("Attempted to traverse an empty path in mapval.walkFull, this should never happen.") - } - - err = wo(walkObserverInfo{*lastPathComponent, o, root, path}) - if err != nil { - return err - } - - switch reflect.TypeOf(o).Kind() { - case reflect.Map: - converted := interfaceToMapStr(o) - err := walkFullMap(converted, root, path, expandPaths, wo) - if err != nil { - return err - } - case reflect.Slice: - converted := sliceToSliceOfInterfaces(o) - - for idx, v := range converted { - newPath := path.extendSlice(idx) - err := walkFull(v, root, newPath, expandPaths, wo) - if err != nil { - return err - } - } - } - - return nil -} - -// walkFullMap walks the given MapStr tree. -func walkFullMap(m common.MapStr, root common.MapStr, p path, expandPaths bool, wo walkObserver) (err error) { - for k, v := range m { - var newPath path - if !expandPaths { - newPath = p.extendMap(k) - } else { - additionalPath, err := parsePath(k) - if err != nil { - return err - } - newPath = p.concat(additionalPath) - } - - err = walkFull(v, root, newPath, expandPaths, wo) - if err != nil { - return err - } - } - - return nil -} diff --git a/packetbeat/flows/worker_test.go b/packetbeat/flows/worker_test.go index 87656139cc39..c6d59c849a64 100644 --- a/packetbeat/flows/worker_test.go +++ b/packetbeat/flows/worker_test.go @@ -24,8 +24,11 @@ import ( "testing" "time" + "github.com/elastic/go-lookslike/isdef" + + "github.com/elastic/go-lookslike" + "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/mapval" "github.com/elastic/beats/libbeat/logp" ) @@ -66,36 +69,36 @@ func TestCreateEvent(t *testing.T) { event := createEvent(time.Now(), bif, true, nil, []string{"bytes", "packets"}, nil) // Validate the contents of the event. - validate := mapval.MustCompile(mapval.Map{ - "source": mapval.Map{ + validate := lookslike.MustCompile(map[string]interface{}{ + "source": map[string]interface{}{ "mac": "01:02:03:04:05:06", "ip": "203.0.113.3", "port": port1, "bytes": uint64(10), "packets": uint64(1), }, - "destination": mapval.Map{ + "destination": map[string]interface{}{ "mac": "06:05:04:03:02:01", "ip": "198.51.100.2", "port": port2, "bytes": uint64(460), "packets": uint64(2), }, - "flow": mapval.Map{ - "id": mapval.KeyPresent, + "flow": map[string]interface{}{ + "id": isdef.KeyPresent, "final": true, - "vlan": mapval.KeyPresent, + "vlan": isdef.KeyPresent, }, - "network": mapval.Map{ + "network": map[string]interface{}{ "bytes": uint64(470), "packets": uint64(3), "type": "ipv4", "transport": "tcp", }, - "event": mapval.Map{ - "start": mapval.KeyPresent, - "end": mapval.KeyPresent, - "duration": mapval.KeyPresent, + "event": map[string]interface{}{ + "start": isdef.KeyPresent, + "end": isdef.KeyPresent, + "duration": isdef.KeyPresent, "dataset": "flow", }, "type": "flow", diff --git a/x-pack/filebeat/module/cisco/ios/pipeline_test.go b/x-pack/filebeat/module/cisco/ios/pipeline_test.go index c3e92afc8a7c..fb1f4f8237a5 100644 --- a/x-pack/filebeat/module/cisco/ios/pipeline_test.go +++ b/x-pack/filebeat/module/cisco/ios/pipeline_test.go @@ -9,9 +9,13 @@ import ( "fmt" "testing" + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/isdef" + + "github.com/elastic/go-lookslike/validator" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/mapval" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/script/javascript" @@ -30,13 +34,13 @@ var logInputHeaders = []string{ type testCase struct { message string - validator mapval.Validator + validator validator.Validator } var testCases = []testCase{ { "%SEC-6-IPACCESSLOGP: list 100 denied udp 198.51.100.1(55934) -> 198.51.100.255(15600), 1 packet", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.access_list": "100", "cisco.ios.facility": "SEC", "destination.ip": "198.51.100.255", @@ -47,9 +51,9 @@ var testCases = []testCase{ "event.severity": int64(6), "event.type": "firewall", "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list 100 denied udp 198.51.100.1(55934) -> 198.51.100.255(15600), 1 packet", - "network.community_id": mapval.IsNonEmptyString, + "network.community_id": isdef.IsNonEmptyString, "network.packets": int64(1), "network.transport": "udp", "source.ip": "198.51.100.1", @@ -60,7 +64,7 @@ var testCases = []testCase{ { "%SEC-6-IPACCESSLOGDP: list 100 denied icmp 198.51.100.1 -> 198.51.100.2 (3/5), 1 packet", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.access_list": "100", "cisco.ios.facility": "SEC", "destination.ip": "198.51.100.2", @@ -72,9 +76,9 @@ var testCases = []testCase{ "icmp.code": int64(5), "icmp.type": int64(3), "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list 100 denied icmp 198.51.100.1 -> 198.51.100.2 (3/5), 1 packet", - "network.community_id": mapval.IsNonEmptyString, + "network.community_id": isdef.IsNonEmptyString, "network.packets": int64(1), "network.transport": "icmp", "source.ip": "198.51.100.1", @@ -84,7 +88,7 @@ var testCases = []testCase{ { "%SEC-6-IPACCESSLOGRP: list 170 denied igmp 198.51.100.1 -> 224.168.168.168, 1 packet", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.access_list": "170", "cisco.ios.facility": "SEC", "destination.ip": "224.168.168.168", @@ -94,9 +98,9 @@ var testCases = []testCase{ "event.severity": int64(6), "event.type": "firewall", "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list 170 denied igmp 198.51.100.1 -> 224.168.168.168, 1 packet", - "network.community_id": mapval.IsNonEmptyString, + "network.community_id": isdef.IsNonEmptyString, "network.packets": int64(1), "network.transport": "igmp", "source.ip": "198.51.100.1", @@ -106,7 +110,7 @@ var testCases = []testCase{ { "%SEC-6-IPACCESSLOGSP: list INBOUND-ON-AP denied igmp 198.51.100.1 -> 224.0.0.2 (20), 1 packet", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.access_list": "INBOUND-ON-AP", "cisco.ios.facility": "SEC", "destination.ip": "224.0.0.2", @@ -117,9 +121,9 @@ var testCases = []testCase{ "event.type": "firewall", "igmp.type": int64(20), "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list INBOUND-ON-AP denied igmp 198.51.100.1 -> 224.0.0.2 (20), 1 packet", - "network.community_id": mapval.IsNonEmptyString, + "network.community_id": isdef.IsNonEmptyString, "network.packets": int64(1), "network.transport": "igmp", "source.ip": "198.51.100.1", @@ -129,7 +133,7 @@ var testCases = []testCase{ { "%SEC-6-IPACCESSLOGNP: list 1 permitted 0 198.51.100.1 -> 239.10.10.10, 1 packet", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.access_list": "1", "cisco.ios.facility": "SEC", "destination.ip": "239.10.10.10", @@ -139,9 +143,9 @@ var testCases = []testCase{ "event.severity": int64(6), "event.type": "firewall", "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list 1 permitted 0 198.51.100.1 -> 239.10.10.10, 1 packet", - "network.community_id": mapval.IsNonEmptyString, + "network.community_id": isdef.IsNonEmptyString, "network.packets": int64(1), "network.iana_number": "0", "source.ip": "198.51.100.1", @@ -151,24 +155,24 @@ var testCases = []testCase{ { "%SEC-6-IPACCESSLOGRL: access-list logging rate-limited or missed 18 packets", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.facility": "SEC", "event.code": "IPACCESSLOGRL", "event.severity": int64(6), "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "access-list logging rate-limited or missed 18 packets", }), }, { "%IPV6-6-ACCESSLOGP: list ACL-IPv6-E0/0-IN/10 permitted tcp 2001:DB8::3(1027) -> 2001:DB8:1000::1(22), 9 packets", - mapval.MustCompile(map[string]interface{}{ + lookslike.MustCompile(map[string]interface{}{ "cisco.ios.facility": "IPV6", "event.code": "ACCESSLOGP", "event.severity": int64(6), "log.level": "informational", - "log.original": mapval.IsNonEmptyString, + "log.original": isdef.IsNonEmptyString, "message": "list ACL-IPv6-E0/0-IN/10 permitted tcp 2001:DB8::3(1027) -> 2001:DB8:1000::1(22), 9 packets", }), }, From ac84b3bc54a120484a70d07dc4a86e05dffd193c Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Mon, 21 Oct 2019 14:32:27 -0500 Subject: [PATCH 2/4] Add changelog --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 2d740169c100..ad85d61d707f 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -25,6 +25,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Build docker and kubernetes features only on supported platforms. {pull}13509[13509] - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509] - The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610] +- Altered all remaining uses of mapval to use the renamed and enhanced version: [go-lookslike](https://github.com/elastic/go-lookslike) instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165] ==== Bugfixes From b92b64b4c163f7c212d66c004bb5981772351bda Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Mon, 21 Oct 2019 14:33:10 -0500 Subject: [PATCH 3/4] Fix imports --- x-pack/filebeat/module/cisco/ios/pipeline_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/module/cisco/ios/pipeline_test.go b/x-pack/filebeat/module/cisco/ios/pipeline_test.go index fb1f4f8237a5..147cc72bc484 100644 --- a/x-pack/filebeat/module/cisco/ios/pipeline_test.go +++ b/x-pack/filebeat/module/cisco/ios/pipeline_test.go @@ -9,16 +9,14 @@ import ( "fmt" "testing" - "github.com/elastic/go-lookslike" - "github.com/elastic/go-lookslike/isdef" - - "github.com/elastic/go-lookslike/validator" - "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/script/javascript" + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/isdef" + "github.com/elastic/go-lookslike/validator" // Register JS "require" modules. _ "github.com/elastic/beats/libbeat/processors/script/javascript/module" From 0f1e101fb0b5f2dc1d733b8924839e2240e32ed5 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Mon, 21 Oct 2019 15:05:56 -0500 Subject: [PATCH 4/4] Use asciidoc syntax --- CHANGELOG-developer.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index ad85d61d707f..110b12647432 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -25,7 +25,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Build docker and kubernetes features only on supported platforms. {pull}13509[13509] - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509] - The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610] -- Altered all remaining uses of mapval to use the renamed and enhanced version: [go-lookslike](https://github.com/elastic/go-lookslike) instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165] +- Altered all remaining uses of mapval to use the renamed and enhanced version: https://github.com/elastic/go-lookslike[go-lookslike] instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165] ==== Bugfixes