Validate that event.type is aligned with event.category (#961)

Fields like `event.category` in ECS define a list of allowed values for `event.type`,
check that these values are aligned.

This validation is only enabled for packages using at least format version 2.0.0.

internal/fields/model.go

@@ -205,7 +205,7 @@ func cleanNested(parent *FieldDefinition) (base []FieldDefinition) {
 // AllowedValues is the list of allowed values for a field.
 type AllowedValues []AllowedValue
 
-// Allowed returns true if a given value is allowed.
+// IsAllowed returns true if a given value is allowed.
 func (avs AllowedValues) IsAllowed(value string) bool {
 	if len(avs) == 0 {
 		// No configured allowed values, any value is allowed.
@@ -214,6 +214,37 @@ func (avs AllowedValues) IsAllowed(value string) bool {
 	return common.StringSliceContains(avs.Values(), value)
 }
 
+// IsExpectedEventType returns true if the event type is allowed for the given value.
+// This method can be used to check single values of event type or arrays of them.
+func (avs AllowedValues) IsExpectedEventType(value string, eventType interface{}) bool {
+	expected := avs.ExpectedEventTypes(value)
+	if len(expected) == 0 {
+		// No restrictions defined, all good to go.
+		return true
+	}
+	switch eventType := eventType.(type) {
+	case string:
+		return common.StringSliceContains(expected, eventType)
+	case []interface{}:
+		if len(eventType) == 0 {
+			return false
+		}
+		for _, elem := range eventType {
+			elem, ok := elem.(string)
+			if !ok {
+				return false
+			}
+			if !common.StringSliceContains(expected, elem) {
+				return false
+			}
+		}
+		return true
+	default:
+		// It must be a string, or an array of strings.
+		return false
+	}
+}
+
 // Values returns the list of allowed values.
 func (avs AllowedValues) Values() []string {
 	var values []string
@@ -223,6 +254,18 @@ func (avs AllowedValues) Values() []string {
 	return values
 }
 
+// ExpectedEventTypes returns the list of expected event types for a given value.
+func (avs AllowedValues) ExpectedEventTypes(value string) []string {
+	for _, v := range avs {
+		if v.Name == value {
+			return v.ExpectedEventTypes
+		}
+	}
+
+	// If we are here, IsAllowed(value) is also false.
+	return nil
+}
+
 // AllowedValue is one of the allowed values for a field.
 type AllowedValue struct {
 	Name               string   `yaml:"name"`

internal/fields/testdata/fields/fields.yml

@@ -17,3 +17,50 @@
   type: keyword
   normalize:
   - array
+- name: event.category
+  type: keyword
+  allowed_values:
+  - name: authentication
+    expected_event_types:
+    - start
+    - end
+    - info
+  - name: configuration
+    expected_event_types:
+    - access
+    - change
+    - creation
+    - deletion
+    - info
+  - name: network
+    expected_event_types:
+    - access
+    - allowed
+    - connection
+    - denied
+    - end
+    - info
+    - protocol
+    - start
+- name: event.type
+  type: keyword
+  normalize:
+    - array
+  allowed_values:
+    - name: access
+    - name: admin
+    - name: allowed
+    - name: change
+    - name: connection
+    - name: creation
+    - name: deletion
+    - name: denied
+    - name: end
+    - name: error
+    - name: group
+    - name: indicator
+    - name: info
+    - name: installation
+    - name: protocol
+    - name: start
+    - name: user

internal/fields/validate.go

@@ -26,6 +26,8 @@ import (
 	"github.com/elastic/elastic-package/internal/packages/buildmanifest"
 )
 
+var semver2_0_0 = semver.MustParse("2.0.0")
+
 // Validator is responsible for fields validation.
 type Validator struct {
 	// Schema contains definition records.
@@ -195,31 +197,27 @@ func (v *Validator) ValidateDocumentBody(body json.RawMessage) multierror.Error
 		return errs
 	}
 
-	errs := v.validateMapElement("", c)
-	if len(errs) == 0 {
-		return nil
-	}
-	return errs
+	return v.ValidateDocumentMap(c)
 }
 
 // ValidateDocumentMap validates the provided document as common.MapStr.
 func (v *Validator) ValidateDocumentMap(body common.MapStr) multierror.Error {
-	errs := v.validateMapElement("", body)
+	errs := v.validateMapElement("", body, body)
 	if len(errs) == 0 {
 		return nil
 	}
 	return errs
 }
 
-func (v *Validator) validateMapElement(root string, elem common.MapStr) multierror.Error {
+func (v *Validator) validateMapElement(root string, elem common.MapStr, doc common.MapStr) multierror.Error {
 	var errs multierror.Error
 	for name, val := range elem {
 		key := strings.TrimLeft(root+"."+name, ".")
 
 		switch val := val.(type) {
 		case []map[string]interface{}:
 			for _, m := range val {
-				err := v.validateMapElement(key, m)
+				err := v.validateMapElement(key, m, doc)
 				if err != nil {
 					errs = append(errs, err...)
 				}
@@ -230,12 +228,12 @@ func (v *Validator) validateMapElement(root string, elem common.MapStr) multierr
 				// because the entire object is mapped as a single field.
 				continue
 			}
-			err := v.validateMapElement(key, val)
+			err := v.validateMapElement(key, val, doc)
 			if err != nil {
 				errs = append(errs, err...)
 			}
 		default:
-			err := v.validateScalarElement(key, val)
+			err := v.validateScalarElement(key, val, doc)
 			if err != nil {
 				errs = append(errs, err)
 			}
@@ -244,7 +242,7 @@ func (v *Validator) validateMapElement(root string, elem common.MapStr) multierr
 	return errs
 }
 
-func (v *Validator) validateScalarElement(key string, val interface{}) error {
+func (v *Validator) validateScalarElement(key string, val interface{}, doc common.MapStr) error {
 	if key == "" {
 		return nil // root key is always valid
 	}
@@ -276,7 +274,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error {
 		return errors.Wrapf(err, "field %q is not normalized as expected", key)
 	}
 
-	err = v.parseElementValue(key, *definition, val)
+	err = v.parseElementValue(key, *definition, val, doc)
 	if err != nil {
 		return errors.Wrap(err, "parsing field value failed")
 	}
@@ -384,7 +382,7 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool {
 
 func (v *Validator) validateExpectedNormalization(definition FieldDefinition, val interface{}) error {
 	// Validate expected normalization starting with packages following spec v2 format.
-	if v.specVersion.LessThan(semver.MustParse("2.0.0")) {
+	if v.specVersion.LessThan(semver2_0_0) {
 		return nil
 	}
 	for _, normalize := range definition.Normalize {
@@ -433,11 +431,11 @@ func validSubField(def FieldDefinition, extraPart string) bool {
 
 // parseElementValue checks that the value stored in a field matches the field definition. For
 // arrays it checks it for each Element.
-func (v *Validator) parseElementValue(key string, definition FieldDefinition, val interface{}) error {
-	return forEachElementValue(key, definition, val, v.parseSingleElementValue)
+func (v *Validator) parseElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr) error {
+	return forEachElementValue(key, definition, val, doc, v.parseSingleElementValue)
 }
 
-func (v *Validator) parseSingleElementValue(key string, definition FieldDefinition, val interface{}) error {
+func (v *Validator) parseSingleElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr) error {
 	invalidTypeError := func() error {
 		return fmt.Errorf("field %q's Go type, %T, does not match the expected field type: %s (field value: %v)", key, val, definition.Type, val)
 	}
@@ -461,6 +459,11 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti
 		if err := ensureAllowedValues(key, valStr, definition); err != nil {
 			return err
 		}
+		if !v.specVersion.LessThan(semver2_0_0) {
+			if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil {
+				return err
+			}
+		}
 	// Normal text fields should be of type string.
 	// If a pattern is provided, it checks if the value matches.
 	case "keyword", "text":
@@ -475,6 +478,11 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti
 		if err := ensureAllowedValues(key, valStr, definition); err != nil {
 			return err
 		}
+		if !v.specVersion.LessThan(semver2_0_0) {
+			if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil {
+				return err
+			}
+		}
 	// Dates are expected to be formatted as strings or as seconds or milliseconds
 	// since epoch.
 	// If it is a string and a pattern is provided, it checks if the value matches.
@@ -563,13 +571,13 @@ func (v *Validator) isAllowedIPValue(s string) bool {
 
 // forEachElementValue visits a function for each element in the given value if
 // it is an array. If it is not an array, it calls the function with it.
-func forEachElementValue(key string, definition FieldDefinition, val interface{}, fn func(string, FieldDefinition, interface{}) error) error {
+func forEachElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr, fn func(string, FieldDefinition, interface{}, common.MapStr) error) error {
 	arr, isArray := val.([]interface{})
 	if !isArray {
-		return fn(key, definition, val)
+		return fn(key, definition, val, doc)
 	}
 	for _, element := range arr {
-		err := fn(key, definition, element)
+		err := fn(key, definition, element, doc)
 		if err != nil {
 			return err
 		}
@@ -616,3 +624,14 @@ func ensureAllowedValues(key, value string, definition FieldDefinition) error {
 	}
 	return nil
 }
+
+// ensureExpectedEventType validates that the document's `event.type` field is one of the expected
+// one for the given value.
+func ensureExpectedEventType(key, value string, definition FieldDefinition, doc common.MapStr) error {
+	eventType, _ := doc.GetValue("event.type")
+	if !definition.AllowedValues.IsExpectedEventType(value, eventType) {
+		expected := definition.AllowedValues.ExpectedEventTypes(value)
+		return fmt.Errorf("field \"event.type\" value \"%v\" (%T) is not one of the expected values (%s) for %s=%q", eventType, eventType, strings.Join(expected, ", "), key, value)
+	}
+	return nil
+}

internal/fields/validate_test.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-package/internal/common"
 )
 
 type results struct {
@@ -113,6 +115,56 @@ func TestValidate_WithSpecVersion(t *testing.T) {
 	require.Empty(t, errs)
 }
 
+func TestValidate_ExpectedEventType(t *testing.T) {
+	validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"))
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	cases := []struct {
+		title string
+		doc   common.MapStr
+		valid bool
+	}{
+		{
+			title: "valid event type",
+			doc: common.MapStr{
+				"event.category": "authentication",
+				"event.type":     []interface{}{"info"},
+			},
+			valid: true,
+		},
+		{
+			title: "multiple valid event type",
+			doc: common.MapStr{
+				"event.category": "network",
+				"event.type":     []interface{}{"protocol", "connection", "end"},
+			},
+			valid: true,
+		},
+		{
+			title: "unexpected event type",
+			doc: common.MapStr{
+				"event.category": "authentication",
+				"event.type":     []interface{}{"access"},
+			},
+			valid: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			errs := validator.ValidateDocumentMap(c.doc)
+			if c.valid {
+				assert.Empty(t, errs)
+			} else {
+				if assert.Len(t, errs, 1) {
+					assert.Contains(t, errs[0].Error(), "is not one of the expected values")
+				}
+			}
+		})
+	}
+}
+
 func Test_parseElementValue(t *testing.T) {
 	for _, test := range []struct {
 		key        string
@@ -433,7 +485,7 @@ func Test_parseElementValue(t *testing.T) {
 		}
 
 		t.Run(test.key, func(t *testing.T) {
-			err := v.parseElementValue(test.key, test.definition, test.value)
+			err := v.parseElementValue(test.key, test.definition, test.value, common.MapStr{})
 			if test.fail {
 				require.Error(t, err)
 			} else {