Add support for constant_keyword "value" in field definition (#386)

For the constant_keyword field type Elasticsearch allows for the value to be specified in the mapping. If the value is not specified in the mapping then the value is set based on the first document that is indexed. This adds support for specifying a constant_keyword "value" in a field definition. https://www.elastic.co/guide/en/elasticsearch//reference/7.12/keyword.html#constant-keyword-params Package data streams will be able to explicitly declare a constant_keyword value and will not have to include the field with every document sent. This gives packages more control over constant_keyword field values. It can be used optimize storage for an integration since you could omit the constant_keyword field entirely an event's _source. Relates: elastic/package-spec#194 Co-authored-by: Marcin Tojek <[email protected]>
elastic · Jun 28, 2021 · 2278b9e · 2278b9e
1 parent 229667e
commit 2278b9e
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 9 deletions.
diff --git a/go.mod b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.9.0
 	github.com/elastic/go-licenser v0.3.1
 	github.com/elastic/go-ucfg v0.8.3
-	github.com/elastic/package-spec/code/go v0.0.0-20210622082315-342d612eb2a9
+	github.com/elastic/package-spec/code/go v0.0.0-20210623152222-b358e974b7f9
 	github.com/fatih/color v1.10.0
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0

diff --git a/go.sum b/go.sum
@@ -204,8 +204,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE
 github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
 github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo=
 github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
-github.com/elastic/package-spec/code/go v0.0.0-20210622082315-342d612eb2a9 h1:Q6hJeu5XL7C65QIW6OTYYF7ViyedrTfu6QnFW9xppjk=
-github.com/elastic/package-spec/code/go v0.0.0-20210622082315-342d612eb2a9/go.mod h1:t0uvhLQGg3D4iQ5lSQEQs4YYS53MIIS05v0zm0fIBPM=
+github.com/elastic/package-spec/code/go v0.0.0-20210623152222-b358e974b7f9 h1:qvoqy6W/mhBY1t4xxP82oy34VTeF+MEqfVLiIGNBsEs=
+github.com/elastic/package-spec/code/go v0.0.0-20210623152222-b358e974b7f9/go.mod h1:t0uvhLQGg3D4iQ5lSQEQs4YYS53MIIS05v0zm0fIBPM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

diff --git a/internal/fields/model.go b/internal/fields/model.go
@@ -9,6 +9,7 @@ type FieldDefinition struct {
 	Name        string            `yaml:"name"`
 	Description string            `yaml:"description"`
 	Type        string            `yaml:"type"`
+	Value       string            `yaml:"value"` // The value to associate with a constant_keyword field.
 	Pattern     string            `yaml:"pattern"`
 	Unit        string            `yaml:"unit"`
 	MetricType  string            `yaml:"metric_type"`

diff --git a/internal/fields/testdata/constant-keyword-invalid.json b/internal/fields/testdata/constant-keyword-invalid.json
@@ -0,0 +1,5 @@
+{
+  "foo": {
+    "constant": "wrong"
+  }
+}
diff --git a/internal/fields/testdata/constant-keyword-valid.json b/internal/fields/testdata/constant-keyword-valid.json
@@ -0,0 +1,5 @@
+{
+  "foo": {
+    "constant": "correct"
+  }
+}
diff --git a/internal/fields/testdata/fields/fields.yml b/internal/fields/testdata/fields/fields.yml
@@ -8,3 +8,6 @@
       fields:
         - name: request_parameters
           type: flattened
+    - name: constant
+      type: constant_keyword
+      value: correct
diff --git a/internal/fields/validate.go b/internal/fields/validate.go
@@ -293,19 +293,28 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va
 
 	var valid bool
 	switch definition.Type {
-	case "date", "ip", "constant_keyword", "keyword", "text":
+	case "constant_keyword":
 		var valStr string
 		valStr, valid = val.(string)
-		if !valid || definition.Pattern == "" {
+		if !valid {
 			break
 		}
 
-		valid, err := regexp.MatchString(definition.Pattern, valStr)
-		if err != nil {
-			return errors.Wrap(err, "invalid pattern")
+		if err := ensureConstantKeywordValueMatches(key, valStr, definition.Value); err != nil {
+			return err
+		}
+		if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil {
+			return err
 		}
+	case "date", "ip", "keyword", "text":
+		var valStr string
+		valStr, valid = val.(string)
 		if !valid {
-			return fmt.Errorf("field %q's value, %s, does not match the expected pattern: %s", key, valStr, definition.Pattern)
+			break
+		}
+
+		if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil {
+			return err
 		}
 	case "float", "long", "double":
 		_, valid = val.(float64)
@@ -331,3 +340,31 @@ func ensureSingleElementValue(val interface{}) (interface{}, bool) {
 	}
 	return nil, false // false: empty array, can't deduce single value type
 }
+
+// ensurePatternMatches validates the document's field value matches the field
+// definitions regular expression pattern.
+func ensurePatternMatches(key, value, pattern string) error {
+	if pattern == "" {
+		return nil
+	}
+	valid, err := regexp.MatchString(pattern, value)
+	if err != nil {
+		return errors.Wrap(err, "invalid pattern")
+	}
+	if !valid {
+		return fmt.Errorf("field %q's value, %s, does not match the expected pattern: %s", key, value, pattern)
+	}
+	return nil
+}
+
+// ensureConstantKeywordValueMatches validates the document's field value
+// matches the definition's constant_keyword value.
+func ensureConstantKeywordValueMatches(key, value, constantKeywordValue string) error {
+	if constantKeywordValue == "" {
+		return nil
+	}
+	if value != constantKeywordValue {
+		return fmt.Errorf("field %q's value %q does not match the declared constant_keyword value %q", key, value, constantKeywordValue)
+	}
+	return nil
+}
diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go
@@ -61,6 +61,20 @@ func TestValidate_WithNumericKeywordFields(t *testing.T) {
 	require.Empty(t, errs)
 }
 
+func TestValidate_constant_keyword(t *testing.T) {
+	validator, err := CreateValidatorForDataStream("testdata")
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	e := readSampleEvent(t, "testdata/constant-keyword-invalid.json")
+	errs := validator.ValidateDocumentBody(e)
+	require.NotEmpty(t, errs)
+
+	e = readSampleEvent(t, "testdata/constant-keyword-valid.json")
+	errs = validator.ValidateDocumentBody(e)
+	require.Empty(t, errs)
+}
+
 func Test_parseElementValue(t *testing.T) {
 	for _, test := range []struct {
 		key        string