fix: Float grok values and conversion to float expressions now serial…

…ize correctly to float in the Elasticsearch output

lc-lib/event/float_value.go

@@ -0,0 +1,26 @@
+package event
+
+import (
+	"strconv"
+	"strings"
+)
+
+type FloatValue64 float64
+
+func (f FloatValue64) MarshalJSON() ([]byte, error) {
+	result := strconv.FormatFloat(float64(f), 'f', -1, 64)
+	if !strings.ContainsRune(result, '.') {
+		result += ".0"
+	}
+	return []byte(result), nil
+}
+
+type FloatValue32 float32
+
+func (f FloatValue32) MarshalJSON() ([]byte, error) {
+	result := strconv.FormatFloat(float64(f), 'f', -1, 32)
+	if !strings.ContainsRune(result, '.') {
+		result += ".0"
+	}
+	return []byte(result), nil
+}

lc-lib/event/float_value_test.go

@@ -0,0 +1,46 @@
+package event
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestFloatValue64Round(t *testing.T) {
+	result, err := json.Marshal(FloatValue64(1.0))
+	if err != nil {
+		t.Fatalf("Failed to encode: %s", err)
+	}
+	if string(result) != "1.0" {
+		t.Fatalf("Unexpected result: %s", result)
+	}
+}
+
+func TestFloatValue64Decimal(t *testing.T) {
+	result, err := json.Marshal(FloatValue64(1.5))
+	if err != nil {
+		t.Fatalf("Failed to encode: %s", err)
+	}
+	if string(result) != "1.5" {
+		t.Fatalf("Unexpected result: %s", result)
+	}
+}
+
+func TestFloatValue32Round(t *testing.T) {
+	result, err := json.Marshal(FloatValue32(1.0))
+	if err != nil {
+		t.Fatalf("Failed to encode: %s", err)
+	}
+	if string(result) != "1.0" {
+		t.Fatalf("Unexpected result: %s", result)
+	}
+}
+
+func TestFloatValue32Decimal(t *testing.T) {
+	result, err := json.Marshal(FloatValue32(1.5))
+	if err != nil {
+		t.Fatalf("Failed to encode: %s", err)
+	}
+	if string(result) != "1.5" {
+		t.Fatalf("Unexpected result: %s", result)
+	}
+}

lc-lib/grok/types.go

@@ -19,6 +19,8 @@ package grok
 import (
 	"fmt"
 	"strconv"
+
+	"github.com/driskell/log-courier/lc-lib/event"
 )
 
 // TypeHint is a type hint specified in a grok pattern
@@ -43,7 +45,7 @@ func parseType(typeHint string) (TypeHint, error) {
 	case string(TypeHintFloat):
 		return TypeHintFloat, nil
 	}
-	return "", fmt.Errorf("Invalid type hint: %s", typeHint)
+	return "", fmt.Errorf("invalid type hint: %s", typeHint)
 }
 
 // convertToType converts the value to the given type
@@ -58,7 +60,7 @@ func convertToType(value string, typeHint TypeHint) interface{} {
 		return result
 	case TypeHintFloat:
 		result, _ := strconv.ParseFloat(value, 64)
-		return result
+		return event.FloatValue64(result)
 	}
 	return nil
 }

lc-lib/grok/types_test.go

@@ -16,7 +16,11 @@
 
 package grok
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/driskell/log-courier/lc-lib/event"
+)
 
 func TestParseType(t *testing.T) {
 	if typeHint, err := parseType(string(TypeHintString)); err != nil || typeHint != TypeHintString {
@@ -69,43 +73,43 @@ func TestIntType(t *testing.T) {
 
 func TestFloatType(t *testing.T) {
 	result := convertToType("value", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("0123tr", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("0123", TypeHintFloat)
-	if result != 123. {
+	if result != event.FloatValue64(123.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("prefix0123", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("45666666", TypeHintFloat)
-	if result != 45666666. {
+	if result != event.FloatValue64(45666666.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("4566.6666", TypeHintFloat)
-	if result != 4566.6666 {
+	if result != event.FloatValue64(4566.6666) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("tr1.4", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("1.4oth", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("1.4e4", TypeHintFloat)
-	if result != 14000. {
+	if result != event.FloatValue64(14000.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 	result = convertToType("1.4e4e", TypeHintFloat)
-	if result != 0. {
+	if result != event.FloatValue64(0.) {
 		t.Fatalf("Unexpected conversion: %s", result)
 	}
 }

lc-lib/processor/cel.go

@@ -21,6 +21,7 @@ import (
 	"github.com/google/cel-go/checker/decls"
 	celext "github.com/google/cel-go/ext"
 
+	"github.com/driskell/log-courier/lc-lib/event"
 	"github.com/driskell/log-courier/lc-lib/processor/ext"
 )
 
@@ -43,6 +44,16 @@ func cachedCelEnv() (*cel.Env, error) {
 	)
 }
 
+func normalizeType(value interface{}) interface{} {
+	if nativeValue, ok := value.(float64); ok {
+		return event.FloatValue64(nativeValue)
+	}
+	if nativeValue, ok := value.(float32); ok {
+		return event.FloatValue32(nativeValue)
+	}
+	return value
+}
+
 // ParseExpression parses an expression using cel-go and returns the evaluatable program
 func ParseExpression(expression string) (cel.Program, error) {
 	env, err := cachedCelEnv()

lc-lib/processor/cel_test.go

@@ -18,6 +18,8 @@ package processor
 
 import (
 	"testing"
+
+	"github.com/driskell/log-courier/lc-lib/event"
 )
 
 func TestCELFieldAccess(t *testing.T) {
@@ -89,3 +91,48 @@ func TestCELMacroHasNotDeep(t *testing.T) {
 		t.Fatalf("Unexpected value: %v (error %s)", val, err)
 	}
 }
+
+func TestCELFieldTypeNormalizeString(t *testing.T) {
+	program, err := ParseExpression("event.test")
+	if err != nil {
+		t.Fatalf("Unexpected parse error: %s", err)
+	}
+	val, _, err := program.Eval(map[string]interface{}{"event": map[string]interface{}{"test": "test"}})
+	if err != nil {
+		t.Fatalf("Unexpected eval error: %s", err)
+	}
+	normalized := normalizeType(val.Value())
+	if _, ok := normalized.(string); !ok {
+		t.Fatalf("Unexpected normalized type: %t", val)
+	}
+}
+
+func TestCELFieldTypeNormalizeInt(t *testing.T) {
+	program, err := ParseExpression("event.test")
+	if err != nil {
+		t.Fatalf("Unexpected parse error: %s", err)
+	}
+	val, _, err := program.Eval(map[string]interface{}{"event": map[string]interface{}{"test": 123}})
+	if err != nil {
+		t.Fatalf("Unexpected eval error: %s", err)
+	}
+	normalized := normalizeType(val.Value())
+	if _, ok := normalized.(int64); !ok {
+		t.Fatalf("Unexpected normalized type: %t", val)
+	}
+}
+
+func TestCELFieldTypeNormalizeFloat(t *testing.T) {
+	program, err := ParseExpression("event.test")
+	if err != nil {
+		t.Fatalf("Unexpected parse error: %s", err)
+	}
+	val, _, err := program.Eval(map[string]interface{}{"event": map[string]interface{}{"test": 123.0}})
+	if err != nil {
+		t.Fatalf("Unexpected eval error: %s", err)
+	}
+	normalized := normalizeType(val.Value())
+	if _, ok := normalized.(event.FloatValue64); !ok {
+		t.Fatalf("Unexpected normalized type: %t", normalized)
+	}
+}