From 2429af154fd2564191b5c7c321318b6518bf637b Mon Sep 17 00:00:00 2001 From: ShourieG <105607378+ShourieG@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:19:07 +0530 Subject: [PATCH] [filebeat][httpjson]- Added min & max functions to the template engine (#36036) * added min/max functions * updated methods to accept integer list * updated the methods to accept atleast 1 arg * extended the min/max functions to be more generic * reworked the min/max implementation * made the min max funcs generic with a custom number ingterface * Implement min/max using text/template lt func --------- Co-authored-by: Andrew Kroh --- CHANGELOG.next.asciidoc | 1 + .../docs/inputs/input-httpjson.asciidoc | 2 + .../filebeat/input/httpjson/texttemplate.go | 107 ++++++++++++++++++ x-pack/filebeat/input/httpjson/value_tpl.go | 28 +++++ .../filebeat/input/httpjson/value_tpl_test.go | 70 ++++++++++++ 5 files changed, 208 insertions(+) create mode 100644 x-pack/filebeat/input/httpjson/texttemplate.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9dd9c35a1a2..14021299b72 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -350,6 +350,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Add device support for Azure AD entity analytics. {pull}35807[35807] - Improve CEL input performance. {pull}35915[35915] - Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044] +- Added support for min/max template functions in httpjson input. {issue}36094[36094] {pull}36036[36036] - Add `clean_session` configuration setting for MQTT input. {pull}35806[16204] - Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734] - Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065] diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 34bcc3b1a7e..45353d7b3ef 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -214,6 +214,8 @@ Some built-in helper functions are provided to work with the input state inside - `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` - `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` - `join`: joins a list using the specified separator. Example: `[[join .body.arr ","]]` +- `max`: returns the maximum of two values. +- `min`: returns the minimum of two values. - `mul`: multiplies two integers. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now. - `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. diff --git a/x-pack/filebeat/input/httpjson/texttemplate.go b/x-pack/filebeat/input/httpjson/texttemplate.go new file mode 100644 index 00000000000..d5f2bf1a346 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/texttemplate.go @@ -0,0 +1,107 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "errors" + "reflect" +) + +// These functions come from Go's text/template/funcs.go (1.19). +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var ( + errBadComparisonType = errors.New("invalid type for comparison") + errBadComparison = errors.New("incompatible types for comparison") +) + +type kind int + +const ( + invalidKind kind = iota + boolKind + complexKind + intKind + floatKind + stringKind + uintKind +) + +func basicKind(v reflect.Value) (kind, error) { + switch v.Kind() { + case reflect.Bool: + return boolKind, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return intKind, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return uintKind, nil + case reflect.Float32, reflect.Float64: + return floatKind, nil + case reflect.Complex64, reflect.Complex128: + return complexKind, nil + case reflect.String: + return stringKind, nil + } + return invalidKind, errBadComparisonType +} + +// indirectInterface returns the concrete value in an interface value, +// or else the zero reflect.Value. +// That is, if v represents the interface value x, the result is the same as reflect.ValueOf(x): +// the fact that x was an interface value is forgotten. +func indirectInterface(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Interface { + return v + } + if v.IsNil() { + return reflect.Value{} + } + return v.Elem() +} + +// lt evaluates the comparison a < b. +func lt(arg1, arg2 reflect.Value) (bool, error) { + arg1 = indirectInterface(arg1) + k1, err := basicKind(arg1) + if err != nil { + return false, err + } + arg2 = indirectInterface(arg2) + k2, err := basicKind(arg2) + if err != nil { + return false, err + } + truth := false + if k1 != k2 { + // Special case: Can compare integer values regardless of type's sign. + switch { + case k1 == intKind && k2 == uintKind: + truth = arg1.Int() < 0 || uint64(arg1.Int()) < arg2.Uint() + case k1 == uintKind && k2 == intKind: + truth = arg2.Int() >= 0 && arg1.Uint() < uint64(arg2.Int()) + default: + return false, errBadComparison + } + } else { + switch k1 { + case boolKind, complexKind: + return false, errBadComparisonType + case floatKind: + truth = arg1.Float() < arg2.Float() + case intKind: + truth = arg1.Int() < arg2.Int() + case stringKind: + truth = arg1.String() < arg2.String() + case uintKind: + truth = arg1.Uint() < arg2.Uint() + default: + panic("invalid kind") + } + } + return truth, nil +} diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index 4decfdc5c19..133271e726f 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -66,6 +66,8 @@ func (t *valueTpl) Unpack(in string) error { "hmacBase64": hmacStringBase64, "join": join, "toJSON": toJSON, + "max": max, + "min": min, "mul": mul, "now": now, "parseDate": parseDate, @@ -295,6 +297,32 @@ func div(a, b int64) int64 { return a / b } +func min(arg1, arg2 reflect.Value) (interface{}, error) { + lessThan, err := lt(arg1, arg2) + if err != nil { + return nil, err + } + + // arg1 is < arg2. + if lessThan { + return arg1.Interface(), nil + } + return arg2.Interface(), nil +} + +func max(arg1, arg2 reflect.Value) (interface{}, error) { + lessThan, err := lt(arg1, arg2) + if err != nil { + return nil, err + } + + // arg1 is < arg2. + if lessThan { + return arg2.Interface(), nil + } + return arg1.Interface(), nil +} + func base64Encode(values ...string) string { data := strings.Join(values, "") if data == "" { diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index aacbda3eede..37589cd8821 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -356,6 +356,76 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "4", }, + { + name: "func min int", + value: `[[min 4 1]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "1", + }, + { + name: "func max int", + value: `[[max 4 1]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "4", + }, + { + name: "func max float", + value: `[[max 1.23 4.666]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "4.666", + }, + { + name: "func min float", + value: `[[min 1.23 4.666]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "1.23", + }, + { + name: "func min string", + value: `[[min "a" "b"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "a", + }, + { + name: "func max string", + value: `[[max "a" "b"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "b", + }, + { + name: "func min int64 unix seconds", + value: `[[ min (now.Unix) 1689771139 ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "1689771139", + }, + { + name: "func min int year", + value: `[[ min (now.Year) 2020 ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020", + }, + { + name: "func max duration", + value: `[[ max (parseDuration "59m") (parseDuration "1h") ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "1h0m0s", + }, + { + name: "func min int ", + value: `[[ min (now.Year) 2020 ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020", + }, { name: "func sha1 hmac Hex", value: `[[hmac "sha1" "secret" "string1" "string2"]]`,