Skip to content

Commit

Permalink
Merge branch 'main' into fix-commented-aws-tests
Browse files Browse the repository at this point in the history
  • Loading branch information
bhapas committed Jul 24, 2023
2 parents acbe41b + 2429af1 commit 9ae7500
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 2 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
- Add device support for Azure AD entity analytics. {pull}35807[35807]
- Improve CEL input performance. {pull}35915[35915]
- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
- Added support for min/max template functions in httpjson input. {issue}36094[36094] {pull}36036[36036]
- Add `clean_session` configuration setting for MQTT input. {pull}35806[16204]
- Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
- Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,8 @@ Some built-in helper functions are provided to work with the input state inside
- `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
- `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
- `join`: joins a list using the specified separator. Example: `[[join .body.arr ","]]`
- `max`: returns the maximum of two values.
- `min`: returns the minimum of two values.
- `mul`: multiplies two integers.
- `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now.
- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
Expand Down
4 changes: 2 additions & 2 deletions x-pack/filebeat/input/awss3/input_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func TestInputRunSQS(t *testing.T) {
assert.EqualValues(t, s3Input.metrics.sqsMessagesDeletedTotal.Get(), 7)
assert.EqualValues(t, s3Input.metrics.sqsMessagesReturnedTotal.Get(), 1) // Invalid JSON is returned so that it can eventually be DLQed.
assert.EqualValues(t, s3Input.metrics.sqsVisibilityTimeoutExtensionsTotal.Get(), 0)
assert.EqualValues(t, s3Input.metrics.sqsMessagesWaiting.Get(), 1) // sqsMessagesWaitingMetric is polled every minute and test completes in less than a minute
assert.EqualValues(t, s3Input.metrics.sqsMessagesWaiting.Get(), 0)
assert.EqualValues(t, s3Input.metrics.s3ObjectsInflight.Get(), 0)
assert.EqualValues(t, s3Input.metrics.s3ObjectsRequestedTotal.Get(), 7)
assert.EqualValues(t, s3Input.metrics.s3EventsCreatedTotal.Get(), 12)
Expand Down Expand Up @@ -426,7 +426,7 @@ func TestInputRunSNS(t *testing.T) {
assert.EqualValues(t, s3Input.metrics.sqsMessagesDeletedTotal.Get(), 7)
assert.EqualValues(t, s3Input.metrics.sqsMessagesReturnedTotal.Get(), 1) // Invalid JSON is returned so that it can eventually be DLQed.
assert.EqualValues(t, s3Input.metrics.sqsVisibilityTimeoutExtensionsTotal.Get(), 0)
assert.EqualValues(t, s3Input.metrics.sqsMessagesWaiting.Get(), 1) // sqsMessagesWaitingMetric is polled every minute and test completes in less than a minute
assert.EqualValues(t, s3Input.metrics.sqsMessagesWaiting.Get(), 0)
assert.EqualValues(t, s3Input.metrics.s3ObjectsInflight.Get(), 0)
assert.EqualValues(t, s3Input.metrics.s3ObjectsRequestedTotal.Get(), 7)
assert.EqualValues(t, s3Input.metrics.s3EventsCreatedTotal.Get(), 12)
Expand Down
107 changes: 107 additions & 0 deletions x-pack/filebeat/input/httpjson/texttemplate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package httpjson

import (
"errors"
"reflect"
)

// These functions come from Go's text/template/funcs.go (1.19).
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

var (
errBadComparisonType = errors.New("invalid type for comparison")
errBadComparison = errors.New("incompatible types for comparison")
)

type kind int

const (
invalidKind kind = iota
boolKind
complexKind
intKind
floatKind
stringKind
uintKind
)

func basicKind(v reflect.Value) (kind, error) {
switch v.Kind() {
case reflect.Bool:
return boolKind, nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return intKind, nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return uintKind, nil
case reflect.Float32, reflect.Float64:
return floatKind, nil
case reflect.Complex64, reflect.Complex128:
return complexKind, nil
case reflect.String:
return stringKind, nil
}
return invalidKind, errBadComparisonType
}

// indirectInterface returns the concrete value in an interface value,
// or else the zero reflect.Value.
// That is, if v represents the interface value x, the result is the same as reflect.ValueOf(x):
// the fact that x was an interface value is forgotten.
func indirectInterface(v reflect.Value) reflect.Value {
if v.Kind() != reflect.Interface {
return v
}
if v.IsNil() {
return reflect.Value{}
}
return v.Elem()
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 reflect.Value) (bool, error) {
arg1 = indirectInterface(arg1)
k1, err := basicKind(arg1)
if err != nil {
return false, err
}
arg2 = indirectInterface(arg2)
k2, err := basicKind(arg2)
if err != nil {
return false, err
}
truth := false
if k1 != k2 {
// Special case: Can compare integer values regardless of type's sign.
switch {
case k1 == intKind && k2 == uintKind:
truth = arg1.Int() < 0 || uint64(arg1.Int()) < arg2.Uint()
case k1 == uintKind && k2 == intKind:
truth = arg2.Int() >= 0 && arg1.Uint() < uint64(arg2.Int())
default:
return false, errBadComparison
}
} else {
switch k1 {
case boolKind, complexKind:
return false, errBadComparisonType
case floatKind:
truth = arg1.Float() < arg2.Float()
case intKind:
truth = arg1.Int() < arg2.Int()
case stringKind:
truth = arg1.String() < arg2.String()
case uintKind:
truth = arg1.Uint() < arg2.Uint()
default:
panic("invalid kind")
}
}
return truth, nil
}
28 changes: 28 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ func (t *valueTpl) Unpack(in string) error {
"hmacBase64": hmacStringBase64,
"join": join,
"toJSON": toJSON,
"max": max,
"min": min,
"mul": mul,
"now": now,
"parseDate": parseDate,
Expand Down Expand Up @@ -295,6 +297,32 @@ func div(a, b int64) int64 {
return a / b
}

func min(arg1, arg2 reflect.Value) (interface{}, error) {
lessThan, err := lt(arg1, arg2)
if err != nil {
return nil, err
}

// arg1 is < arg2.
if lessThan {
return arg1.Interface(), nil
}
return arg2.Interface(), nil
}

func max(arg1, arg2 reflect.Value) (interface{}, error) {
lessThan, err := lt(arg1, arg2)
if err != nil {
return nil, err
}

// arg1 is < arg2.
if lessThan {
return arg2.Interface(), nil
}
return arg1.Interface(), nil
}

func base64Encode(values ...string) string {
data := strings.Join(values, "")
if data == "" {
Expand Down
70 changes: 70 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,76 @@ func TestValueTpl(t *testing.T) {
paramTr: transformable{},
expectedVal: "4",
},
{
name: "func min int",
value: `[[min 4 1]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1",
},
{
name: "func max int",
value: `[[max 4 1]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "4",
},
{
name: "func max float",
value: `[[max 1.23 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "4.666",
},
{
name: "func min float",
value: `[[min 1.23 4.666]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1.23",
},
{
name: "func min string",
value: `[[min "a" "b"]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "a",
},
{
name: "func max string",
value: `[[max "a" "b"]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "b",
},
{
name: "func min int64 unix seconds",
value: `[[ min (now.Unix) 1689771139 ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1689771139",
},
{
name: "func min int year",
value: `[[ min (now.Year) 2020 ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020",
},
{
name: "func max duration",
value: `[[ max (parseDuration "59m") (parseDuration "1h") ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "1h0m0s",
},
{
name: "func min int ",
value: `[[ min (now.Year) 2020 ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020",
},
{
name: "func sha1 hmac Hex",
value: `[[hmac "sha1" "secret" "string1" "string2"]]`,
Expand Down

0 comments on commit 9ae7500

Please sign in to comment.