diff --git a/NOTICE.txt b/NOTICE.txt index 0d9cc89157f..3c6370df1a4 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -722,11 +722,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.8 +Version: v0.6.13 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.13/LICENSE: Apache License Version 2.0, January 2004 diff --git a/changelog/fragments/1709656001-logshints.yaml b/changelog/fragments/1709656001-logshints.yaml new file mode 100644 index 00000000000..a600168c28a --- /dev/null +++ b/changelog/fragments/1709656001-logshints.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# Change summary; a 80ish characters long description of the change. +summary: Introduce log message for not supported annotations for Hints based autodiscover + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/4360 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: https://github.com/elastic/elastic-agent/issues/3064 diff --git a/go.mod b/go.mod index 5bffbb42c9e..accf970129b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 github.com/elastic/e2e-testing v1.2.1 - github.com/elastic/elastic-agent-autodiscover v0.6.8 + github.com/elastic/elastic-agent-autodiscover v0.6.13 github.com/elastic/elastic-agent-client/v7 v7.8.1 github.com/elastic/elastic-agent-libs v0.9.7 github.com/elastic/elastic-agent-system-metrics v0.9.2 diff --git a/go.sum b/go.sum index 8353d606485..60120ed891b 100644 --- a/go.sum +++ b/go.sum @@ -791,8 +791,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/e2e-testing v1.2.1 h1:jIuikohPtTxtO+bfoVEyKAWmcsAl21lxiiTK8Fj+G8U= github.com/elastic/e2e-testing v1.2.1/go.mod h1:8q2d8dmwavJXISowwaoreHFBnbR/uK4qanfRGhC/W9A= -github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8= -github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= +github.com/elastic/elastic-agent-autodiscover v0.6.13 h1:zBeTxV+o2efEKntY+o6iMMNJ1AVjDXUqY3o6uzIkKaw= +github.com/elastic/elastic-agent-autodiscover v0.6.13/go.mod h1:7P6YVKxuBT0qE/VxuA87obwZUAEU0O44mCN3r4/6x8w= github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM= github.com/elastic/elastic-agent-client/v7 v7.8.1/go.mod h1:axl1nkdqc84YRFkeJGD9jExKNPUrOrzf3DFo2m653nY= github.com/elastic/elastic-agent-libs v0.9.7 h1:LZdfxbq724Y1zAdE3COp+OIPwU8SquOCLIXpI/twcdQ= diff --git a/internal/pkg/composable/providers/kubernetes/hints.go b/internal/pkg/composable/providers/kubernetes/hints.go index 8557007c87e..44d4db8b018 100644 --- a/internal/pkg/composable/providers/kubernetes/hints.go +++ b/internal/pkg/composable/providers/kubernetes/hints.go @@ -30,6 +30,8 @@ const ( processors = "processors" ) +var allSupportedHints = []string{"enabled", integration, datastreams, host, period, timeout, metricspath, username, password, stream, processors} + type hintsBuilder struct { Key string @@ -288,7 +290,7 @@ func GetHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, pre } } - hintsExtracted := utils.GenerateHints(annotations, cName, prefix) + hintsExtracted, _ := utils.GenerateHints(annotations, cName, prefix, false, allSupportedHints) if len(hintsExtracted) == 0 { return hintData } diff --git a/internal/pkg/composable/providers/kubernetes/hints_test.go b/internal/pkg/composable/providers/kubernetes/hints_test.go index 0cc709acaf7..e442afff6f9 100644 --- a/internal/pkg/composable/providers/kubernetes/hints_test.go +++ b/internal/pkg/composable/providers/kubernetes/hints_test.go @@ -367,6 +367,137 @@ func TestGenerateHintsMappingWithLogStream(t *testing.T) { assert.Equal(t, expected, hintsMapping) } +func TestGenerateHintsMappingWithDefaultsandTypo(t *testing.T) { + logger := getLogger() + pod := &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + UID: types.UID(uid), + Namespace: "testns", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: kubernetes.PodSpec{ + NodeName: "testnode", + }, + Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, + } + + mapping := map[string]interface{}{ + "namespace": pod.GetNamespace(), + "pod": mapstr.M{ + "uid": string(pod.GetUID()), + "name": pod.GetName(), + "ip": pod.Status.PodIP, + }, + "namespace_annotations": mapstr.M{ + "nsa": "nsb", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + } + hints := mapstr.M{ + "hints": mapstr.M{ + "host": "${kubernetes.pod.ip}:6379", + "package": "redis", + "notsupportedhint": "/metrics", // on purpose we have introduced a typo + "timeout": "42s", + "period": "42s", + }, + } + + expected := mapstr.M{ + "redis": mapstr.M{ + "enabled": true, + "host": "127.0.0.5:6379", + "timeout": "42s", + "period": "42s", + }, + } + + hintsMapping := GenerateHintsMapping(hints, mapping, logger, "") + + assert.Equal(t, expected, hintsMapping) +} + +func TestGenerateHintsMappingWithInfoandTypo(t *testing.T) { + logger := getLogger() + pod := &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + UID: types.UID(uid), + Namespace: "testns", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: kubernetes.PodSpec{ + NodeName: "testnode", + }, + Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, + } + + mapping := map[string]interface{}{ + "namespace": pod.GetNamespace(), + "pod": mapstr.M{ + "uid": string(pod.GetUID()), + "name": pod.GetName(), + "ip": pod.Status.PodIP, + }, + "namespace_annotations": mapstr.M{ + "nsa": "nsb", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + } + hints := mapstr.M{ + "hints": mapstr.M{ + "package": "redis", + "data_streams": "info", + "info": mapstr.M{ + "period": "5m", + "notsupportedhint": "stdout", // On purpose we have added this typo + }, + }, + } + + expected := mapstr.M{ + "redis": mapstr.M{ + "info": mapstr.M{ + "enabled": true, + "period": "5m", + }, + }, + } + + hintsMapping := GenerateHintsMapping(hints, mapping, logger, "") + + assert.Equal(t, expected, hintsMapping) +} + func TestGenerateHintsMappingWithProcessors(t *testing.T) { logger := getLogger() pod := &kubernetes.Pod{ diff --git a/internal/pkg/composable/providers/kubernetes/pod.go b/internal/pkg/composable/providers/kubernetes/pod.go index 9f39f63fa4e..4722a6adc18 100644 --- a/internal/pkg/composable/providers/kubernetes/pod.go +++ b/internal/pkg/composable/providers/kubernetes/pod.go @@ -9,10 +9,9 @@ import ( "sync" "time" - "github.com/elastic/elastic-agent-autodiscover/utils" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/utils" c "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -217,7 +216,7 @@ func (p *pod) emitRunning(pod *kubernetes.Pod) { if !p.managed { if ann, ok := data.mapping["annotations"]; ok { annotations, _ := ann.(mapstr.M) - hints := utils.GenerateHints(annotations, "", p.config.Prefix) + hints, _ := hintsCheck(annotations, "", p.config.Prefix, true, allSupportedHints, p.logger, pod) if len(hints) > 0 { p.logger.Debugf("Extracted hints are :%v", hints) hintsMapping := GenerateHintsMapping(hints, data.mapping, p.logger, "") @@ -525,3 +524,12 @@ func updateProcessors(newprocessors []mapstr.M, processors []map[string]interfac return processors } + +// HintsCheck geenrates hints from provided annotations of the pod and logs any possible incorrect annotations that have been provided in the pod +func hintsCheck(annotations mapstr.M, container string, prefix string, validate bool, allSupportedHints []string, logger *logp.Logger, pod *kubernetes.Pod) (mapstr.M, []string) { + hints, incorrecthints := utils.GenerateHints(annotations, container, prefix, validate, allSupportedHints) + for _, value := range incorrecthints { //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + logger.Warnf("provided hint: %s/%s is not recognised as supported annotation for pod %s in namespace %s", prefix, value, pod.Name, pod.ObjectMeta.Namespace) + } + return hints, incorrecthints +} diff --git a/internal/pkg/composable/providers/kubernetes/pod_test.go b/internal/pkg/composable/providers/kubernetes/pod_test.go index 8ff7ac19c6d..6c59069747f 100644 --- a/internal/pkg/composable/providers/kubernetes/pod_test.go +++ b/internal/pkg/composable/providers/kubernetes/pod_test.go @@ -375,6 +375,59 @@ func TestEphemeralContainers(t *testing.T) { } +func TestGenerateHints(t *testing.T) { + pod := &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testpod", + UID: types.UID(uid), + Namespace: "testns", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + "co.elastic.hints/host": "${kubernetes.pod.ip}:6379", + "co.elastic.hints/package": "redis", + "co.elastic.hints/metricssssssspath": "/metrics", + "co.elastic.hints/period": "42s", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: kubernetes.PodSpec{ + NodeName: "testnode", + }, + Status: kubernetes.PodStatus{PodIP: "127.0.0.5"}, + } + + data := generatePodData(pod, &podMeta{}, mapstr.M{}) + + hints_result := mapstr.M{ + "hints": mapstr.M{ + "host": "${kubernetes.pod.ip}:6379", + "package": "redis", + "metricssssssspath": "/metrics", // on purpose we have introduced a typo + "period": "42s", + }, + } + incorrecthints_results := []string{"hints/metricssssssspath"} + + ann := data.mapping["annotations"] + annotations, _ := ann.(mapstr.M) + prefix := "co.elastic" + + log, err := logger.New("hint-test", true) + assert.NoError(t, err) + + hints, incorrecthints := hintsCheck(annotations, "", prefix, true, allSupportedHints, log, pod) + + assert.Equal(t, string(pod.GetUID()), data.uid) + assert.Equal(t, hints, hints_result) + assert.Equal(t, incorrecthints, incorrecthints_results) +} + func TestPodEventer_Namespace_Node_Watcher(t *testing.T) { client := k8sfake.NewSimpleClientset()