From 58de56ad8c3b99053cc5f2dd10beb61b4efd630f Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Thu, 4 Jul 2024 10:52:32 +0200 Subject: [PATCH] Selection of metric attributes by "glob" in metric name (#991) * attribute metric glob selection * allow glob selection of metric attributes --- pkg/internal/export/attributes/attr_select.go | 23 ++++++ .../export/attributes/attr_select_test.go | 30 ++++++++ .../export/attributes/attr_selector.go | 72 +++++++++++-------- .../export/attributes/attr_selector_test.go | 49 +++++++++++++ .../configs/instrumenter-config-java.yml | 14 +--- .../k8s/manifests/06-beyla-daemonset.yml | 12 +--- 6 files changed, 146 insertions(+), 54 deletions(-) create mode 100644 pkg/internal/export/attributes/attr_select_test.go diff --git a/pkg/internal/export/attributes/attr_select.go b/pkg/internal/export/attributes/attr_select.go index f63f34280..b67e2e1ff 100644 --- a/pkg/internal/export/attributes/attr_select.go +++ b/pkg/internal/export/attributes/attr_select.go @@ -3,6 +3,7 @@ package attributes import ( "maps" "path" + "slices" "strings" attr "github.com/grafana/beyla/pkg/internal/export/attributes/names" @@ -69,3 +70,25 @@ func (incl Selection) Normalize() { maps.DeleteFunc(incl, func(_ Section, _ InclusionLists) bool { return true }) maps.Copy(incl, normalized) } + +// Matching returns all the entries of the inclusion list matching the provided metric name. +// This would include "glob-like" entries. +// They are returned from more to less broad scope (for example, for a metric named foo_bar +// it could return the inclusion lists defined with keys "*", "foo_*" and "foo_bar", in that order). +func (incl Selection) Matching(metricName Name) []InclusionLists { + if incl == nil { + return nil + } + var matchingMetricGlobs []Section + for glob := range incl { + if ok, _ := path.Match(string(glob), string(metricName.Section)); ok { + matchingMetricGlobs = append(matchingMetricGlobs, glob) + } + } + slices.Sort(matchingMetricGlobs) + inclusionLists := make([]InclusionLists, 0, len(matchingMetricGlobs)) + for _, glob := range matchingMetricGlobs { + inclusionLists = append(inclusionLists, incl[glob]) + } + return inclusionLists +} diff --git a/pkg/internal/export/attributes/attr_select_test.go b/pkg/internal/export/attributes/attr_select_test.go new file mode 100644 index 000000000..1a24adc71 --- /dev/null +++ b/pkg/internal/export/attributes/attr_select_test.go @@ -0,0 +1,30 @@ +package attributes + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSelectorMatch(t *testing.T) { + fbb := InclusionLists{Include: []string{"foo_bar_baz"}} + f := InclusionLists{Include: []string{"foo"}} + fbt := InclusionLists{Include: []string{"foo_bar_traca"}} + pp := InclusionLists{Include: []string{"pim_pam"}} + selection := Selection{ + "foo.bar.baz": fbb, + "foo.*": f, + "foo.bar.traca": fbt, + "pim.pam": pp, + } + assert.Equal(t, + []InclusionLists{f, fbb}, + selection.Matching(Name{Section: "foo.bar.baz"})) + assert.Equal(t, + []InclusionLists{f, fbt}, + selection.Matching(Name{Section: "foo.bar.traca"})) + assert.Equal(t, + []InclusionLists{pp}, + selection.Matching(Name{Section: "pim.pam"})) + assert.Empty(t, selection.Matching(Name{Section: "pam.pum"})) +} diff --git a/pkg/internal/export/attributes/attr_selector.go b/pkg/internal/export/attributes/attr_selector.go index af1ab0301..fd5033bb9 100644 --- a/pkg/internal/export/attributes/attr_selector.go +++ b/pkg/internal/export/attributes/attr_selector.go @@ -51,14 +51,14 @@ func NewAttrSelector(groups AttrGroups, selectorCfg Selection) (*AttrSelector, e }, nil } -// For returns the list of attribute names for a given metric +// For returns the list of enabled attribute names for a given metric func (p *AttrSelector) For(metricName Name) Sections[[]attr.Name] { attributeNames, ok := p.definition[metricName.Section] if !ok { panic(fmt.Sprintf("BUG! metric not found %+v", metricName)) } - inclusionLists, ok := p.selector[metricName.Section] - if !ok { + allInclusionLists := p.selector.Matching(metricName) + if len(allInclusionLists) == 0 { attrs := attributeNames.Default() // if the user did not provide any selector, return the default attributes for that metric sas := Sections[[]attr.Name]{ @@ -69,42 +69,52 @@ func (p *AttrSelector) For(metricName Name) Sections[[]attr.Name] { slices.Sort(sas.Resource) return sas } - var addAttributes Sections[map[attr.Name]struct{}] - // if the "include" list is empty, we use the default attributes + matchingAttrs := Sections[map[attr.Name]struct{}]{ + Metric: map[attr.Name]struct{}{}, + Resource: map[attr.Name]struct{}{}, + } + for _, il := range allInclusionLists { + p.addIncludedAttributes(&matchingAttrs, attributeNames, il) + } + // if the "include" lists are empty, we use the default attributes // as included - if len(inclusionLists.Include) == 0 { - addAttributes = attributeNames.Default() - } else { - addAttributes = Sections[map[attr.Name]struct{}]{ - Metric: map[attr.Name]struct{}{}, - Resource: map[attr.Name]struct{}{}, - } - allAttributes := attributeNames.All() - for attrName := range allAttributes.Metric { - if inclusionLists.includes(attrName) { - addAttributes.Metric[attrName] = struct{}{} - } + if len(matchingAttrs.Metric) == 0 && len(matchingAttrs.Resource) == 0 { + matchingAttrs = attributeNames.Default() + } + // now remove any attribute specified in the "exclude" lists + for _, il := range allInclusionLists { + p.rmExcludedAttributes(&matchingAttrs, il) + } + sas := Sections[[]attr.Name]{ + Metric: helpers.SetToSlice(matchingAttrs.Metric), + Resource: helpers.SetToSlice(matchingAttrs.Resource), + } + slices.Sort(sas.Metric) + slices.Sort(sas.Resource) + return sas +} + +func (p *AttrSelector) addIncludedAttributes(matchingAttrs *Sections[map[attr.Name]struct{}], attributeNames AttrReportGroup, inclusionLists InclusionLists) { + allAttributes := attributeNames.All() + for attrName := range allAttributes.Metric { + if inclusionLists.includes(attrName) { + matchingAttrs.Metric[attrName] = struct{}{} } - for attrName := range allAttributes.Resource { - if inclusionLists.includes(attrName) { - addAttributes.Resource[attrName] = struct{}{} - } + } + for attrName := range allAttributes.Resource { + if inclusionLists.includes(attrName) { + matchingAttrs.Resource[attrName] = struct{}{} } } - // now remove any attribute specified in the "exclude" list - maps.DeleteFunc(addAttributes.Metric, func(attr attr.Name, _ struct{}) bool { +} + +func (p *AttrSelector) rmExcludedAttributes(matchingAttrs *Sections[map[attr.Name]struct{}], inclusionLists InclusionLists) { + maps.DeleteFunc(matchingAttrs.Metric, func(attr attr.Name, _ struct{}) bool { return inclusionLists.excludes(attr) }) - maps.DeleteFunc(addAttributes.Resource, func(attr attr.Name, _ struct{}) bool { + maps.DeleteFunc(matchingAttrs.Resource, func(attr attr.Name, _ struct{}) bool { return inclusionLists.excludes(attr) }) - sas := Sections[[]attr.Name]{ - Metric: helpers.SetToSlice(addAttributes.Metric), - Resource: helpers.SetToSlice(addAttributes.Resource), - } - slices.Sort(sas.Metric) - slices.Sort(sas.Resource) - return sas } // All te attributes for this group and their subgroups, unless they are disabled. diff --git a/pkg/internal/export/attributes/attr_selector_test.go b/pkg/internal/export/attributes/attr_selector_test.go index 4b2c84ea8..79e293281 100644 --- a/pkg/internal/export/attributes/attr_selector_test.go +++ b/pkg/internal/export/attributes/attr_selector_test.go @@ -46,6 +46,55 @@ func TestFor(t *testing.T) { }, p.For(BeylaNetworkFlow)) } +func TestFor_GlobEntries(t *testing.T) { + // include all groups just to verify that other attributes aren't anyway selected + p, err := NewAttrSelector(GroupKubernetes, Selection{ + "*": InclusionLists{ + Include: []string{"beyla_ip"}, + Exclude: []string{"k8s_*_name"}, + }, + "beyla_network_flow_bytes_total": InclusionLists{ + Include: []string{"src.*", "k8s.*"}, + Exclude: []string{"k8s.*.type"}, + }, + }) + require.NoError(t, err) + assert.Equal(t, Sections[[]attr.Name]{ + Metric: []attr.Name{ + "beyla.ip", + "k8s.dst.namespace", + "k8s.dst.node.ip", + "k8s.src.namespace", + "k8s.src.node.ip", + "src.address", + "src.name", + "src.port", + }, + Resource: []attr.Name{}, + }, p.For(BeylaNetworkFlow)) +} + +// if no include lists are defined, it takes the default arguments +func TestFor_GlobEntries_NoInclusion(t *testing.T) { + p, err := NewAttrSelector(GroupKubernetes|GroupNetCIDR, Selection{ + "*": InclusionLists{ + Exclude: []string{"*dst*"}, + }, + "beyla_network_flow_bytes_total": InclusionLists{ + Exclude: []string{"k8s.*.namespace"}, + }, + }) + require.NoError(t, err) + assert.Equal(t, Sections[[]attr.Name]{ + Metric: []attr.Name{ + "k8s.cluster.name", + "k8s.src.owner.name", + "src.cidr", + }, + Resource: []attr.Name{}, + }, p.For(BeylaNetworkFlow)) +} + func TestFor_KubeDisabled(t *testing.T) { p, err := NewAttrSelector(0, Selection{ "beyla_network_flow_bytes_total": InclusionLists{ diff --git a/test/integration/configs/instrumenter-config-java.yml b/test/integration/configs/instrumenter-config-java.yml index 614c6696d..33f0f9970 100644 --- a/test/integration/configs/instrumenter-config-java.yml +++ b/test/integration/configs/instrumenter-config-java.yml @@ -6,17 +6,7 @@ otel_metrics_export: endpoint: http://otelcol:4318 attributes: select: - process_cpu_time: + process_*: include: ["*"] + process_cpu_*: exclude: ["process_cpu_state"] - process_cpu_utilization: - include: ["*"] - exclude: ["process_cpu_state"] - process_memory_usage: - include: ["*"] - process_memory_virtual: - include: ["*"] - process_disk_io: - include: ["*"] - process_network_io: - include: ["*"] \ No newline at end of file diff --git a/test/integration/k8s/manifests/06-beyla-daemonset.yml b/test/integration/k8s/manifests/06-beyla-daemonset.yml index 4d13bf1d6..6be5de269 100644 --- a/test/integration/k8s/manifests/06-beyla-daemonset.yml +++ b/test/integration/k8s/manifests/06-beyla-daemonset.yml @@ -9,17 +9,7 @@ data: enable: true cluster_name: beyla select: - process_cpu_time: - include: ["*"] - process_cpu_utilization: - include: ["*"] - process_memory_usage: - include: ["*"] - process_memory_virtual: - include: ["*"] - process_disk_io: - include: ["*"] - process_network_io: + process_*: include: ["*"] print_traces: true log_level: debug