Add internal pkg for matching metrics by metric properties (#640)

Add metric helper package to internal/process that adds a MatchMetric function that can be used by processors to filter metrics by metric name. MatchMetric can be extended to match on additional metric properties down the line.

`import "github.com/open-telemetry/opentelemetry-collector/internal/processor/metric"`

In addition
- Add a Factory class for creating FilterSets easily
- Add "testutils/configtestutils" to help with testing yaml configs of subpackages

**Link to tracking Issue:** Second part of Filter Processor Proposal #560, though the package can be used by other processors too

**Testing:** Unit tests

internal/processor/filtermetric/config.go

@@ -0,0 +1,30 @@
+// Copyright 2020, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filtermetric
+
+import (
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset"
+)
+
+// MatchProperties specifies the set of properties in a metric to match against and the
+// type of string pattern matching to use.
+type MatchProperties struct {
+	// MatchConfig configures the matching patterns used when matching metric properties.
+	filterset.Config `mapstructure:",squash"`
+
+	// MetricNames specifies the list of string patterns to match metric names against.
+	// A match occurs if the metric name matches at least one string pattern in this list.
+	MetricNames []string `mapstructure:"metric_names"`
+}

internal/processor/filtermetric/config_test.go

@@ -0,0 +1,95 @@
+// Copyright 2020 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filtermetric
+
+import (
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset"
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset/regexp"
+	"github.com/open-telemetry/opentelemetry-collector/testutils/configtestutils"
+)
+
+var (
+	// regexpNameMatches matches the metrics names specified in testdata/config.yaml
+	regexpNameMatches = []string{
+		"prefix/.*",
+		".*contains.*",
+		".*_suffix",
+		"full_name_match",
+	}
+
+	strictNameMatches = []string{
+		"exact_string_match",
+	}
+)
+
+func createConfigWithRegexpOptions(filters []string, rCfg *regexp.Config) *MatchProperties {
+	cfg := createConfig(filters, filterset.Regexp)
+	cfg.Config.RegexpConfig = rCfg
+	return cfg
+}
+
+func TestConfig(t *testing.T) {
+	testFile := path.Join(".", "testdata", "config.yaml")
+	v, err := configtestutils.CreateViperYamlUnmarshaler(testFile)
+	if err != nil {
+		t.Errorf("Error creating Viper config loader: %v", err)
+	}
+
+	testYamls := map[string]MatchProperties{}
+	if err = v.UnmarshalExact(&testYamls); err != nil {
+		t.Errorf("Error unmarshaling yaml from test file %v: %v", testFile, err)
+	}
+
+	tests := []struct {
+		name   string
+		expCfg *MatchProperties
+	}{
+		{
+			name:   "config/regexp",
+			expCfg: createConfig(regexpNameMatches, filterset.Regexp),
+		}, {
+			name: "config/regexpoptions",
+			expCfg: createConfigWithRegexpOptions(
+				regexpNameMatches,
+				&regexp.Config{
+					CacheEnabled:       true,
+					CacheMaxNumEntries: 5,
+				},
+			),
+		}, {
+			name:   "config/strict",
+			expCfg: createConfig(strictNameMatches, filterset.Strict),
+		}, {
+			name:   "config/emptyproperties",
+			expCfg: createConfig(nil, filterset.Regexp),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cfg := testYamls[test.name]
+			assert.Equal(t, *test.expCfg, cfg)
+
+			matcher, err := NewMatcher(&cfg)
+			assert.NotNil(t, matcher)
+			assert.Nil(t, err)
+		})
+	}
+}

internal/processor/filtermetric/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2020, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package filtermetric is a helper package for processing metrics.
+package filtermetric

internal/processor/filtermetric/filtermetric.go

@@ -0,0 +1,51 @@
+// Copyright 2020, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filtermetric
+
+import (
+	metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1"
+
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset"
+)
+
+// Matcher matches metrics by metric properties against prespecified values for each property.
+type Matcher struct {
+	nameFilters filterset.FilterSet
+}
+
+// MatchMetric matches a metric using the metric properties configured on the Matcher.
+// A metric only matches if every metric property configured on the Matcher is a match.
+func (m *Matcher) MatchMetric(metric *metricspb.Metric) bool {
+	name := metric.GetMetricDescriptor().GetName()
+	return m.nameFilters.Matches(name)
+}
+
+// NewMatcher constructs a metric Matcher that can be used to match metrics by metric properties.
+// For each supported metric property, the Matcher accepts a set of prespecified values. An incoming metric
+// matches on a property if the property matches at least one of the prespecified values.
+// A metric only matches if every metric property configured on the Matcher is a match.
+//
+// The metric Matcher supports matching by the following metric properties:
+// - Metric name
+func NewMatcher(config *MatchProperties) (Matcher, error) {
+	nameFS, err := filterset.CreateFilterSet(config.MetricNames, &config.Config)
+	if err != nil {
+		return Matcher{}, err
+	}
+
+	return Matcher{
+		nameFilters: nameFS,
+	}, nil
+}

internal/processor/filtermetric/filtermetric_test.go

@@ -0,0 +1,97 @@
+// Copyright 2020 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filtermetric
+
+import (
+	"testing"
+
+	metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset"
+)
+
+var (
+	regexpFilters = []string{
+		"prefix/.*",
+		"prefix_.*",
+		".*/suffix",
+		".*_suffix",
+		".*/contains/.*",
+		".*_contains_.*",
+		"full/name/match",
+		"full_name_match",
+	}
+
+	strictFilters = []string{
+		"exact_string_match",
+		".*/suffix",
+		"(a|b)",
+	}
+)
+
+func createMetric(name string) *metricspb.Metric {
+	return &metricspb.Metric{
+		MetricDescriptor: &metricspb.MetricDescriptor{
+			Name: name,
+		},
+	}
+}
+
+func TestMatcherMatches(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         *MatchProperties
+		metric      *metricspb.Metric
+		shouldMatch bool
+	}{
+		{
+			name:        "regexpNameMatch",
+			cfg:         createConfig(regexpFilters, filterset.Regexp),
+			metric:      createMetric("test/match/suffix"),
+			shouldMatch: true,
+		}, {
+			name:        "regexpNameMisatch",
+			cfg:         createConfig(regexpFilters, filterset.Regexp),
+			metric:      createMetric("test/match/wrongsuffix"),
+			shouldMatch: false,
+		}, {
+			name:        "strictNameMatch",
+			cfg:         createConfig(strictFilters, filterset.Strict),
+			metric:      createMetric("exact_string_match"),
+			shouldMatch: true,
+		}, {
+			name:        "strictNameMismatch",
+			cfg:         createConfig(regexpFilters, filterset.Regexp),
+			metric:      createMetric("wrong_string_match"),
+			shouldMatch: false,
+		}, {
+			name:        "matcherWithNoPropertyFilters",
+			cfg:         createConfig([]string{}, filterset.Strict),
+			metric:      createMetric("metric"),
+			shouldMatch: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matcher, err := NewMatcher(test.cfg)
+			assert.NotNil(t, matcher)
+			assert.Nil(t, err)
+
+			assert.Equal(t, test.shouldMatch, matcher.MatchMetric(test.metric))
+		})
+	}
+}

internal/processor/filterset/filterset_test.go → internal/processor/filtermetric/helpers_test.go

@@ -12,6 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Empty test file so the CI build does not fail when enforcing a test file in every directory.
-// The filterset package only defines an interface so there's nothing to test.
-package filterset
+package filtermetric
+
+import (
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset"
+)
+
+func createConfig(filters []string, matchType filterset.MatchType) *MatchProperties {
+	return &MatchProperties{
+		Config: filterset.Config{
+			MatchType: matchType,
+		},
+		MetricNames: filters,
+	}
+}

internal/processor/filtermetric/testdata/config.yaml

@@ -0,0 +1,24 @@
+# Yaml form of the configuration for matching metrics
+# This configuration can be embedded into other component's yamls
+# The top level here are just test names and do not represent part of the actual configuration.
+
+config/regexp:
+    match_type: regexp
+    metric_names: [prefix/.*, .*contains.*, .*_suffix, full_name_match]
+config/regexpoptions:
+    match_type: regexp
+    regexp:
+        cacheenabled: true
+        cachemaxnumentries: 5
+    metric_names:
+        - prefix/.*
+        - .*contains.*
+        - .*_suffix
+        - full_name_match
+config/strict:
+    match_type: strict
+    metric_names:
+        - exact_string_match
+config/emptyproperties:
+    match_type: regexp
+    metric_names:

internal/processor/filterset/config.go

@@ -0,0 +1,51 @@
+// Copyright 2020 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filterset
+
+import (
+	"fmt"
+
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset/regexp"
+	"github.com/open-telemetry/opentelemetry-collector/internal/processor/filterset/strict"
+)
+
+// MatchType describes the type of pattern matching a FilterSet uses to filter strings.
+type MatchType string
+
+const (
+	// Regexp is the FilterType for filtering by regexp string matches.
+	Regexp MatchType = "regexp"
+	// Strict is the FilterType for filtering by exact string matches.
+	Strict MatchType = "strict"
+)
+
+// Config configures the matching behavior of a FilterSet.
+type Config struct {
+	MatchType    MatchType      `mapstructure:"match_type"`
+	RegexpConfig *regexp.Config `mapstructure:"regexp"`
+}
+
+// CreateFilterSet creates a FilterSet from yaml config.
+func CreateFilterSet(filters []string, cfg *Config) (FilterSet, error) {
+	switch cfg.MatchType {
+	case Regexp:
+		return regexp.CreateRegexpFilterSet(filters, cfg.RegexpConfig)
+	case Strict:
+		// Strict FilterSets do not have any extra configuration options, so call the constructor directly.
+		return strict.NewStrictFilterSet(filters)
+	default:
+		return nil, fmt.Errorf("unrecognized filter type: %v", cfg.MatchType)
+	}
+}