From 21114a8fc944633100627bb6f8c6b03931a49953 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 29 Nov 2021 16:12:12 -0800 Subject: [PATCH] Apply new experimental metrics builder for hostmetrics/cpu scraper Introduce new experimental metrics builder interface for metrics scrapers and apply it to hostmetrics/cpu scraper. --- cmd/mdatagen/loader.go | 9 + cmd/mdatagen/main.go | 38 +++- cmd/mdatagen/main_test.go | 25 ++- cmd/mdatagen/metricdata.go | 47 +++- cmd/mdatagen/metrics_v2.tmpl | 211 ++++++++++++++++++ receiver/hostmetricsreceiver/config_test.go | 2 +- .../hostmetrics_receiver_test.go | 2 +- .../internal/scraper/cpuscraper/codegen.go | 2 +- .../internal/scraper/cpuscraper/config.go | 2 + .../scraper/cpuscraper/cpu_scraper.go | 40 +--- .../scraper/cpuscraper/cpu_scraper_linux.go | 18 +- .../scraper/cpuscraper/cpu_scraper_others.go | 10 +- .../scraper/cpuscraper/cpu_scraper_test.go | 78 ++++--- .../internal/scraper/cpuscraper/factory.go | 5 +- .../internal/metadata/generated_metrics.go | 129 ----------- .../internal/metadata/generated_metrics_v2.go | 185 +++++++++++++++ .../internal/scraper/cpuscraper/metadata.yaml | 2 + 17 files changed, 586 insertions(+), 219 deletions(-) create mode 100644 cmd/mdatagen/metrics_v2.tmpl delete mode 100644 receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics.go create mode 100644 receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics_v2.go diff --git a/cmd/mdatagen/loader.go b/cmd/mdatagen/loader.go index f08c84decca6..8c8542be4085 100644 --- a/cmd/mdatagen/loader.go +++ b/cmd/mdatagen/loader.go @@ -33,6 +33,10 @@ func (mn metricName) Render() (string, error) { return formatIdentifier(string(mn), true) } +func (mn metricName) RenderUnexported() (string, error) { + return formatIdentifier(string(mn), false) +} + type attributeName string func (mn attributeName) Render() (string, error) { @@ -40,6 +44,9 @@ func (mn attributeName) Render() (string, error) { } type metric struct { + // Enabled defines whether the metric is enabled by default. + Enabled bool `yaml:"enabled"` + // Description of the metric. Description string `validate:"required,notblank"` @@ -98,6 +105,8 @@ type templateContext struct { metadata // Package name for generated code. Package string + // ExpFileNote contains a note about experimental metrics builder. + ExpFileNote string } func loadMetadata(ymlData []byte) (metadata, error) { diff --git a/cmd/mdatagen/main.go b/cmd/mdatagen/main.go index cb0fca12d08e..1442861510c0 100644 --- a/cmd/mdatagen/main.go +++ b/cmd/mdatagen/main.go @@ -30,15 +30,23 @@ import ( "text/template" ) +const ( + tmplFileV1 = "metrics.tmpl" + outputFileV1 = "generated_metrics.go" + tmplFileV2 = "metrics_v2.tmpl" + outputFileV2 = "generated_metrics_v2.go" +) + func main() { + useExpGen := flag.Bool("experimental-gen", false, "Use experimental generator") flag.Parse() yml := flag.Arg(0) - if err := run(yml); err != nil { + if err := run(yml, *useExpGen); err != nil { log.Fatal(err) } } -func run(ymlPath string) error { +func run(ymlPath string, useExpGen bool) error { if ymlPath == "" { return errors.New("argument must be metadata.yaml file") } @@ -61,22 +69,29 @@ func run(ymlPath string) error { } thisDir := path.Dir(filename) - if err = generateMetrics(ymlDir, thisDir, md); err != nil { + if err = generateMetrics(ymlDir, thisDir, md, useExpGen); err != nil { return err } return generateDocumentation(ymlDir, thisDir, md) } -func generateMetrics(ymlDir string, thisDir string, md metadata) error { +func generateMetrics(ymlDir string, thisDir string, md metadata, useExpGen bool) error { + tmplFile := tmplFileV1 + outputFile := outputFileV1 + if useExpGen { + tmplFile = tmplFileV2 + outputFile = outputFileV2 + } + tmpl := template.Must( template. - New("metrics.tmpl"). + New(tmplFile). Option("missingkey=error"). Funcs(map[string]interface{}{ "publicVar": func(s string) (string, error) { return formatIdentifier(s, true) }, - }).ParseFiles(path.Join(thisDir, "metrics.tmpl"))) + }).ParseFiles(path.Join(thisDir, tmplFile))) buf := bytes.Buffer{} if err := tmpl.Execute(&buf, templateContext{metadata: md, Package: "metadata"}); err != nil { @@ -95,12 +110,17 @@ func generateMetrics(ymlDir string, thisDir string, md metadata) error { } outputDir := path.Join(ymlDir, "internal", "metadata") - outputFile := path.Join(outputDir, "generated_metrics.go") if err := os.MkdirAll(outputDir, 0700); err != nil { return fmt.Errorf("unable to create output directory %q: %v", outputDir, err) } - if err := ioutil.WriteFile(outputFile, formatted, 0600); err != nil { - return fmt.Errorf("failed writing %q: %v", outputFile, err) + for _, f := range []string{path.Join(outputDir, outputFileV1), path.Join(outputDir, outputFileV2)} { + if err := os.Remove(f); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("unable to remove genererated file %q: %v", f, err) + } + } + outputFilepath := path.Join(outputDir, outputFile) + if err := ioutil.WriteFile(outputFilepath, formatted, 0600); err != nil { + return fmt.Errorf("failed writing %q: %v", outputFilepath, err) } return nil diff --git a/cmd/mdatagen/main_test.go b/cmd/mdatagen/main_test.go index b966c5e1f80b..7377d3cbdda5 100644 --- a/cmd/mdatagen/main_test.go +++ b/cmd/mdatagen/main_test.go @@ -33,13 +33,15 @@ metrics: unit: s sum: aggregation: cumulative + number_type: double attributes: [] ` ) func Test_runContents(t *testing.T) { type args struct { - yml string + yml string + useExpGen bool } tests := []struct { name string @@ -49,12 +51,17 @@ func Test_runContents(t *testing.T) { }{ { name: "valid metadata", - args: args{validMetadata}, + args: args{validMetadata, false}, + want: "", + }, + { + name: "valid metadata v2", + args: args{validMetadata, true}, want: "", }, { name: "invalid yaml", - args: args{"invalid"}, + args: args{"invalid", false}, want: "", wantErr: "cannot unmarshal", }, @@ -70,13 +77,19 @@ func Test_runContents(t *testing.T) { metadataFile := path.Join(tmpdir, "metadata.yaml") require.NoError(t, ioutil.WriteFile(metadataFile, []byte(tt.args.yml), 0600)) - err = run(metadataFile) + err = run(metadataFile, tt.args.useExpGen) if tt.wantErr != "" { require.Regexp(t, tt.wantErr, err) } else { require.NoError(t, err) - require.FileExists(t, path.Join(tmpdir, "internal/metadata/generated_metrics.go")) + + genFilePath := path.Join(tmpdir, "internal/metadata/generated_metrics.go") + if tt.args.useExpGen { + genFilePath = path.Join(tmpdir, "internal/metadata/generated_metrics_v2.go") + } + require.FileExists(t, genFilePath) + require.FileExists(t, path.Join(tmpdir, "documentation.md")) } }) @@ -105,7 +118,7 @@ func Test_run(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := run(tt.args.ymlPath); (err != nil) != tt.wantErr { + if err := run(tt.args.ymlPath, false); (err != nil) != tt.wantErr { t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/cmd/mdatagen/metricdata.go b/cmd/mdatagen/metricdata.go index df6b3358db27..efac124d4b7f 100644 --- a/cmd/mdatagen/metricdata.go +++ b/cmd/mdatagen/metricdata.go @@ -14,6 +14,11 @@ package main +import ( + "fmt" + "strings" +) + var ( _ MetricData = &gauge{} _ MetricData = &sum{} @@ -25,6 +30,7 @@ type MetricData interface { Type() string HasMonotonic() bool HasAggregated() bool + HasNumberDataPoints() bool } // Aggregated defines a metric aggregation type. @@ -52,6 +58,30 @@ type Mono struct { Monotonic bool `yaml:"monotonic"` } +// NumberDataPoints defines the metric number type. +type NumberDataPoints struct { + // Type is type of the metric number, options are "double", "int". + // TODO: Add validation once the metric number type added to all metadata files. + NumberType string `yaml:"number_type"` +} + +// Type returns name of the datapoint type. +func (ndp NumberDataPoints) Type() string { + return strings.Title(ndp.NumberType) +} + +// BasicType returns name of a golang basic type for the datapoint type. +func (ndp NumberDataPoints) BasicType() string { + switch ndp.NumberType { + case "int": + return "int64" + case "double": + return "float64" + default: + panic(fmt.Sprintf("unknown number data point type: %v", ndp.NumberType)) + } +} + type gauge struct { } @@ -67,9 +97,14 @@ func (d gauge) HasAggregated() bool { return false } +func (d gauge) HasNumberDataPoints() bool { + return true +} + type sum struct { - Aggregated `yaml:",inline"` - Mono `yaml:",inline"` + Aggregated `yaml:",inline"` + Mono `yaml:",inline"` + NumberDataPoints `yaml:",inline"` } func (d sum) Type() string { @@ -84,6 +119,10 @@ func (d sum) HasAggregated() bool { return true } +func (d sum) HasNumberDataPoints() bool { + return true +} + type histogram struct { Aggregated `yaml:",inline"` } @@ -99,3 +138,7 @@ func (d histogram) HasMonotonic() bool { func (d histogram) HasAggregated() bool { return true } + +func (d histogram) HasNumberDataPoints() bool { + return false +} diff --git a/cmd/mdatagen/metrics_v2.tmpl b/cmd/mdatagen/metrics_v2.tmpl new file mode 100644 index 000000000000..a56b2204ea16 --- /dev/null +++ b/cmd/mdatagen/metrics_v2.tmpl @@ -0,0 +1,211 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package {{ .Package }} + +import ( + "time" + + "go.opentelemetry.io/collector/model/pdata" +) + +// MetricSettings provides common settings for a particular metric. +type MetricSettings struct { + Enabled bool `mapstructure:"enabled"` +} + +// MetricsSettings provides settings for {{ .Name }} metrics. +type MetricsSettings struct { + {{- range $name, $metric := .Metrics }} + {{ $name.Render }} MetricSettings `mapstructure:"{{ $name }}"` + {{- end }} +} + +func DefaultMetricsSettings() MetricsSettings { + return MetricsSettings{ + {{- range $name, $metric := .Metrics }} + {{ $name.Render }}: MetricSettings{ + Enabled: {{ $metric.Enabled }}, + }, + {{- end }} + } +} + +type metrics struct { + {{- range $name, $metric := .Metrics }} + {{ $name.Render }} pdata.Metric + {{- end }} +} + +// MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations +// required to produce metric representation defined in metadata and user configuration. +type MetricsBuilder struct { + config MetricsSettings + startTime pdata.Timestamp + {{ range $attr, $_ := .Attributes -}} + attribute{{ $attr.Render }}Capacity int + {{ end -}} + metrics metrics +} + +// metricBuilderOption applies changes to default metrics builder. +type metricBuilderOption func(*MetricsBuilder) + +// WithStartTime sets startTime on the metrics builder. +func WithStartTime(startTime pdata.Timestamp) metricBuilderOption { + return func(mb *MetricsBuilder) { + mb.startTime = startTime + } +} + +{{ range $attr, $_ := .Attributes -}} +// WithAttribute{{ $attr.Render }}Capacity sets an expected number of values of {{ $attr }} attribute that will be +// used to calculate data points capacity for each metric report. +func WithAttribute{{ $attr.Render }}Capacity(cap int) metricBuilderOption { + return func(mb *MetricsBuilder) { + mb.attribute{{ $attr.Render }}Capacity = cap + } +} + +{{ end -}} + +func NewMetricsBuilder(config MetricsSettings, options ...metricBuilderOption) *MetricsBuilder { + mb := &MetricsBuilder{ + config: config, + startTime: pdata.NewTimestampFromTime(time.Now()), + {{- range $name, $info := .Attributes }} + {{- if $info.Enum }} + attribute{{ $name.Render }}Capacity: {{ len $info.Enum }}, + {{- end }} + {{- end }} + } + + for _, op := range options { + op(mb) + } + + mb.clearMetrics() + return mb +} + +// Emit appends generated metrics to a pdata.MetricsSlice and updates the internal state to be ready for recording +// another set of data points. This function will be doing all transformations required to produce metric representation +// defined in metadata and user configuration, e.g. delta/cumulative translation. +func (mb *MetricsBuilder) Emit(metrics pdata.MetricSlice) { + {{- range $name, $metric := .Metrics }} + if mb.config.{{- $name.Render }}.Enabled { + mb.metrics.{{- $name.Render }}.CopyTo(metrics.AppendEmpty()) + } + {{- end }} + + // Reset metric data points collection. + mb.clearMetrics() +} + +{{ range $name, $metric := .Metrics -}} +// {{ $name.RenderUnexported }}DataPointsCapacity calculates initial data points capacity for {{ $name }} metric. +func (mb *MetricsBuilder) {{ $name.RenderUnexported }}DataPointsCapacity() int { + return {{ range $idx, $attr := $metric.Attributes -}} + {{- if $idx }} * {{ end -}} + mb.attribute{{ $attr.Render }}Capacity + {{- end }} +} + +// {{ $name.RenderUnexported }}Metric builds new {{ $name }} metric. +func (mb *MetricsBuilder) {{ $name.RenderUnexported }}Metric() pdata.Metric { + metric := pdata.NewMetric() + metric.SetName("{{ $name }}") + metric.SetDescription("{{ $metric.Description }}") + metric.SetUnit("{{ $metric.Unit }}") + metric.SetDataType(pdata.MetricDataType{{ $metric.Data.Type }}) + {{- if $metric.Data.HasMonotonic }} + metric.{{ $metric.Data.Type }}().SetIsMonotonic({{ $metric.Data.Monotonic }}) + {{- end }} + {{- if $metric.Data.HasAggregated }} + metric.{{ $metric.Data.Type }}().SetAggregationTemporality({{ $metric.Data.Aggregated.Type }}) + {{- end }} + metric.{{ $metric.Data.Type }}().DataPoints().EnsureCapacity(mb.{{ $name.RenderUnexported }}DataPointsCapacity()) + return metric +} + +{{ end -}} + +// clearMetrics clears metrics structure. +func (mb *MetricsBuilder) clearMetrics() { + {{- range $name, $metric := .Metrics }} + if mb.config.{{- $name.Render }}.Enabled { + // TODO: Use mb.metrics.{{ $name.Render }}.{{ $metric.Data.Type }}().DataPoints().Clear() instead of rebuilding + // the metrics once the Clear method is available. + mb.metrics.{{ $name.Render }} = mb.{{ $name.RenderUnexported }}Metric() + } + {{- end }} +} + +{{ range $name, $metric := .Metrics -}} +// Record{{ $name.Render }}DataPoint adds a data point to {{ $name }} metric. +// Any attribute of AttributeValueTypeEmpty type will be skipped. +func (mb *MetricsBuilder) Record{{ $name.Render }}DataPoint(ts pdata.Timestamp + {{- if $metric.Data.HasNumberDataPoints}}, val {{ $metric.Data.NumberDataPoints.BasicType }}{{ end }} + {{- range $metric.Attributes -}}, {{ . }}AttributeValue string {{ end }}) { + if !mb.config.{{- $name.Render }}.Enabled { + return + } + + dp := mb.metrics.{{ $name.Render }}.{{ $metric.Data.Type }}().DataPoints().AppendEmpty() + dp.SetStartTimestamp(mb.startTime) + dp.SetTimestamp(ts) + {{- if $metric.Data.HasNumberDataPoints}} + dp.Set{{ $metric.Data.NumberDataPoints.Type }}Val(val) + {{- end }} + {{ range $metric.Attributes -}} + dp.Attributes().Insert(A.{{ .Render }}, pdata.NewAttributeValueString({{ . }}AttributeValue)) + {{ end -}} +} +{{ end }} + +// Attributes contains the possible metric attributes that can be used. +var Attributes = struct { +{{- range $name, $info := .Attributes }} + // {{ $name.Render }} ({{ $info.Description }}) + {{ $name.Render }} string +{{- end }} +}{ +{{- range $name, $info := .Attributes }} + {{- if $info.Value }} + "{{ $info.Value }}", + {{- else }} + "{{ $name }}", + {{- end }} +{{- end }} +} + +// A is an alias for Attributes. +var A = Attributes + +{{ range $name, $info := .Attributes -}} +{{ if $info.Enum -}} +// Attribute{{ $name.Render }} are the possible values that the attribute "{{ $name }}" can have. +var Attribute{{ $name.Render }} = struct { +{{- range $info.Enum }} + {{ . | publicVar }} string +{{- end }} +}{ +{{- range $info.Enum }} + "{{ . }}", +{{- end }} +} +{{ end }} +{{ end }} diff --git a/receiver/hostmetricsreceiver/config_test.go b/receiver/hostmetricsreceiver/config_test.go index fb91d0a1319f..5adf0853c9ff 100644 --- a/receiver/hostmetricsreceiver/config_test.go +++ b/receiver/hostmetricsreceiver/config_test.go @@ -67,7 +67,7 @@ func TestLoadConfig(t *testing.T) { CollectionInterval: 30 * time.Second, }, Scrapers: map[string]internal.Config{ - cpuscraper.TypeStr: &cpuscraper.Config{}, + cpuscraper.TypeStr: (&cpuscraper.Factory{}).CreateDefaultConfig(), diskscraper.TypeStr: &diskscraper.Config{}, loadscraper.TypeStr: &loadscraper.Config{}, filesystemscraper.TypeStr: &filesystemscraper.Config{}, diff --git a/receiver/hostmetricsreceiver/hostmetrics_receiver_test.go b/receiver/hostmetricsreceiver/hostmetrics_receiver_test.go index 51bc4429c0fc..65b3d362ac92 100644 --- a/receiver/hostmetricsreceiver/hostmetrics_receiver_test.go +++ b/receiver/hostmetricsreceiver/hostmetrics_receiver_test.go @@ -104,7 +104,7 @@ func TestGatherMetrics_EndToEnd(t *testing.T) { CollectionInterval: 100 * time.Millisecond, }, Scrapers: map[string]internal.Config{ - cpuscraper.TypeStr: &cpuscraper.Config{}, + cpuscraper.TypeStr: scraperFactories[cpuscraper.TypeStr].CreateDefaultConfig(), diskscraper.TypeStr: &diskscraper.Config{}, filesystemscraper.TypeStr: &filesystemscraper.Config{}, loadscraper.TypeStr: &loadscraper.Config{}, diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/codegen.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/codegen.go index 66d337ec5674..30c002c0253e 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/codegen.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/codegen.go @@ -15,6 +15,6 @@ //go:build !windows // +build !windows -//go:generate mdatagen metadata.yaml +//go:generate mdatagen --experimental-gen metadata.yaml package cpuscraper // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/cpuscraper" diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/config.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/config.go index 8b0134310198..3074eddb540b 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/config.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/config.go @@ -16,9 +16,11 @@ package cpuscraper // import "github.com/open-telemetry/opentelemetry-collector- import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata" ) // Config relating to CPU Metric Scraper. type Config struct { internal.ConfigSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct + Metrics metadata.MetricsSettings `mapstructure:"metrics"` } diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper.go index 4f2aa062f114..ad9e9f58e913 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper.go @@ -16,6 +16,7 @@ package cpuscraper // import "github.com/open-telemetry/opentelemetry-collector- import ( "context" + "runtime" "time" "github.com/shirou/gopsutil/v3/cpu" @@ -31,8 +32,8 @@ const metricsLen = 1 // scraper for CPU Metrics type scraper struct { - config *Config - startTime pdata.Timestamp + config *Config + mb *metadata.MetricsBuilder // for mocking bootTime func() (uint64, error) @@ -49,8 +50,11 @@ func (s *scraper) start(context.Context, component.Host) error { if err != nil { return err } - - s.startTime = pdata.Timestamp(bootTime * 1e9) + s.mb = metadata.NewMetricsBuilder( + s.config.Metrics, + metadata.WithStartTime(pdata.Timestamp(bootTime*1e9)), + metadata.WithAttributeStateCapacity(cpuStatesLen), + metadata.WithAttributeCpuCapacity(runtime.NumCPU())) return nil } @@ -64,31 +68,9 @@ func (s *scraper) scrape(_ context.Context) (pdata.Metrics, error) { return md, scrapererror.NewPartialScrapeError(err, metricsLen) } - initializeCPUTimeMetric(metrics.AppendEmpty(), s.startTime, now, cpuTimes) - return md, nil -} - -func initializeCPUTimeMetric(metric pdata.Metric, startTime, now pdata.Timestamp, cpuTimes []cpu.TimesStat) { - metadata.Metrics.SystemCPUTime.Init(metric) - - ddps := metric.Sum().DataPoints() - ddps.EnsureCapacity(len(cpuTimes) * cpuStatesLen) for _, cpuTime := range cpuTimes { - appendCPUTimeStateDataPoints(ddps, startTime, now, cpuTime) + s.recordCPUTimeStateDataPoints(now, cpuTime) } -} - -const gopsCPUTotal string = "cpu-total" - -func initializeCPUTimeDataPoint(dataPoint pdata.NumberDataPoint, startTime, now pdata.Timestamp, cpuLabel string, stateLabel string, value float64) { - attributes := dataPoint.Attributes() - // ignore cpu attribute if reporting "total" cpu usage - if cpuLabel != gopsCPUTotal { - attributes.InsertString(metadata.Attributes.Cpu, cpuLabel) - } - attributes.InsertString(metadata.Attributes.State, stateLabel) - - dataPoint.SetStartTimestamp(startTime) - dataPoint.SetTimestamp(now) - dataPoint.SetDoubleVal(value) + s.mb.Emit(metrics) + return md, nil } diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_linux.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_linux.go index 92c71ce63b69..450c4ad73d87 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_linux.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_linux.go @@ -26,13 +26,13 @@ import ( const cpuStatesLen = 8 -func appendCPUTimeStateDataPoints(ddps pdata.NumberDataPointSlice, startTime, now pdata.Timestamp, cpuTime cpu.TimesStat) { - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.User, cpuTime.User) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.System, cpuTime.System) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Idle, cpuTime.Idle) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Interrupt, cpuTime.Irq) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Nice, cpuTime.Nice) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Softirq, cpuTime.Softirq) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Steal, cpuTime.Steal) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Wait, cpuTime.Iowait) +func (s *scraper) recordCPUTimeStateDataPoints(now pdata.Timestamp, cpuTime cpu.TimesStat) { + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.User, cpuTime.CPU, metadata.AttributeState.User) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.System, cpuTime.CPU, metadata.AttributeState.System) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Idle, cpuTime.CPU, metadata.AttributeState.Idle) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Irq, cpuTime.CPU, metadata.AttributeState.Interrupt) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Nice, cpuTime.CPU, metadata.AttributeState.Nice) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Softirq, cpuTime.CPU, metadata.AttributeState.Softirq) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Steal, cpuTime.CPU, metadata.AttributeState.Steal) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Iowait, cpuTime.CPU, metadata.AttributeState.Wait) } diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_others.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_others.go index bf2251f708bc..81f4d9154a76 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_others.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_others.go @@ -26,9 +26,9 @@ import ( const cpuStatesLen = 4 -func appendCPUTimeStateDataPoints(ddps pdata.NumberDataPointSlice, startTime, now pdata.Timestamp, cpuTime cpu.TimesStat) { - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.User, cpuTime.User) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.System, cpuTime.System) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Idle, cpuTime.Idle) - initializeCPUTimeDataPoint(ddps.AppendEmpty(), startTime, now, cpuTime.CPU, metadata.AttributeState.Interrupt, cpuTime.Irq) +func (s *scraper) recordCPUTimeStateDataPoints(now pdata.Timestamp, cpuTime cpu.TimesStat) { + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.User, cpuTime.CPU, metadata.AttributeState.User) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.System, cpuTime.CPU, metadata.AttributeState.System) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Idle, cpuTime.CPU, metadata.AttributeState.Idle) + s.mb.RecordSystemCPUTimeDataPoint(now, cpuTime.Irq, cpuTime.CPU, metadata.AttributeState.Interrupt) } diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_test.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_test.go index 1d58b1ee087c..b4715334ddb6 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_test.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/cpu_scraper_test.go @@ -33,38 +33,57 @@ import ( func TestScrape(t *testing.T) { type testCase struct { - name string - bootTimeFunc func() (uint64, error) - timesFunc func(bool) ([]cpu.TimesStat, error) - expectedStartTime pdata.Timestamp - initializationErr string - expectedErr string + name string + bootTimeFunc func() (uint64, error) + timesFunc func(bool) ([]cpu.TimesStat, error) + metricsConfig metadata.MetricsSettings + expectedMetricCount int + expectedStartTime pdata.Timestamp + initializationErr string + expectedErr string } testCases := []testCase{ { - name: "Standard", + name: "Standard", + metricsConfig: metadata.DefaultMetricsSettings(), + expectedMetricCount: 1, }, { - name: "Validate Start Time", - bootTimeFunc: func() (uint64, error) { return 100, nil }, - expectedStartTime: 100 * 1e9, + name: "Validate Start Time", + bootTimeFunc: func() (uint64, error) { return 100, nil }, + metricsConfig: metadata.DefaultMetricsSettings(), + expectedMetricCount: 1, + expectedStartTime: 100 * 1e9, }, { - name: "Boot Time Error", - bootTimeFunc: func() (uint64, error) { return 0, errors.New("err1") }, - initializationErr: "err1", + name: "Boot Time Error", + bootTimeFunc: func() (uint64, error) { return 0, errors.New("err1") }, + metricsConfig: metadata.DefaultMetricsSettings(), + expectedMetricCount: 1, + initializationErr: "err1", }, { - name: "Times Error", - timesFunc: func(bool) ([]cpu.TimesStat, error) { return nil, errors.New("err2") }, - expectedErr: "err2", + name: "Times Error", + timesFunc: func(bool) ([]cpu.TimesStat, error) { return nil, errors.New("err2") }, + metricsConfig: metadata.DefaultMetricsSettings(), + expectedMetricCount: 1, + expectedErr: "err2", + }, + { + name: "SystemCPUTime metric is disabled ", + metricsConfig: metadata.MetricsSettings{ + SystemCPUTime: metadata.MetricSettings{ + Enabled: false, + }, + }, + expectedMetricCount: 0, }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - scraper := newCPUScraper(context.Background(), &Config{}) + scraper := newCPUScraper(context.Background(), &Config{Metrics: test.metricsConfig}) if test.bootTimeFunc != nil { scraper.bootTime = test.bootTimeFunc } @@ -93,22 +112,29 @@ func TestScrape(t *testing.T) { } require.NoError(t, err, "Failed to scrape metrics: %v", err) - assert.Equal(t, 1, md.MetricCount()) + assert.Equal(t, test.expectedMetricCount, md.MetricCount()) - metrics := md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics() - assertCPUMetricValid(t, metrics.At(0), metadata.Metrics.SystemCPUTime.New(), test.expectedStartTime) + if test.expectedMetricCount > 0 { + metrics := md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics() + assertCPUMetricValid(t, metrics.At(0), test.expectedStartTime) - if runtime.GOOS == "linux" { - assertCPUMetricHasLinuxSpecificStateLabels(t, metrics.At(0)) - } + if runtime.GOOS == "linux" { + assertCPUMetricHasLinuxSpecificStateLabels(t, metrics.At(0)) + } - internal.AssertSameTimeStampForAllMetrics(t, metrics) + internal.AssertSameTimeStampForAllMetrics(t, metrics) + } }) } } -func assertCPUMetricValid(t *testing.T, metric pdata.Metric, descriptor pdata.Metric, startTime pdata.Timestamp) { - internal.AssertDescriptorEqual(t, descriptor, metric) +func assertCPUMetricValid(t *testing.T, metric pdata.Metric, startTime pdata.Timestamp) { + expected := pdata.NewMetric() + expected.SetName("system.cpu.time") + expected.SetDescription("Total CPU seconds broken down by different states.") + expected.SetUnit("s") + expected.SetDataType(pdata.MetricDataTypeSum) + internal.AssertDescriptorEqual(t, expected, metric) if startTime != 0 { internal.AssertSumMetricStartTimeEquals(t, metric, startTime) } diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/factory.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/factory.go index a91e2dfea62c..4291d343489a 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/factory.go +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/factory.go @@ -21,6 +21,7 @@ import ( "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata" ) // This file implements Factory for CPU scraper. @@ -36,7 +37,9 @@ type Factory struct { // CreateDefaultConfig creates the default configuration for the Scraper. func (f *Factory) CreateDefaultConfig() internal.Config { - return &Config{} + return &Config{ + Metrics: metadata.DefaultMetricsSettings(), + } } // CreateMetricsScraper creates a scraper based on provided config. diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics.go deleted file mode 100644 index ff5a1b84d023..000000000000 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/model/pdata" -) - -// Type is the component type name. -const Type config.Type = "cpu" - -// MetricIntf is an interface to generically interact with generated metric. -type MetricIntf interface { - Name() string - New() pdata.Metric - Init(metric pdata.Metric) -} - -// Intentionally not exposing this so that it is opaque and can change freely. -type metricImpl struct { - name string - initFunc func(pdata.Metric) -} - -// Name returns the metric name. -func (m *metricImpl) Name() string { - return m.name -} - -// New creates a metric object preinitialized. -func (m *metricImpl) New() pdata.Metric { - metric := pdata.NewMetric() - m.Init(metric) - return metric -} - -// Init initializes the provided metric object. -func (m *metricImpl) Init(metric pdata.Metric) { - m.initFunc(metric) -} - -type metricStruct struct { - SystemCPUTime MetricIntf -} - -// Names returns a list of all the metric name strings. -func (m *metricStruct) Names() []string { - return []string{ - "system.cpu.time", - } -} - -var metricsByName = map[string]MetricIntf{ - "system.cpu.time": Metrics.SystemCPUTime, -} - -func (m *metricStruct) ByName(n string) MetricIntf { - return metricsByName[n] -} - -// Metrics contains a set of methods for each metric that help with -// manipulating those metrics. -var Metrics = &metricStruct{ - &metricImpl{ - "system.cpu.time", - func(metric pdata.Metric) { - metric.SetName("system.cpu.time") - metric.SetDescription("Total CPU seconds broken down by different states.") - metric.SetUnit("s") - metric.SetDataType(pdata.MetricDataTypeSum) - metric.Sum().SetIsMonotonic(true) - metric.Sum().SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) - }, - }, -} - -// M contains a set of methods for each metric that help with -// manipulating those metrics. M is an alias for Metrics -var M = Metrics - -// Attributes contains the possible metric attributes that can be used. -var Attributes = struct { - // Cpu (CPU number starting at 0.) - Cpu string - // State (Breakdown of CPU usage by type.) - State string -}{ - "cpu", - "state", -} - -// A is an alias for Attributes. -var A = Attributes - -// AttributeState are the possible values that the attribute "state" can have. -var AttributeState = struct { - Idle string - Interrupt string - Nice string - Softirq string - Steal string - System string - User string - Wait string -}{ - "idle", - "interrupt", - "nice", - "softirq", - "steal", - "system", - "user", - "wait", -} diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics_v2.go b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics_v2.go new file mode 100644 index 000000000000..7d4fe279eba4 --- /dev/null +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/internal/metadata/generated_metrics_v2.go @@ -0,0 +1,185 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "time" + + "go.opentelemetry.io/collector/model/pdata" +) + +// MetricSettings provides common settings for a particular metric. +type MetricSettings struct { + Enabled bool `mapstructure:"enabled"` +} + +// MetricsSettings provides settings for cpu metrics. +type MetricsSettings struct { + SystemCPUTime MetricSettings `mapstructure:"system.cpu.time"` +} + +func DefaultMetricsSettings() MetricsSettings { + return MetricsSettings{ + SystemCPUTime: MetricSettings{ + Enabled: true, + }, + } +} + +type metrics struct { + SystemCPUTime pdata.Metric +} + +// MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations +// required to produce metric representation defined in metadata and user configuration. +type MetricsBuilder struct { + config MetricsSettings + startTime pdata.Timestamp + attributeCpuCapacity int + attributeStateCapacity int + metrics metrics +} + +// metricBuilderOption applies changes to default metrics builder. +type metricBuilderOption func(*MetricsBuilder) + +// WithStartTime sets startTime on the metrics builder. +func WithStartTime(startTime pdata.Timestamp) metricBuilderOption { + return func(mb *MetricsBuilder) { + mb.startTime = startTime + } +} + +// WithAttributeCpuCapacity sets an expected number of values of cpu attribute that will be +// used to calculate data points capacity for each metric report. +func WithAttributeCpuCapacity(cap int) metricBuilderOption { + return func(mb *MetricsBuilder) { + mb.attributeCpuCapacity = cap + } +} + +// WithAttributeStateCapacity sets an expected number of values of state attribute that will be +// used to calculate data points capacity for each metric report. +func WithAttributeStateCapacity(cap int) metricBuilderOption { + return func(mb *MetricsBuilder) { + mb.attributeStateCapacity = cap + } +} + +func NewMetricsBuilder(config MetricsSettings, options ...metricBuilderOption) *MetricsBuilder { + mb := &MetricsBuilder{ + config: config, + startTime: pdata.NewTimestampFromTime(time.Now()), + attributeStateCapacity: 8, + } + + for _, op := range options { + op(mb) + } + + mb.clearMetrics() + return mb +} + +// Emit appends generated metrics to a pdata.MetricsSlice and updates the internal state to be ready for recording +// another set of data points. This function will be doing all transformations required to produce metric representation +// defined in metadata and user configuration, e.g. delta/cumulative translation. +func (mb *MetricsBuilder) Emit(metrics pdata.MetricSlice) { + if mb.config.SystemCPUTime.Enabled { + mb.metrics.SystemCPUTime.CopyTo(metrics.AppendEmpty()) + } + + // Reset metric data points collection. + mb.clearMetrics() +} + +// systemCPUTimeDataPointsCapacity calculates initial data points capacity for system.cpu.time metric. +func (mb *MetricsBuilder) systemCPUTimeDataPointsCapacity() int { + return mb.attributeCpuCapacity * mb.attributeStateCapacity +} + +// systemCPUTimeMetric builds new system.cpu.time metric. +func (mb *MetricsBuilder) systemCPUTimeMetric() pdata.Metric { + metric := pdata.NewMetric() + metric.SetName("system.cpu.time") + metric.SetDescription("Total CPU seconds broken down by different states.") + metric.SetUnit("s") + metric.SetDataType(pdata.MetricDataTypeSum) + metric.Sum().SetIsMonotonic(true) + metric.Sum().SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + metric.Sum().DataPoints().EnsureCapacity(mb.systemCPUTimeDataPointsCapacity()) + return metric +} + +// clearMetrics clears metrics structure. +func (mb *MetricsBuilder) clearMetrics() { + if mb.config.SystemCPUTime.Enabled { + // TODO: Use mb.metrics.SystemCPUTime.Sum().DataPoints().Clear() instead of rebuilding + // the metrics once the Clear method is available. + mb.metrics.SystemCPUTime = mb.systemCPUTimeMetric() + } +} + +// RecordSystemCPUTimeDataPoint adds a data point to system.cpu.time metric. +// Any attribute of AttributeValueTypeEmpty type will be skipped. +func (mb *MetricsBuilder) RecordSystemCPUTimeDataPoint(ts pdata.Timestamp, val float64, cpuAttributeValue string, stateAttributeValue string) { + if !mb.config.SystemCPUTime.Enabled { + return + } + + dp := mb.metrics.SystemCPUTime.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(mb.startTime) + dp.SetTimestamp(ts) + dp.SetDoubleVal(val) + dp.Attributes().Insert(A.Cpu, pdata.NewAttributeValueString(cpuAttributeValue)) + dp.Attributes().Insert(A.State, pdata.NewAttributeValueString(stateAttributeValue)) +} + +// Attributes contains the possible metric attributes that can be used. +var Attributes = struct { + // Cpu (CPU number starting at 0.) + Cpu string + // State (Breakdown of CPU usage by type.) + State string +}{ + "cpu", + "state", +} + +// A is an alias for Attributes. +var A = Attributes + +// AttributeState are the possible values that the attribute "state" can have. +var AttributeState = struct { + Idle string + Interrupt string + Nice string + Softirq string + Steal string + System string + User string + Wait string +}{ + "idle", + "interrupt", + "nice", + "softirq", + "steal", + "system", + "user", + "wait", +} diff --git a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/metadata.yaml b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/metadata.yaml index 510a0ba51980..a8236533402d 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/metadata.yaml +++ b/receiver/hostmetricsreceiver/internal/scraper/cpuscraper/metadata.yaml @@ -10,9 +10,11 @@ attributes: metrics: system.cpu.time: + enabled: true description: Total CPU seconds broken down by different states. unit: s sum: + number_type: double aggregation: cumulative monotonic: true attributes: [cpu, state]