From 3c126d66a9800265d809d20239ddccb0d87ec9d7 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Thu, 13 Oct 2022 12:00:36 +0200 Subject: [PATCH] [receiver/hostmetrics/processscraper] Introduce renamed process memory metrics Fixes #14327 --- .chloggen/rename-process-memory-metrics.yaml | 28 ++++ receiver/hostmetricsreceiver/README.md | 48 ++++++- .../scraper/processscraper/documentation.md | 6 +- .../internal/metadata/generated_metrics.go | 130 +++++++++++++++++- .../scraper/processscraper/metadata.yaml | 20 ++- .../scraper/processscraper/process_scraper.go | 2 + .../processscraper/process_scraper_test.go | 2 +- 7 files changed, 229 insertions(+), 7 deletions(-) create mode 100755 .chloggen/rename-process-memory-metrics.yaml diff --git a/.chloggen/rename-process-memory-metrics.yaml b/.chloggen/rename-process-memory-metrics.yaml new file mode 100755 index 000000000000..022210ecd082 --- /dev/null +++ b/.chloggen/rename-process-memory-metrics.yaml @@ -0,0 +1,28 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: hostmetricsreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Introduce renamed process memory metrics + +# One or more tracking issues related to the change +issues: [14327] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + This starts the process of phasing out incorrect metric names: + + - `process.memory.physical_usage` + - `process.memory.virtual_usage` + + and replacing them with the names adhering to the semantic conventions: + + - `process.memory.usage` + - `process.memory.virtual` + + At this stage, the new metrics are added, but they are disabled by default. + See the "Deprecations" section of the Host Metrics receiver's README for details. diff --git a/receiver/hostmetricsreceiver/README.md b/receiver/hostmetricsreceiver/README.md index 3cd9ca4d1575..55b11c68bcb7 100644 --- a/receiver/hostmetricsreceiver/README.md +++ b/receiver/hostmetricsreceiver/README.md @@ -149,7 +149,53 @@ Currently, the hostmetrics receiver does not set any Resource attributes on the export OTEL_RESOURCE_ATTRIBUTES="service.name=,service.namespace=,service.instance.id=" ``` +## Deprecations + +### Transition to process memory metric names aligned with OpenTelemetry specification + +The Host Metrics receiver has been emitting the following process memory metrics: + +- [process.memory.physical_usage] for the amount of physical memory used by the process, +- [process.memory.virtual_usage] for the amount of virtual memory used by the process. + +This is in conflict with the OpenTelemetry specification, +which defines [process.memory.usage] and [process.memory.virtual] as the names for these metrics. + +To align the emitted metric names with the OpenTelemetry specification, +the following process will be followed to phase out the old metrics: + +- Until and including `v0.63.0`, only the old metrics `process.memory.physical_usage` and `process.memory.virtual_usage` are emitted. + You can use the [Metrics Transform processor][metricstransformprocessor_docs] to rename them. +- Between `v0.64.0` and `v0.66.0`, the new metrics are introduced and the old metrics are marked as deprecated. + Both the new and the old metrics are emitted by default. +- Between `v0.67.0` and `v0.69.0`, the old metrics are disabled by default. +- In `v0.70.0` and up, the old metrics are removed. + +To change the enabled state for the specific metrics, use the standard configuration options that are available for all metrics. + +Here's an example configuration to disable the old metrics and enable the new metrics: + +```yaml +receivers: + hostmetrics: + scrapers: + process: + metrics: + process.memory.physical_usage: + enabled: false + process.memory.virtual_usage: + enabled: false + process.memory.usage: + enabled: true + process.memory.virtual: + enabled: true +``` + [beta]: https://github.com/open-telemetry/opentelemetry-collector#beta [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib [core]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol - +[process.memory.physical_usage]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.63.0/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml#L61 +[process.memory.virtual_usage]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.63.0/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml#L70 +[process.memory.usage]: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.14.0/specification/metrics/semantic_conventions/process-metrics.md?plain=1#L38 +[process.memory.virtual]: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.14.0/specification/metrics/semantic_conventions/process-metrics.md?plain=1#L39 +[metricstransformprocessor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.63.0/processor/metricstransformprocessor/README.md diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/documentation.md b/receiver/hostmetricsreceiver/internal/scraper/processscraper/documentation.md index 3b3c9cc41e78..72be3c02fb4f 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/documentation.md +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/documentation.md @@ -11,8 +11,10 @@ These are the metrics available for this scraper. | process.context_switches | Number of times the process has been context switched. | {count} | Sum(Int) |
  • context_switch_type
| | **process.cpu.time** | Total CPU seconds broken down by different states. | s | Sum(Double) |
  • state
| | **process.disk.io** | Disk bytes transferred. | By | Sum(Int) |
  • direction
| -| **process.memory.physical_usage** | The amount of physical memory in use | By | Sum(Int) |
| -| **process.memory.virtual_usage** | Virtual memory size. | By | Sum(Int) |
| +| **process.memory.physical_usage** | Deprecated: use `process.memory.usage` metric instead. The amount of physical memory in use. | By | Sum(Int) |
| +| process.memory.usage | The amount of physical memory in use. | By | Sum(Int) |
| +| process.memory.virtual | Virtual memory size. | By | Sum(Int) |
| +| **process.memory.virtual_usage** | Deprecated: Use `process.memory.virtual` metric instead. Virtual memory size. | By | Sum(Int) |
| | process.open_file_descriptors | Number of file descriptors in use by the process. | {count} | Sum(Int) |
| | process.paging.faults | Number of page faults the process has made. This metric is only available on Linux. | {faults} | Sum(Int) |
  • paging_fault_type
| | process.threads | Process threads count. | {threads} | Sum(Int) |
| diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/metadata/generated_metrics.go b/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/metadata/generated_metrics.go index 84fc7a8720c1..4cb9bc77f3ad 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/metadata/generated_metrics.go +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/metadata/generated_metrics.go @@ -42,6 +42,8 @@ type MetricsSettings struct { ProcessCPUTime MetricSettings `mapstructure:"process.cpu.time"` ProcessDiskIo MetricSettings `mapstructure:"process.disk.io"` ProcessMemoryPhysicalUsage MetricSettings `mapstructure:"process.memory.physical_usage"` + ProcessMemoryUsage MetricSettings `mapstructure:"process.memory.usage"` + ProcessMemoryVirtual MetricSettings `mapstructure:"process.memory.virtual"` ProcessMemoryVirtualUsage MetricSettings `mapstructure:"process.memory.virtual_usage"` ProcessOpenFileDescriptors MetricSettings `mapstructure:"process.open_file_descriptors"` ProcessPagingFaults MetricSettings `mapstructure:"process.paging.faults"` @@ -62,6 +64,12 @@ func DefaultMetricsSettings() MetricsSettings { ProcessMemoryPhysicalUsage: MetricSettings{ Enabled: true, }, + ProcessMemoryUsage: MetricSettings{ + Enabled: false, + }, + ProcessMemoryVirtual: MetricSettings{ + Enabled: false, + }, ProcessMemoryVirtualUsage: MetricSettings{ Enabled: true, }, @@ -353,7 +361,7 @@ type metricProcessMemoryPhysicalUsage struct { // init fills process.memory.physical_usage metric with initial data. func (m *metricProcessMemoryPhysicalUsage) init() { m.data.SetName("process.memory.physical_usage") - m.data.SetDescription("The amount of physical memory in use") + m.data.SetDescription("Deprecated: use `process.memory.usage` metric instead. The amount of physical memory in use.") m.data.SetUnit("By") m.data.SetEmptySum() m.data.Sum().SetIsMonotonic(false) @@ -395,6 +403,108 @@ func newMetricProcessMemoryPhysicalUsage(settings MetricSettings) metricProcessM return m } +type metricProcessMemoryUsage struct { + data pmetric.Metric // data buffer for generated metric. + settings MetricSettings // metric settings provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills process.memory.usage metric with initial data. +func (m *metricProcessMemoryUsage) init() { + m.data.SetName("process.memory.usage") + m.data.SetDescription("The amount of physical memory in use.") + m.data.SetUnit("By") + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) +} + +func (m *metricProcessMemoryUsage) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.settings.Enabled { + return + } + dp := m.data.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricProcessMemoryUsage) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricProcessMemoryUsage) emit(metrics pmetric.MetricSlice) { + if m.settings.Enabled && m.data.Sum().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricProcessMemoryUsage(settings MetricSettings) metricProcessMemoryUsage { + m := metricProcessMemoryUsage{settings: settings} + if settings.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricProcessMemoryVirtual struct { + data pmetric.Metric // data buffer for generated metric. + settings MetricSettings // metric settings provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills process.memory.virtual metric with initial data. +func (m *metricProcessMemoryVirtual) init() { + m.data.SetName("process.memory.virtual") + m.data.SetDescription("Virtual memory size.") + m.data.SetUnit("By") + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) +} + +func (m *metricProcessMemoryVirtual) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.settings.Enabled { + return + } + dp := m.data.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricProcessMemoryVirtual) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricProcessMemoryVirtual) emit(metrics pmetric.MetricSlice) { + if m.settings.Enabled && m.data.Sum().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricProcessMemoryVirtual(settings MetricSettings) metricProcessMemoryVirtual { + m := metricProcessMemoryVirtual{settings: settings} + if settings.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricProcessMemoryVirtualUsage struct { data pmetric.Metric // data buffer for generated metric. settings MetricSettings // metric settings provided by user. @@ -404,7 +514,7 @@ type metricProcessMemoryVirtualUsage struct { // init fills process.memory.virtual_usage metric with initial data. func (m *metricProcessMemoryVirtualUsage) init() { m.data.SetName("process.memory.virtual_usage") - m.data.SetDescription("Virtual memory size.") + m.data.SetDescription("Deprecated: Use `process.memory.virtual` metric instead. Virtual memory size.") m.data.SetUnit("By") m.data.SetEmptySum() m.data.Sum().SetIsMonotonic(false) @@ -613,6 +723,8 @@ type MetricsBuilder struct { metricProcessCPUTime metricProcessCPUTime metricProcessDiskIo metricProcessDiskIo metricProcessMemoryPhysicalUsage metricProcessMemoryPhysicalUsage + metricProcessMemoryUsage metricProcessMemoryUsage + metricProcessMemoryVirtual metricProcessMemoryVirtual metricProcessMemoryVirtualUsage metricProcessMemoryVirtualUsage metricProcessOpenFileDescriptors metricProcessOpenFileDescriptors metricProcessPagingFaults metricProcessPagingFaults @@ -638,6 +750,8 @@ func NewMetricsBuilder(settings MetricsSettings, buildInfo component.BuildInfo, metricProcessCPUTime: newMetricProcessCPUTime(settings.ProcessCPUTime), metricProcessDiskIo: newMetricProcessDiskIo(settings.ProcessDiskIo), metricProcessMemoryPhysicalUsage: newMetricProcessMemoryPhysicalUsage(settings.ProcessMemoryPhysicalUsage), + metricProcessMemoryUsage: newMetricProcessMemoryUsage(settings.ProcessMemoryUsage), + metricProcessMemoryVirtual: newMetricProcessMemoryVirtual(settings.ProcessMemoryVirtual), metricProcessMemoryVirtualUsage: newMetricProcessMemoryVirtualUsage(settings.ProcessMemoryVirtualUsage), metricProcessOpenFileDescriptors: newMetricProcessOpenFileDescriptors(settings.ProcessOpenFileDescriptors), metricProcessPagingFaults: newMetricProcessPagingFaults(settings.ProcessPagingFaults), @@ -748,6 +862,8 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { mb.metricProcessCPUTime.emit(ils.Metrics()) mb.metricProcessDiskIo.emit(ils.Metrics()) mb.metricProcessMemoryPhysicalUsage.emit(ils.Metrics()) + mb.metricProcessMemoryUsage.emit(ils.Metrics()) + mb.metricProcessMemoryVirtual.emit(ils.Metrics()) mb.metricProcessMemoryVirtualUsage.emit(ils.Metrics()) mb.metricProcessOpenFileDescriptors.emit(ils.Metrics()) mb.metricProcessPagingFaults.emit(ils.Metrics()) @@ -791,6 +907,16 @@ func (mb *MetricsBuilder) RecordProcessMemoryPhysicalUsageDataPoint(ts pcommon.T mb.metricProcessMemoryPhysicalUsage.recordDataPoint(mb.startTime, ts, val) } +// RecordProcessMemoryUsageDataPoint adds a data point to process.memory.usage metric. +func (mb *MetricsBuilder) RecordProcessMemoryUsageDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricProcessMemoryUsage.recordDataPoint(mb.startTime, ts, val) +} + +// RecordProcessMemoryVirtualDataPoint adds a data point to process.memory.virtual metric. +func (mb *MetricsBuilder) RecordProcessMemoryVirtualDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricProcessMemoryVirtual.recordDataPoint(mb.startTime, ts, val) +} + // RecordProcessMemoryVirtualUsageDataPoint adds a data point to process.memory.virtual_usage metric. func (mb *MetricsBuilder) RecordProcessMemoryVirtualUsageDataPoint(ts pcommon.Timestamp, val int64) { mb.metricProcessMemoryVirtualUsage.recordDataPoint(mb.startTime, ts, val) diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml b/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml index 9176656def24..c75cd6aa504e 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/metadata.yaml @@ -70,7 +70,16 @@ metrics: process.memory.physical_usage: enabled: true - description: The amount of physical memory in use + description: "Deprecated: use `process.memory.usage` metric instead. The amount of physical memory in use." + unit: By + sum: + value_type: int + aggregation: cumulative + monotonic: false + + process.memory.usage: + enabled: false + description: The amount of physical memory in use. unit: By sum: value_type: int @@ -79,6 +88,15 @@ metrics: process.memory.virtual_usage: enabled: true + description: "Deprecated: Use `process.memory.virtual` metric instead. Virtual memory size." + unit: By + sum: + value_type: int + aggregation: cumulative + monotonic: false + + process.memory.virtual: + enabled: false description: Virtual memory size. unit: By sum: diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go index 2374d5ea7fee..89e7619171b2 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go @@ -237,6 +237,8 @@ func (s *scraper) scrapeAndAppendMemoryUsageMetrics(now pcommon.Timestamp, handl s.mb.RecordProcessMemoryPhysicalUsageDataPoint(now, int64(mem.RSS)) s.mb.RecordProcessMemoryVirtualUsageDataPoint(now, int64(mem.VMS)) + s.mb.RecordProcessMemoryUsageDataPoint(now, int64(mem.RSS)) + s.mb.RecordProcessMemoryVirtualDataPoint(now, int64(mem.VMS)) return nil } diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper_test.go b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper_test.go index 9a7c4b48f60d..117559399a22 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper_test.go +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper_test.go @@ -63,7 +63,7 @@ func TestScrape(t *testing.T) { } testCases := []testCase{ { - name: "Standard", + name: "Default set of metrics", }, { name: "Enable Linux-only metrics",