Skip to content

Commit

Permalink
[receiver/elasticsearch]: add fielddata memory size metrics on index …
Browse files Browse the repository at this point in the history 
…level (open-telemetry#14922)

feat: add fielddata memory size metrics on index level
  • Loading branch information
@aboguszewski-sumo @shalper2
aboguszewski-sumo authored and shalper2 committed Dec 6, 2022
1 parent c0bab00 commit 74b6099
Show file tree
Hide file tree
Showing 8 changed files with 203 additions and 5 deletions.
16 changes: 16 additions & 0 deletions .chloggen/elasticsearch-fielddata.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: elasticsearchreceiver

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: add fielddata memory size metrics on index level

# One or more tracking issues related to the change
issues: [14635]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
1 change: 1 addition & 0 deletions receiver/elasticsearchreceiver/documentation.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ These are the metrics available for this scraper.
| **elasticsearch.cluster.state_queue** | Number of cluster states in queue. | 1 | Sum(Int) | <ul> <li>cluster_state_queue_state</li> </ul> |
| **elasticsearch.cluster.state_update.count** | The number of cluster state update attempts that changed the cluster state since the node started. | 1 | Sum(Int) | <ul> <li>cluster_state_update_state</li> </ul> |
| **elasticsearch.cluster.state_update.time** | The cumulative amount of time updating the cluster state since the node started. | ms | Sum(Int) | <ul> <li>cluster_state_update_state</li> <li>cluster_state_update_type</li> </ul> |
| elasticsearch.index.cache.memory.usage | The size in bytes of the cache for an index. | By | Sum(Int) | <ul> <li>cache_name</li> <li>index_aggregation_type</li> </ul> |
| **elasticsearch.index.operations.completed** | The number of operations completed for an index. | {operations} | Sum(Int) | <ul> <li>operation</li> <li>index_aggregation_type</li> </ul> |
| elasticsearch.index.operations.merge.docs_count | The total number of documents in merge operations for an index. | {documents} | Sum(Int) | <ul> <li>index_aggregation_type</li> </ul> |
| elasticsearch.index.operations.merge.size | The total size of merged segments for an index. | By | Sum(Int) | <ul> <li>index_aggregation_type</li> </ul> |
Expand Down
66 changes: 66 additions & 0 deletions receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions receiver/elasticsearchreceiver/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -869,3 +869,12 @@ metrics:
value_type: int
attributes: [index_aggregation_type]
enabled: false
elasticsearch.index.cache.memory.usage:
description: The size in bytes of the cache for an index.
unit: By
sum:
monotonic: false
aggregation: cumulative
value_type: int
attributes: [cache_name, index_aggregation_type]
enabled: false
9 changes: 8 additions & 1 deletion receiver/elasticsearchreceiver/scraper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ func (r *elasticsearchScraper) scrapeIndicesMetrics(ctx context.Context, now pco
indexStats, err := r.client.IndexStats(ctx, r.cfg.Indices)

if err != nil {
errs.AddPartial(16, err)
errs.AddPartial(18, err)
return
}

Expand Down Expand Up @@ -452,5 +452,12 @@ func (r *elasticsearchScraper) scrapeOneIndexMetrics(now pcommon.Timestamp, name
now, stats.Total.TranslogStats.SizeInBy, metadata.AttributeIndexAggregationTypeTotal,
)

r.mb.RecordElasticsearchIndexCacheMemoryUsageDataPoint(
now, stats.Primaries.FieldDataCache.MemorySizeInBy, metadata.AttributeCacheNameFielddata, metadata.AttributeIndexAggregationTypePrimaryShards,
)
r.mb.RecordElasticsearchIndexCacheMemoryUsageDataPoint(
now, stats.Total.FieldDataCache.MemorySizeInBy, metadata.AttributeCacheNameFielddata, metadata.AttributeIndexAggregationTypeTotal,
)

r.mb.EmitForResource(metadata.WithElasticsearchIndexName(name), metadata.WithElasticsearchClusterName(r.clusterName))
}
1 change: 1 addition & 0 deletions receiver/elasticsearchreceiver/scraper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func TestScraper(t *testing.T) {
config.Metrics.ElasticsearchIndexSegmentsMemory.Enabled = true
config.Metrics.ElasticsearchIndexTranslogOperations.Enabled = true
config.Metrics.ElasticsearchIndexTranslogSize.Enabled = true
config.Metrics.ElasticsearchIndexCacheMemoryUsage.Enabled = true

sc := newElasticSearchScraper(componenttest.NewNopReceiverCreateSettings(), config)

Expand Down
98 changes: 98 additions & 0 deletions receiver/elasticsearchreceiver/testdata/expected_metrics/full.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2958,6 +2958,55 @@
]
},
"unit": "By"
},
{
"description": "The size in bytes of the cache for an index.",
"name": "elasticsearch.index.cache.memory.usage",
"sum": {
"aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE",
"isMonotonic": false,
"dataPoints": [
{
"asInt": "3",
"attributes": [
{
"key": "cache_name",
"value": {
"stringValue": "fielddata"
}
},
{
"key": "aggregation",
"value": {
"stringValue": "primary_shards"
}
}
],
"startTimeUnixNano": "1661811689941624000",
"timeUnixNano": "1661811689943245000"
},
{
"asInt": "3",
"attributes": [
{
"key": "cache_name",
"value": {
"stringValue": "fielddata"
}
},
{
"key": "aggregation",
"value": {
"stringValue": "total"
}
}
],
"startTimeUnixNano": "1661811689941624000",
"timeUnixNano": "1661811689943245000"
}
]
},
"unit": "By"
}
],
"scope": {
Expand Down Expand Up @@ -3460,6 +3509,55 @@
]
},
"unit": "By"
},
{
"description": "The size in bytes of the cache for an index.",
"name": "elasticsearch.index.cache.memory.usage",
"sum": {
"aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE",
"isMonotonic": false,
"dataPoints": [
{
"asInt": "3",
"attributes": [
{
"key": "cache_name",
"value": {
"stringValue": "fielddata"
}
},
{
"key": "aggregation",
"value": {
"stringValue": "primary_shards"
}
}
],
"startTimeUnixNano": "1661811689941624000",
"timeUnixNano": "1661811689943245000"
},
{
"asInt": "3",
"attributes": [
{
"key": "cache_name",
"value": {
"stringValue": "fielddata"
}
},
{
"key": "aggregation",
"value": {
"stringValue": "total"
}
}
],
"startTimeUnixNano": "1661811689941624000",
"timeUnixNano": "1661811689943245000"
}
]
},
"unit": "By"
}
],
"scope": {
Expand Down
8 changes: 4 additions & 4 deletions receiver/elasticsearchreceiver/testdata/sample_payloads/indices.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"memory_size_in_bytes" : 3,
"evictions" : 0
},
"completion" : {
Expand Down Expand Up @@ -221,7 +221,7 @@
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"memory_size_in_bytes" : 3,
"evictions" : 0
},
"completion" : {
Expand Down Expand Up @@ -353,7 +353,7 @@
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"memory_size_in_bytes" : 3,
"evictions" : 0
},
"completion" : {
Expand Down Expand Up @@ -481,7 +481,7 @@
"evictions" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"memory_size_in_bytes" : 3,
"evictions" : 0
},
"completion" : {
Expand Down

0 comments on commit 74b6099

Please sign in to comment.