test: refactor TestNodesStats

As per prometheus-community#419, this functionality is failing for elasticsearch version 7.13+ due to a breaking change in the response from elasticsearch. This adds some test fixtures for a range of elasticsearch versions including 7.13 to help ensure that future changes fix the incompatibility with 7.13 and maintain compatibility with versions < 7.13.

Signed-off-by: Joe Adams <[email protected]>

collector/versions_test.go

@@ -0,0 +1,24 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+var testElasticsearchVersions = []string{
+	"5.4.2",
+	"5.6.16",
+	"6.5.4",
+	"6.8.8",
+	"7.3.0",
+	"7.6.2",
+	"7.13.1",
+}

fixtures/nodestats/5.4.2.json

@@ -0,0 +1 @@
+{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"oDXKc-AOSEG9KfDb-qc7Vg":{"timestamp":1622148124411,"name":"oDXKc-A","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":5337,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":47,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":629145600},"refresh":{"total":5,"total_time_in_millis":98,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":35,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":12940,"terms_memory_in_bytes":9960,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":5,"size_in_bytes":1743},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1622148124416,"cpu":{"percent":29,"load_average":{"1m":62.46,"5m":160.53,"15m":98.03}},"mem":{"total_in_bytes":33623236608,"free_in_bytes":7032000512,"used_in_bytes":26591236096,"free_percent":21,"used_percent":79},"swap":{"total_in_bytes":2147479552,"free_in_bytes":85319680,"used_in_bytes":2062159872}},"process":{"timestamp":1622148124416,"open_file_descriptors":305,"max_file_descriptors":1048576,"cpu":{"percent":9,"total_in_millis":13610},"mem":{"total_virtual_in_bytes":8294236160}},"jvm":{"timestamp":1622148124416,"uptime_in_millis":13951,"mem":{"heap_used_in_bytes":569163128,"heap_used_percent":27,"heap_committed_in_bytes":2077753344,"heap_max_in_bytes":2077753344,"non_heap_used_in_bytes":68846616,"non_heap_committed_in_bytes":73859072,"pools":{"young":{"used_in_bytes":518547608,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":50615520,"max_in_bytes":69730304,"peak_used_in_bytes":50615520,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":0,"max_in_bytes":1449590784,"peak_used_in_bytes":0,"peak_max_in_bytes":1449590784}}},"threads":{"count":61,"peak_count":61},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":24},"old":{"collection_count":1,"collection_time_in_millis":120}}},"buffer_pools":{"direct":{"count":47,"used_in_bytes":252727741,"total_capacity_in_bytes":252727740},"mapped":{"count":5,"used_in_bytes":15007,"total_capacity_in_bytes":15007}},"classes":{"current_loaded_count":10411,"total_loaded_count":10411,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":5,"queue":0,"active":0,"rejected":0,"largest":5,"completed":5},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":38},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":5},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":2},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1622148124418,"total":{"total_in_bytes":476630163456,"free_in_bytes":77533327360,"available_in_bytes":77533327360,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/mapper/vg0-root)","type":"xfs","total_in_bytes":476630163456,"free_in_bytes":77533327360,"available_in_bytes":77533327360,"spins":"true"}],"io_stats":{"devices":[{"device_name":"dm-2","operations":5916,"read_operations":4854,"write_operations":1062,"read_kilobytes":71588,"write_kilobytes":8340}],"total":{"operations":5916,"read_operations":4854,"write_operations":1062,"read_kilobytes":71588,"write_kilobytes":8340}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":1,"total_opened":16},"breakers":{"request":{"limit_size_in_bytes":1246652006,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1246652006,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2077753344,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1454427340,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}

fixtures/nodestats/5.6.16.json

@@ -0,0 +1 @@
+{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"bVrN1HxvQLy795ZLNg2XNw":{"timestamp":1622148141237,"name":"bVrN1Hx","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":5721,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":39,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":629145600},"refresh":{"total":65,"total_time_in_millis":86,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":35,"total_time_in_millis":1},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":12940,"terms_memory_in_bytes":9960,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":5,"size_in_bytes":1743},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1622148141242,"cpu":{"percent":23,"load_average":{"1m":45.68,"5m":150.34,"15m":96.01}},"mem":{"total_in_bytes":33623236608,"free_in_bytes":7009173504,"used_in_bytes":26614063104,"free_percent":21,"used_percent":79},"swap":{"total_in_bytes":2147479552,"free_in_bytes":86630400,"used_in_bytes":2060849152}},"process":{"timestamp":1622148141242,"open_file_descriptors":308,"max_file_descriptors":1048576,"cpu":{"percent":8,"total_in_millis":14510},"mem":{"total_virtual_in_bytes":8293711872}},"jvm":{"timestamp":1622148141243,"uptime_in_millis":14845,"mem":{"heap_used_in_bytes":333706928,"heap_used_percent":16,"heap_committed_in_bytes":2077753344,"heap_max_in_bytes":2077753344,"non_heap_used_in_bytes":70212664,"non_heap_committed_in_bytes":75362304,"pools":{"young":{"used_in_bytes":53925336,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":69730304,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":210051288,"max_in_bytes":1449590784,"peak_used_in_bytes":210051288,"peak_max_in_bytes":1449590784}}},"threads":{"count":60,"peak_count":60},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":143},"old":{"collection_count":1,"collection_time_in_millis":109}}},"buffer_pools":{"direct":{"count":47,"used_in_bytes":252727869,"total_capacity_in_bytes":252727868},"mapped":{"count":5,"used_in_bytes":15007,"total_capacity_in_bytes":15007}},"classes":{"current_loaded_count":10502,"total_loaded_count":10502,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":5,"queue":0,"active":0,"rejected":0,"largest":5,"completed":5},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":38},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":5},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":2},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1622148141244,"total":{"total_in_bytes":476630163456,"free_in_bytes":77533405184,"available_in_bytes":77533405184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/mapper/vg0-root)","type":"xfs","total_in_bytes":476630163456,"free_in_bytes":77533405184,"available_in_bytes":77533405184,"spins":"true"}],"io_stats":{"devices":[{"device_name":"dm-2","operations":2517,"read_operations":706,"write_operations":1811,"read_kilobytes":12916,"write_kilobytes":17760}],"total":{"operations":2517,"read_operations":706,"write_operations":1811,"read_kilobytes":12916,"write_kilobytes":17760}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":1,"total_opened":16},"breakers":{"request":{"limit_size_in_bytes":1246652006,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1246652006,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2077753344,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1454427340,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}