Recovery: add throttle stats

This commit adds throttling statistics to the index stats API and to the recovery state. Closes #10097
elastic · Mar 20, 2015 · 8c69535 · 8c69535
1 parent 4348959
commit 8c69535
Show file tree

Hide file tree

Showing 29 changed files with 511 additions and 100 deletions.
diff --git a/rest-api-spec/test/indices.recovery/10_basic.yaml b/rest-api-spec/test/indices.recovery/10_basic.yaml
@@ -11,29 +11,29 @@
 
   - do:
       cluster.health:
-        wait_for_status: yellow
+        wait_for_status: green
 
   - do:
       indices.recovery:
         index: [test_1]
 
-  - match: { test_1.shards.0.type:                              "GATEWAY"               }
-  - match: { test_1.shards.0.stage:                             "DONE"                  }
-  - match: { test_1.shards.0.primary:                           true                    }
-  - match: { test_1.shards.0.target.ip:                         /^\d+\.\d+\.\d+\.\d+$/  }
-  - gte:   { test_1.shards.0.index.files.total:                 0                       }
-  - gte:   { test_1.shards.0.index.files.reused:                0                       }
-  - gte:   { test_1.shards.0.index.files.recovered:             0                       }
-  - match: { test_1.shards.0.index.files.percent:               /^\d+\.\d\%$/           }
-  - gte:   { test_1.shards.0.index.size.total_in_bytes:         0                       }
-  - gte:   { test_1.shards.0.index.size.reused_in_bytes:        0                       }
-  - gte:   { test_1.shards.0.index.size.recovered_in_bytes:     0                       }
-  - match: { test_1.shards.0.index.size.percent:                /^\d+\.\d\%$/           }
-  - gte:   { test_1.shards.0.translog.recovered:                0                       }
-  - gte:   { test_1.shards.0.translog.total:                    -1                      }
-  - gte:   { test_1.shards.0.translog.total_on_start:           0                       }
-  - gte:   { test_1.shards.0.translog.total_time_in_millis:     0                       }
-  - gte:   { test_1.shards.0.start.check_index_time_in_millis:  0                       }
-  - gte:   { test_1.shards.0.start.total_time_in_millis:        0                       }
-
-
+  - match: { test_1.shards.0.type:                                 "GATEWAY"               }
+  - match: { test_1.shards.0.stage:                                "DONE"                  }
+  - match: { test_1.shards.0.primary:                              true                    }
+  - match: { test_1.shards.0.target.ip:                            /^\d+\.\d+\.\d+\.\d+$/  }
+  - gte:   { test_1.shards.0.index.files.total:                    0                       }
+  - gte:   { test_1.shards.0.index.files.reused:                   0                       }
+  - gte:   { test_1.shards.0.index.files.recovered:                0                       }
+  - match: { test_1.shards.0.index.files.percent:                  /^\d+\.\d\%$/           }
+  - gte:   { test_1.shards.0.index.size.total_in_bytes:            0                       }
+  - gte:   { test_1.shards.0.index.size.reused_in_bytes:           0                       }
+  - gte:   { test_1.shards.0.index.size.recovered_in_bytes:        0                       }
+  - match: { test_1.shards.0.index.size.percent:                   /^\d+\.\d\%$/           }
+  - gte:   { test_1.shards.0.index.source_throttle_time_in_millis: 0                       }
+  - gte:   { test_1.shards.0.index.target_throttle_time_in_millis: 0                       }
+  - gte:   { test_1.shards.0.translog.recovered:                   0                       }
+  - gte:   { test_1.shards.0.translog.total:                       -1                      }
+  - gte:   { test_1.shards.0.translog.total_on_start:              0                       }
+  - gte:   { test_1.shards.0.translog.total_time_in_millis:        0                       }
+  - gte:   { test_1.shards.0.start.check_index_time_in_millis:     0                       }
+  - gte:   { test_1.shards.0.start.total_time_in_millis:           0                       }
diff --git a/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/test/indices.stats/11_metric.yaml
@@ -37,6 +37,7 @@ setup:
   - is_true:  _all.total.segments
   - is_true:  _all.total.translog
   - is_true:  _all.total.suggest
+  - is_true:  _all.total.recovery
 
 ---
 "Metric - _all":
@@ -60,6 +61,7 @@ setup:
   - is_true:  _all.total.segments
   - is_true:  _all.total.translog
   - is_true:  _all.total.suggest
+  - is_true:  _all.total.recovery
 
 ---
 "Metric - one":
@@ -83,6 +85,7 @@ setup:
   - is_false:  _all.total.segments
   - is_false:  _all.total.translog
   - is_false:  _all.total.suggest
+  - is_false:  _all.total.recovery
 
 ---
 "Metric - multi":
@@ -106,5 +109,30 @@ setup:
   - is_false:  _all.total.segments
   - is_false:  _all.total.translog
   - is_false:  _all.total.suggest
+  - is_false:  _all.total.recovery
 
 
+---
+"Metric - recovery":
+  - do:
+      indices.stats: { metric: [ recovery ] }
+
+  - is_false:  _all.total.docs
+  - is_false:  _all.total.store
+  - is_false:  _all.total.indexing
+  - is_false:  _all.total.get
+  - is_false:  _all.total.search
+  - is_false:  _all.total.merges
+  - is_false:  _all.total.refresh
+  - is_false:  _all.total.flush
+  - is_false:  _all.total.warmer
+  - is_false:  _all.total.filter_cache
+  - is_false:  _all.total.id_cache
+  - is_false:  _all.total.fielddata
+  - is_false:  _all.total.percolate
+  - is_false:  _all.total.completion
+  - is_false:  _all.total.segments
+  - is_false:  _all.total.translog
+  - is_false:  _all.total.suggest
+  - is_true:   _all.total.recovery
+
diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.percolator.stats.PercolateStats;
+import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
@@ -115,6 +116,9 @@ public CommonStats(CommonStatsFlags flags) {
                 case QueryCache:
                     queryCache = new QueryCacheStats();
                     break;
+                case Recovery:
+                    recoveryStats = new RecoveryStats();
+                    break;
                 default:
                     throw new IllegalStateException("Unknown Flag: " + flag);
             }
@@ -181,6 +185,9 @@ public CommonStats(IndexShard indexShard, CommonStatsFlags flags) {
                 case QueryCache:
                     queryCache = indexShard.queryCache().stats();
                     break;
+                case Recovery:
+                    recoveryStats = indexShard.recoveryStats();
+                    break;
                 default:
                     throw new IllegalStateException("Unknown Flag: " + flag);
             }
@@ -241,6 +248,9 @@ public CommonStats(IndexShard indexShard, CommonStatsFlags flags) {
     @Nullable
     public QueryCacheStats queryCache;
 
+    @Nullable
+    public RecoveryStats recoveryStats;
+
     public void add(CommonStats stats) {
         if (docs == null) {
             if (stats.getDocs() != null) {
@@ -388,6 +398,14 @@ public void add(CommonStats stats) {
         } else {
             queryCache.add(stats.getQueryCache());
         }
+        if (recoveryStats == null) {
+            if (stats.getRecoveryStats() != null) {
+                recoveryStats = new RecoveryStats();
+                recoveryStats.add(stats.getRecoveryStats());
+            }
+        } else {
+            recoveryStats.add(stats.getRecoveryStats());
+        }
     }
 
     @Nullable
@@ -480,6 +498,11 @@ public QueryCacheStats getQueryCache() {
         return queryCache;
     }
 
+    @Nullable
+    public RecoveryStats getRecoveryStats() {
+        return recoveryStats;
+    }
+
     public static CommonStats readCommonStats(StreamInput in) throws IOException {
         CommonStats stats = new CommonStats();
         stats.readFrom(in);
@@ -563,6 +586,7 @@ public void readFrom(StreamInput in) throws IOException {
         translog = in.readOptionalStreamable(new TranslogStats());
         suggest = in.readOptionalStreamable(new SuggestStats());
         queryCache = in.readOptionalStreamable(new QueryCacheStats());
+        recoveryStats = in.readOptionalStreamable(new RecoveryStats());
     }
 
     @Override
@@ -660,6 +684,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalStreamable(translog);
         out.writeOptionalStreamable(suggest);
         out.writeOptionalStreamable(queryCache);
+        out.writeOptionalStreamable(recoveryStats);
     }
 
     // note, requires a wrapping object
@@ -719,6 +744,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (queryCache != null) {
             queryCache.toXContent(builder, params);
         }
+        if (recoveryStats != null) {
+            recoveryStats.toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -225,7 +225,9 @@ public static enum Flag {
         Segments("segments"),
         Translog("translog"),
         Suggest("suggest"),
-        QueryCache("query_cache");
+        QueryCache("query_cache"),
+        Recovery("recovery");
+
 
         private final String restName;
 

diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
@@ -265,6 +265,15 @@ public boolean queryCache() {
         return flags.isSet(Flag.QueryCache);
     }
 
+    public IndicesStatsRequest recovery(boolean recovery) {
+        flags.set(Flag.Recovery, recovery);
+        return this;
+    }
+
+    public boolean recovery() {
+        return flags.isSet(Flag.Recovery);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);

diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
@@ -168,6 +168,11 @@ public IndicesStatsRequestBuilder setQueryCache(boolean queryCache) {
         return this;
     }
 
+    public IndicesStatsRequestBuilder setRecovery(boolean recovery) {
+        request.recovery(recovery);
+        return this;
+    }
+
     @Override
     protected void doExecute(ActionListener<IndicesStatsResponse> listener) {
         client.stats(request, listener);

diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -37,10 +37,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexShardMissingException;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.IndexShardMissingException;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -201,6 +201,9 @@ protected ShardStats shardOperation(IndexShardStatsRequest request) throws Elast
         if (request.request.queryCache()) {
             flags.set(CommonStatsFlags.Flag.QueryCache);
         }
+        if (request.request.recovery()) {
+            flags.set(CommonStatsFlags.Flag.Recovery);
+        }
 
         return new ShardStats(indexShard, indexShard.routingEntry(), flags);
     }