-
Notifications
You must be signed in to change notification settings - Fork 24.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] Return statistics about forecasts as part of the jobsstats and u…
…sage API (#31647) This change adds stats about forecasts, to the jobstats api as well as xpack/_usage. The following information is collected: _xpack/ml/anomaly_detectors/{jobid|_all}/_stats: - total number of forecasts - memory statistics (mean/min/max) - runtime statistics - record statistics - counts by status _xpack/usage - collected by job status as well as overall (_all): - total number of forecasts - number of jobs that have at least 1 forecast - memory, runtime, record statistics - counts by status Fixes #31395
- Loading branch information
Hendrik Muhs
committed
Jul 4, 2018
1 parent
1ff7066
commit 85177b6
Showing
18 changed files
with
1,071 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 82 additions & 0 deletions
82
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.core.ml.stats; | ||
|
|
||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.io.stream.Writeable; | ||
| import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.Map.Entry; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| /** | ||
| * An accumulator for simple counts where statistical measures | ||
| * are not of interest. | ||
| */ | ||
| public class CountAccumulator implements Writeable { | ||
|
|
||
| private Map<String, Long> counts; | ||
|
|
||
| public CountAccumulator() { | ||
| this.counts = new HashMap<String, Long>(); | ||
| } | ||
|
|
||
| private CountAccumulator(Map<String, Long> counts) { | ||
| this.counts = counts; | ||
| } | ||
|
|
||
| public CountAccumulator(StreamInput in) throws IOException { | ||
| this.counts = in.readMap(StreamInput::readString, StreamInput::readLong); | ||
| } | ||
|
|
||
| public void merge(CountAccumulator other) { | ||
| counts = Stream.of(counts, other.counts).flatMap(m -> m.entrySet().stream()) | ||
| .collect(Collectors.toMap(Entry::getKey, Entry::getValue, (x, y) -> x + y)); | ||
| } | ||
|
|
||
| public void add(String key, Long count) { | ||
| counts.put(key, counts.getOrDefault(key, 0L) + count); | ||
| } | ||
|
|
||
| public Map<String, Long> asMap() { | ||
| return counts; | ||
| } | ||
|
|
||
| public static CountAccumulator fromTermsAggregation(StringTerms termsAggregation) { | ||
| return new CountAccumulator(termsAggregation.getBuckets().stream() | ||
| .collect(Collectors.toMap(bucket -> bucket.getKeyAsString(), bucket -> bucket.getDocCount()))); | ||
| } | ||
|
|
||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeMap(counts, StreamOutput::writeString, StreamOutput::writeLong); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(counts); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == null) { | ||
| return false; | ||
| } | ||
|
|
||
| if (getClass() != obj.getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| CountAccumulator other = (CountAccumulator) obj; | ||
| return Objects.equals(counts, other.counts); | ||
| } | ||
| } |
152 changes: 152 additions & 0 deletions
152
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/stats/ForecastStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.core.ml.stats; | ||
|
|
||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.io.stream.Writeable; | ||
| import org.elasticsearch.common.xcontent.ToXContentObject; | ||
| import org.elasticsearch.common.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| /** | ||
| * A class to hold statistics about forecasts. | ||
| */ | ||
| public class ForecastStats implements ToXContentObject, Writeable { | ||
|
|
||
| public static class Fields { | ||
| public static final String TOTAL = "total"; | ||
| public static final String FORECASTED_JOBS = "forecasted_jobs"; | ||
| public static final String MEMORY = "memory_bytes"; | ||
| public static final String RUNTIME = "processing_time_ms"; | ||
| public static final String RECORDS = "records"; | ||
| public static final String STATUSES = "status"; | ||
| } | ||
|
|
||
| private long total; | ||
| private long forecastedJobs; | ||
| private StatsAccumulator memoryStats; | ||
| private StatsAccumulator recordStats; | ||
| private StatsAccumulator runtimeStats; | ||
| private CountAccumulator statusCounts; | ||
|
|
||
| public ForecastStats() { | ||
| this.total = 0; | ||
| this.forecastedJobs = 0; | ||
| this.memoryStats = new StatsAccumulator(); | ||
| this.recordStats = new StatsAccumulator(); | ||
| this.runtimeStats = new StatsAccumulator(); | ||
| this.statusCounts = new CountAccumulator(); | ||
| } | ||
|
|
||
| /* | ||
| * Construct ForecastStats for 1 job. Additional statistics can be added by merging other ForecastStats into it. | ||
| */ | ||
| public ForecastStats(long total, StatsAccumulator memoryStats, StatsAccumulator recordStats, StatsAccumulator runtimeStats, | ||
| CountAccumulator statusCounts) { | ||
| this.total = total; | ||
| this.forecastedJobs = total > 0 ? 1 : 0; | ||
| this.memoryStats = Objects.requireNonNull(memoryStats); | ||
| this.recordStats = Objects.requireNonNull(recordStats); | ||
| this.runtimeStats = Objects.requireNonNull(runtimeStats); | ||
| this.statusCounts = Objects.requireNonNull(statusCounts); | ||
| } | ||
|
|
||
| public ForecastStats(StreamInput in) throws IOException { | ||
| this.total = in.readLong(); | ||
| this.forecastedJobs = in.readLong(); | ||
| this.memoryStats = new StatsAccumulator(in); | ||
| this.recordStats = new StatsAccumulator(in); | ||
| this.runtimeStats = new StatsAccumulator(in); | ||
| this.statusCounts = new CountAccumulator(in); | ||
| } | ||
|
|
||
| public ForecastStats merge(ForecastStats other) { | ||
| if (other == null) { | ||
| return this; | ||
| } | ||
| total += other.total; | ||
| forecastedJobs += other.forecastedJobs; | ||
| memoryStats.merge(other.memoryStats); | ||
| recordStats.merge(other.recordStats); | ||
| runtimeStats.merge(other.runtimeStats); | ||
| statusCounts.merge(other.statusCounts); | ||
|
|
||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| doXContentBody(builder, params); | ||
| return builder.endObject(); | ||
| } | ||
|
|
||
| public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { | ||
| builder.field(Fields.TOTAL, total); | ||
| builder.field(Fields.FORECASTED_JOBS, forecastedJobs); | ||
|
|
||
| if (total > 0) { | ||
| builder.field(Fields.MEMORY, memoryStats.asMap()); | ||
| builder.field(Fields.RECORDS, recordStats.asMap()); | ||
| builder.field(Fields.RUNTIME, runtimeStats.asMap()); | ||
| builder.field(Fields.STATUSES, statusCounts.asMap()); | ||
| } | ||
|
|
||
| return builder; | ||
| } | ||
|
|
||
| public Map<String, Object> asMap() { | ||
| Map<String, Object> map = new HashMap<>(); | ||
| map.put(Fields.TOTAL, total); | ||
| map.put(Fields.FORECASTED_JOBS, forecastedJobs); | ||
|
|
||
| if (total > 0) { | ||
| map.put(Fields.MEMORY, memoryStats.asMap()); | ||
| map.put(Fields.RECORDS, recordStats.asMap()); | ||
| map.put(Fields.RUNTIME, runtimeStats.asMap()); | ||
| map.put(Fields.STATUSES, statusCounts.asMap()); | ||
| } | ||
|
|
||
| return map; | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeLong(total); | ||
| out.writeLong(forecastedJobs); | ||
| memoryStats.writeTo(out); | ||
| recordStats.writeTo(out); | ||
| runtimeStats.writeTo(out); | ||
| statusCounts.writeTo(out); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(total, forecastedJobs, memoryStats, recordStats, runtimeStats, statusCounts); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (obj == null) { | ||
| return false; | ||
| } | ||
|
|
||
| if (getClass() != obj.getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| ForecastStats other = (ForecastStats) obj; | ||
| return Objects.equals(total, other.total) && Objects.equals(forecastedJobs, other.forecastedJobs) | ||
| && Objects.equals(memoryStats, other.memoryStats) && Objects.equals(recordStats, other.recordStats) | ||
| && Objects.equals(runtimeStats, other.runtimeStats) && Objects.equals(statusCounts, other.statusCounts); | ||
| } | ||
| } |
Oops, something went wrong.