From 9e374b73ce013f4176457043cf81fd6270a7445a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 28 Apr 2020 20:56:46 +0100 Subject: [PATCH] [ML] Add daily_model_snapshot_retention_after_days to job config This change adds a new setting, daily_model_snapshot_retention_after_days, to the anomaly detection job config. Initially this has no effect, the effect will be added in a followup PR. This PR gets the complexities of making changes that interact with BWC over well before feature freeze. Backport of #55878 --- .../client/ml/job/config/Job.java | 33 +++++++++-- .../client/ml/job/config/JobUpdate.java | 33 +++++++++-- .../client/ml/job/config/JobTests.java | 3 + .../client/ml/job/config/JobUpdateTests.java | 3 + .../xpack/core/ml/job/config/Job.java | 57 ++++++++++++++++--- .../xpack/core/ml/job/config/JobUpdate.java | 48 ++++++++++++++-- .../xpack/core/ml/job/config/JobTests.java | 4 ++ .../core/ml/job/config/JobUpdateTests.java | 3 + .../xpack/ml/job/config/JobBuilderTests.java | 3 + 9 files changed, 161 insertions(+), 26 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java index 3b39a4bda8704..254dc23e73d4f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java @@ -63,6 +63,8 @@ public class Job implements ToXContentObject { public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days"); public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval"); public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days"); + public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = + new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name"); @@ -93,6 +95,7 @@ public class Job implements ToXContentObject { TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID); PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME); @@ -114,6 +117,7 @@ public class Job implements ToXContentObject { private final Long renormalizationWindowDays; private final TimeValue backgroundPersistInterval; private final Long modelSnapshotRetentionDays; + private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; private final Map customSettings; private final String modelSnapshotId; @@ -125,8 +129,9 @@ private Job(String jobId, String jobType, List groups, String descriptio Date createTime, Date finishedTime, AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, - Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map customSettings, - String modelSnapshotId, String resultsIndexName, Boolean deleting, Boolean allowLazyOpen) { + Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, + Map customSettings, String modelSnapshotId, String resultsIndexName, Boolean deleting, + Boolean allowLazyOpen) { this.jobId = jobId; this.jobType = jobType; @@ -141,6 +146,7 @@ private Job(String jobId, String jobType, List groups, String descriptio this.renormalizationWindowDays = renormalizationWindowDays; this.backgroundPersistInterval = backgroundPersistInterval; this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; @@ -259,6 +265,10 @@ public Long getModelSnapshotRetentionDays() { return modelSnapshotRetentionDays; } + public Long getDailyModelSnapshotRetentionAfterDays() { + return dailyModelSnapshotRetentionAfterDays; + } + public Long getResultsRetentionDays() { return resultsRetentionDays; } @@ -319,6 +329,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelSnapshotRetentionDays != null) { builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } @@ -365,6 +378,7 @@ public boolean equals(Object other) { && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) @@ -377,8 +391,8 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, - modelSnapshotId, resultsIndexName, deleting, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen); } @Override @@ -405,6 +419,7 @@ public static class Builder { private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; private Long modelSnapshotRetentionDays; + private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; private Map customSettings; private String modelSnapshotId; @@ -433,6 +448,7 @@ public Builder(Job job) { this.renormalizationWindowDays = job.getRenormalizationWindowDays(); this.backgroundPersistInterval = job.getBackgroundPersistInterval(); this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); + this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); @@ -515,6 +531,11 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { return this; } + public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) { + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; + return this; + } + public Builder setResultsRetentionDays(Long resultsRetentionDays) { this.resultsRetentionDays = resultsRetentionDays; return this; @@ -551,8 +572,8 @@ public Job build() { return new Job( id, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, - modelSnapshotId, resultsIndexName, deleting, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java index f91918d456817..c68ba523bd507 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java @@ -52,6 +52,7 @@ public class JobUpdate implements ToXContentObject { PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); PARSER.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN); @@ -66,6 +67,7 @@ public class JobUpdate implements ToXContentObject { private final Long renormalizationWindowDays; private final TimeValue backgroundPersistInterval; private final Long modelSnapshotRetentionDays; + private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; private final List categorizationFilters; private final Map customSettings; @@ -75,7 +77,8 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long modelSnapshotRetentionDays, @Nullable List categorisationFilters, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, + @Nullable List categorizationFilters, @Nullable Map customSettings, @Nullable Boolean allowLazyOpen) { this.jobId = jobId; this.groups = groups; @@ -86,8 +89,9 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.renormalizationWindowDays = renormalizationWindowDays; this.backgroundPersistInterval = backgroundPersistInterval; this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.categorizationFilters = categorisationFilters; + this.categorizationFilters = categorizationFilters; this.customSettings = customSettings; this.allowLazyOpen = allowLazyOpen; } @@ -172,6 +176,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelSnapshotRetentionDays != null) { builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.field(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } @@ -209,6 +216,7 @@ public boolean equals(Object other) { && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.customSettings, that.customSettings) @@ -218,8 +226,8 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings, - allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + categorizationFilters, customSettings, allowLazyOpen); } public static class DetectorUpdate implements ToXContentObject { @@ -312,6 +320,7 @@ public static class Builder { private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; private Long modelSnapshotRetentionDays; + private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; private List categorizationFilters; private Map customSettings; @@ -422,6 +431,18 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { return this; } + /** + * The time in days after which only one model snapshot per day is retained for the job. + * + * Updates the {@link Job#dailyModelSnapshotRetentionAfterDays} setting + * + * @param dailyModelSnapshotRetentionAfterDays number of days to keep a model snapshot + */ + public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) { + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; + return this; + } + /** * Advanced configuration option. The number of days for which job results are retained * @@ -466,8 +487,8 @@ public Builder setAllowLazyOpen(boolean allowLazyOpen) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings, - allowLazyOpen); + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, + categorizationFilters, customSettings, allowLazyOpen); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java index 0ea3f7903acf3..1586931ebca1f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java @@ -144,6 +144,9 @@ public static Job.Builder createRandomizedJobBuilder() { if (randomBoolean()) { builder.setModelSnapshotRetentionDays(randomNonNegativeLong()); } + if (randomBoolean()) { + builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong()); + } if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java index 8d2d02879a64f..dbe81550a6821 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java @@ -70,6 +70,9 @@ public static JobUpdate createRandom(String jobId) { if (randomBoolean()) { update.setModelSnapshotRetentionDays(randomNonNegativeLong()); } + if (randomBoolean()) { + update.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong()); + } if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 5034a83380ee6..8ef2991a94221 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -72,6 +72,8 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days"); public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval"); public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days"); + public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = + new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField MODEL_SNAPSHOT_MIN_VERSION = new ParseField("model_snapshot_min_version"); @@ -123,6 +125,7 @@ private static ObjectParser createParser(boolean ignoreUnknownFie TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); + parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); parser.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID); parser.declareStringOrNull(Builder::setModelSnapshotMinVersion, MODEL_SNAPSHOT_MIN_VERSION); @@ -155,6 +158,7 @@ private static ObjectParser createParser(boolean ignoreUnknownFie private final Long renormalizationWindowDays; private final TimeValue backgroundPersistInterval; private final Long modelSnapshotRetentionDays; + private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; private final Map customSettings; private final String modelSnapshotId; @@ -167,8 +171,9 @@ private Job(String jobId, String jobType, Version jobVersion, List group Date createTime, Date finishedTime, AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, - Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map customSettings, - String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName, boolean deleting, boolean allowLazyOpen) { + Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, + Map customSettings, String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName, + boolean deleting, boolean allowLazyOpen) { this.jobId = jobId; this.jobType = jobType; @@ -184,6 +189,7 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.renormalizationWindowDays = renormalizationWindowDays; this.backgroundPersistInterval = backgroundPersistInterval; this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; @@ -222,6 +228,11 @@ public Job(StreamInput in) throws IOException { renormalizationWindowDays = in.readOptionalLong(); backgroundPersistInterval = in.readOptionalTimeValue(); modelSnapshotRetentionDays = in.readOptionalLong(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + dailyModelSnapshotRetentionAfterDays = in.readOptionalLong(); + } else { + dailyModelSnapshotRetentionAfterDays = null; + } resultsRetentionDays = in.readOptionalLong(); Map readCustomSettings = in.readMap(); customSettings = readCustomSettings == null ? null : Collections.unmodifiableMap(readCustomSettings); @@ -387,6 +398,10 @@ public Long getModelSnapshotRetentionDays() { return modelSnapshotRetentionDays; } + public Long getDailyModelSnapshotRetentionAfterDays() { + return dailyModelSnapshotRetentionAfterDays; + } + public Long getResultsRetentionDays() { return resultsRetentionDays; } @@ -495,6 +510,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(renormalizationWindowDays); out.writeOptionalTimeValue(backgroundPersistInterval); out.writeOptionalLong(modelSnapshotRetentionDays); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); + } out.writeOptionalLong(resultsRetentionDays); out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); @@ -559,6 +577,9 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (modelSnapshotRetentionDays != null) { builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } @@ -604,6 +625,7 @@ public boolean equals(Object other) { && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) @@ -617,8 +639,8 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, - modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); } // Class already extends from AbstractDiffable, so copied from ToXContentToBytes#toString() @@ -660,6 +682,7 @@ public static class Builder implements Writeable, ToXContentObject { private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS; + private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; private Map customSettings; private String modelSnapshotId; @@ -690,6 +713,7 @@ public Builder(Job job) { this.renormalizationWindowDays = job.getRenormalizationWindowDays(); this.backgroundPersistInterval = job.getBackgroundPersistInterval(); this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); + this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); @@ -728,6 +752,9 @@ public Builder(StreamInput in) throws IOException { renormalizationWindowDays = in.readOptionalLong(); backgroundPersistInterval = in.readOptionalTimeValue(); modelSnapshotRetentionDays = in.readOptionalLong(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + dailyModelSnapshotRetentionAfterDays = in.readOptionalLong(); + } resultsRetentionDays = in.readOptionalLong(); customSettings = in.readMap(); modelSnapshotId = in.readOptionalString(); @@ -831,6 +858,11 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { return this; } + public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) { + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; + return this; + } + public Builder setResultsRetentionDays(Long resultsRetentionDays) { this.resultsRetentionDays = resultsRetentionDays; return this; @@ -927,6 +959,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(renormalizationWindowDays); out.writeOptionalTimeValue(backgroundPersistInterval); out.writeOptionalLong(modelSnapshotRetentionDays); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); + } out.writeOptionalLong(resultsRetentionDays); out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); @@ -985,6 +1020,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelSnapshotRetentionDays != null) { builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } @@ -1031,6 +1069,7 @@ public boolean equals(Object o) { && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) @@ -1044,8 +1083,8 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription, createTime, finishedTime, modelPlotConfig, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, - modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); } /** @@ -1067,6 +1106,8 @@ public void validateInputFields() { checkValidBackgroundPersistInterval(); checkValueNotLessThan(0, RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); checkValueNotLessThan(0, MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); + checkValueNotLessThan(0, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), + dailyModelSnapshotRetentionAfterDays); checkValueNotLessThan(0, RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); if (!MlStrings.isValidId(id)) { @@ -1171,8 +1212,8 @@ public Job build() { return new Job( id, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, - modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen); } private void checkValidBackgroundPersistInterval() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 91c750a46bf7e..e2e384f74650e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -52,6 +52,7 @@ public class JobUpdate implements Writeable, ToXContentObject { parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); + parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); parser.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN); @@ -72,6 +73,7 @@ public class JobUpdate implements Writeable, ToXContentObject { private final Long renormalizationWindowDays; private final TimeValue backgroundPersistInterval; private final Long modelSnapshotRetentionDays; + private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; private final List categorizationFilters; private final Map customSettings; @@ -85,7 +87,8 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long modelSnapshotRetentionDays, @Nullable List categorisationFilters, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, + @Nullable List categorizationFilters, @Nullable Map customSettings, @Nullable String modelSnapshotId, @Nullable Version modelSnapshotMinVersion, @Nullable Version jobVersion, @Nullable Boolean clearJobFinishTime, @Nullable Boolean allowLazyOpen) { @@ -98,8 +101,9 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.renormalizationWindowDays = renormalizationWindowDays; this.backgroundPersistInterval = backgroundPersistInterval; this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.categorizationFilters = categorisationFilters; + this.categorizationFilters = categorizationFilters; this.customSettings = customSettings; this.modelSnapshotId = modelSnapshotId; this.modelSnapshotMinVersion = modelSnapshotMinVersion; @@ -127,6 +131,11 @@ public JobUpdate(StreamInput in) throws IOException { renormalizationWindowDays = in.readOptionalLong(); backgroundPersistInterval = in.readOptionalTimeValue(); modelSnapshotRetentionDays = in.readOptionalLong(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + dailyModelSnapshotRetentionAfterDays = in.readOptionalLong(); + } else { + dailyModelSnapshotRetentionAfterDays = null; + } resultsRetentionDays = in.readOptionalLong(); if (in.readBoolean()) { categorizationFilters = in.readStringList(); @@ -178,6 +187,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(renormalizationWindowDays); out.writeOptionalTimeValue(backgroundPersistInterval); out.writeOptionalLong(modelSnapshotRetentionDays); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); + } out.writeOptionalLong(resultsRetentionDays); out.writeBoolean(categorizationFilters != null); if (categorizationFilters != null) { @@ -249,6 +261,10 @@ public Long getModelSnapshotRetentionDays() { return modelSnapshotRetentionDays; } + public Long getDailyModelSnapshotRetentionAfterDays() { + return dailyModelSnapshotRetentionAfterDays; + } + public Long getResultsRetentionDays() { return resultsRetentionDays; } @@ -313,6 +329,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelSnapshotRetentionDays != null) { builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.field(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } @@ -367,6 +386,9 @@ public Set getUpdateFields() { if (modelSnapshotRetentionDays != null) { updateFields.add(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName()); } + if (dailyModelSnapshotRetentionAfterDays != null) { + updateFields.add(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName()); + } if (resultsRetentionDays != null) { updateFields.add(Job.RESULTS_RETENTION_DAYS.getPreferredName()); } @@ -445,6 +467,9 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { if (modelSnapshotRetentionDays != null) { builder.setModelSnapshotRetentionDays(modelSnapshotRetentionDays); } + if (dailyModelSnapshotRetentionAfterDays != null) { + builder.setDailyModelSnapshotRetentionAfterDays(dailyModelSnapshotRetentionAfterDays); + } if (resultsRetentionDays != null) { builder.setResultsRetentionDays(resultsRetentionDays); } @@ -483,6 +508,8 @@ && updatesDetectors(job) == false && (renormalizationWindowDays == null || Objects.equals(renormalizationWindowDays, job.getRenormalizationWindowDays())) && (backgroundPersistInterval == null || Objects.equals(backgroundPersistInterval, job.getBackgroundPersistInterval())) && (modelSnapshotRetentionDays == null || Objects.equals(modelSnapshotRetentionDays, job.getModelSnapshotRetentionDays())) + && (dailyModelSnapshotRetentionAfterDays == null + || Objects.equals(dailyModelSnapshotRetentionAfterDays, job.getDailyModelSnapshotRetentionAfterDays())) && (resultsRetentionDays == null || Objects.equals(resultsRetentionDays, job.getResultsRetentionDays())) && (categorizationFilters == null || Objects.equals(categorizationFilters, job.getAnalysisConfig().getCategorizationFilters())) @@ -533,6 +560,7 @@ public boolean equals(Object other) { && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.customSettings, that.customSettings) @@ -546,8 +574,9 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, - backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen); + backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, + categorizationFilters, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, + allowLazyOpen); } public static class DetectorUpdate implements Writeable, ToXContentObject { @@ -652,6 +681,7 @@ public static class Builder { private Long renormalizationWindowDays; private TimeValue backgroundPersistInterval; private Long modelSnapshotRetentionDays; + private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; private List categorizationFilters; private Map customSettings; @@ -710,6 +740,11 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { return this; } + public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) { + this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; + return this; + } + public Builder setResultsRetentionDays(Long resultsRetentionDays) { this.resultsRetentionDays = resultsRetentionDays; return this; @@ -762,8 +797,9 @@ public Builder setClearFinishTime(boolean clearJobFinishTime) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen); + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, + categorizationFilters, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, + allowLazyOpen); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 0ca0e1ccd814b..40fd084315d85 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -106,6 +106,7 @@ public void testConstructor_GivenEmptyJobConfiguration() { assertNull(job.getRenormalizationWindowDays()); assertNull(job.getBackgroundPersistInterval()); assertThat(job.getModelSnapshotRetentionDays(), equalTo(1L)); + assertNull(job.getDailyModelSnapshotRetentionAfterDays()); assertNull(job.getResultsRetentionDays()); assertNotNull(job.allInputFields()); assertFalse(job.allInputFields().isEmpty()); @@ -626,6 +627,9 @@ public static Job createRandomizedJob() { if (randomBoolean()) { builder.setModelSnapshotRetentionDays(randomNonNegativeLong()); } + if (randomBoolean()) { + builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong()); + } if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 822edbf77b150..bd337adec0382 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -75,6 +75,9 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (randomBoolean()) { update.setModelSnapshotRetentionDays(randomNonNegativeLong()); } + if (randomBoolean()) { + update.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong()); + } if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java index 78525f6ec2c94..a728a7b1122f4 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java @@ -60,6 +60,9 @@ protected Job.Builder createTestInstance() { if (randomBoolean()) { builder.setModelSnapshotRetentionDays(randomNonNegativeLong()); } + if (randomBoolean()) { + builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong()); + } if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); }