Skip to content

Commit

Permalink
[ML] Add daily_model_snapshot_retention_after_days to job config (#55891
Browse files Browse the repository at this point in the history 
)

This change adds a new setting, daily_model_snapshot_retention_after_days,
to the anomaly detection job config.

Initially this has no effect, the effect will be added in a followup PR.
This PR gets the complexities of making changes that interact with BWC
over well before feature freeze.

Backport of #55878
  • Loading branch information
@droberts195
droberts195 authored Apr 29, 2020
1 parent e982cf4 commit 61ac09a
Show file tree
Hide file tree
Showing 9 changed files with 161 additions and 26 deletions.
33 changes: 27 additions & 6 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ public class Job implements ToXContentObject {
public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS =
new ParseField("daily_model_snapshot_retention_after_days");
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
Expand Down Expand Up @@ -93,6 +95,7 @@ public class Job implements ToXContentObject {
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT);
PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
Expand All @@ -114,6 +117,7 @@ public class Job implements ToXContentObject {
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final Map<String, Object> customSettings;
private final String modelSnapshotId;
Expand All @@ -125,8 +129,9 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
Date createTime, Date finishedTime,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, String resultsIndexName, Boolean deleting, Boolean allowLazyOpen) {
Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays,
Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName, Boolean deleting,
Boolean allowLazyOpen) {

this.jobId = jobId;
this.jobType = jobType;
Expand All @@ -141,6 +146,7 @@ private Job(String jobId, String jobType, List<String> groups, String descriptio
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
this.modelSnapshotId = modelSnapshotId;
Expand Down Expand Up @@ -259,6 +265,10 @@ public Long getModelSnapshotRetentionDays() {
return modelSnapshotRetentionDays;
}

public Long getDailyModelSnapshotRetentionAfterDays() {
return dailyModelSnapshotRetentionAfterDays;
}

public Long getResultsRetentionDays() {
return resultsRetentionDays;
}
Expand Down Expand Up @@ -319,6 +329,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (modelSnapshotRetentionDays != null) {
builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
Expand Down Expand Up @@ -365,6 +378,7 @@ public boolean equals(Object other) {
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.customSettings, that.customSettings)
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
Expand All @@ -377,8 +391,8 @@ public boolean equals(Object other) {
public int hashCode() {
return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
}

@Override
Expand All @@ -405,6 +419,7 @@ public static class Builder {
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private Map<String, Object> customSettings;
private String modelSnapshotId;
Expand Down Expand Up @@ -433,6 +448,7 @@ public Builder(Job job) {
this.renormalizationWindowDays = job.getRenormalizationWindowDays();
this.backgroundPersistInterval = job.getBackgroundPersistInterval();
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays();
this.resultsRetentionDays = job.getResultsRetentionDays();
this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings());
this.modelSnapshotId = job.getModelSnapshotId();
Expand Down Expand Up @@ -515,6 +531,11 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) {
return this;
}

public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}

public Builder setResultsRetentionDays(Long resultsRetentionDays) {
this.resultsRetentionDays = resultsRetentionDays;
return this;
Expand Down Expand Up @@ -551,8 +572,8 @@ public Job build() {
return new Job(
id, jobType, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
}
}
}
33 changes: 27 additions & 6 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ public class JobUpdate implements ToXContentObject {
PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT);
PARSER.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN);
Expand All @@ -66,6 +67,7 @@ public class JobUpdate implements ToXContentObject {
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final List<String> categorizationFilters;
private final Map<String, Object> customSettings;
Expand All @@ -75,7 +77,8 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
@Nullable Long modelSnapshotRetentionDays, @Nullable List<String> categorisationFilters,
@Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays,
@Nullable List<String> categorizationFilters,
@Nullable Map<String, Object> customSettings, @Nullable Boolean allowLazyOpen) {
this.jobId = jobId;
this.groups = groups;
Expand All @@ -86,8 +89,9 @@ private JobUpdate(String jobId, @Nullable List<String> groups, @Nullable String
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.categorizationFilters = categorisationFilters;
this.categorizationFilters = categorizationFilters;
this.customSettings = customSettings;
this.allowLazyOpen = allowLazyOpen;
}
Expand Down Expand Up @@ -172,6 +176,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (modelSnapshotRetentionDays != null) {
builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
Expand Down Expand Up @@ -209,6 +216,7 @@ public boolean equals(Object other) {
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.categorizationFilters, that.categorizationFilters)
&& Objects.equals(this.customSettings, that.customSettings)
Expand All @@ -218,8 +226,8 @@ public boolean equals(Object other) {
@Override
public int hashCode() {
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings,
allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
categorizationFilters, customSettings, allowLazyOpen);
}

public static class DetectorUpdate implements ToXContentObject {
Expand Down Expand Up @@ -312,6 +320,7 @@ public static class Builder {
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private List<String> categorizationFilters;
private Map<String, Object> customSettings;
Expand Down Expand Up @@ -422,6 +431,18 @@ public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) {
return this;
}

/**
* The time in days after which only one model snapshot per day is retained for the job.
*
* Updates the {@link Job#dailyModelSnapshotRetentionAfterDays} setting
*
* @param dailyModelSnapshotRetentionAfterDays number of days to keep a model snapshot
*/
public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}

/**
* Advanced configuration option. The number of days for which job results are retained
*
Expand Down Expand Up @@ -466,8 +487,8 @@ public Builder setAllowLazyOpen(boolean allowLazyOpen) {

public JobUpdate build() {
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings,
allowLazyOpen);
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays,
categorizationFilters, customSettings, allowLazyOpen);
}
}
}
3 changes: 3 additions & 0 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,9 @@ public static Job.Builder createRandomizedJobBuilder() {
if (randomBoolean()) {
builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setResultsRetentionDays(randomNonNegativeLong());
}
Expand Down
3 changes: 3 additions & 0 deletions .../rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ public static JobUpdate createRandom(String jobId) {
if (randomBoolean()) {
update.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setResultsRetentionDays(randomNonNegativeLong());
}
Expand Down
Loading

0 comments on commit 61ac09a

Please sign in to comment.