diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java index 0e7b0bae1e688..674e5bf672084 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java @@ -57,7 +57,6 @@ public class Job implements ToXContentObject { public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); - public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name"); public static final ParseField DELETING = new ParseField("deleting"); @@ -88,7 +87,6 @@ public class Job implements ToXContentObject { PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval( TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); - PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); @@ -116,7 +114,6 @@ public class Job implements ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final Map customSettings; private final String modelSnapshotId; private final String resultsIndexName; @@ -130,7 +127,7 @@ private Job(String jobId, String jobType, List groups, String descriptio AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, - Long systemAnnotationsRetentionDays, Map customSettings, String modelSnapshotId, String resultsIndexName, + Map customSettings, String modelSnapshotId, String resultsIndexName, Boolean deleting, Boolean allowLazyOpen, Blocked blocked, DatafeedConfig datafeedConfig) { this.jobId = jobId; @@ -148,7 +145,6 @@ private Job(String jobId, String jobType, List groups, String descriptio this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; this.resultsIndexName = resultsIndexName; @@ -276,10 +272,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public Map getCustomSettings() { return customSettings; } @@ -354,9 +346,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (customSettings != null) { builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); } @@ -411,7 +400,6 @@ public boolean equals(Object other) { && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.resultsIndexName, that.resultsIndexName) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.deleting, that.deleting) && Objects.equals(this.allowLazyOpen, that.allowLazyOpen) && Objects.equals(this.blocked, that.blocked) @@ -423,7 +411,7 @@ public int hashCode() { return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -453,7 +441,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private Map customSettings; private String modelSnapshotId; private String resultsIndexName; @@ -485,7 +472,6 @@ public Builder(Job job) { this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); - this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); this.resultsIndexName = job.getResultsIndexNameNoPrefix(); @@ -579,11 +565,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setModelSnapshotId(String modelSnapshotId) { this.modelSnapshotId = modelSnapshotId; return this; @@ -626,7 +607,7 @@ public Job build() { id, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig == null ? null : datafeedConfig.build()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java index 1f90be3950855..0d3152d72885f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java @@ -40,7 +40,6 @@ public class JobUpdate implements ToXContentObject { TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); @@ -63,7 +62,6 @@ public class JobUpdate implements ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final List categorizationFilters; private final PerPartitionCategorizationConfig perPartitionCategorizationConfig; private final Map customSettings; @@ -74,7 +72,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List categorizationFilters, @Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig, @Nullable Map customSettings, @Nullable Boolean allowLazyOpen, @Nullable TimeValue modelPruneWindow) { @@ -89,7 +87,6 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.categorizationFilters = categorizationFilters; this.perPartitionCategorizationConfig = perPartitionCategorizationConfig; this.customSettings = customSettings; @@ -137,10 +134,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public List getCategorizationFilters() { return categorizationFilters; } @@ -195,9 +188,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); } @@ -240,7 +230,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig) && Objects.equals(this.customSettings, that.customSettings) @@ -252,7 +241,7 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, + categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, modelPruneWindow); } @@ -348,7 +337,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private List categorizationFilters; private PerPartitionCategorizationConfig perPartitionCategorizationConfig; private Map customSettings; @@ -484,18 +472,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - /** - * Advanced configuration option. The number of days for which job annotations are retained - * - * Updates the {@link Job#systemAnnotationsRetentionDays} setting - * - * @param systemAnnotationsRetentionDays number of days to keep results. - */ - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - /** * Sets the categorization filters on the {@link Job} * @@ -546,7 +522,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays, + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, modelPruneWindow); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java index fd8fe0e656976..7e88c51092be6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java @@ -147,9 +147,6 @@ public static Job.Builder createRandomizedJobBuilder() { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java index 5049317e10162..8731e83d7800c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java @@ -65,9 +65,6 @@ public static JobUpdate createRandom(String jobId) { if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - update.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); } diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index 72c88ec937622..401656da05e53 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -379,10 +379,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-index-name] (Optional, long) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] -`system_annotations_retention_days`:: -(Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days] - [[ml-put-job-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index d3c24210eef3d..26b5ccf0cf434 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -235,10 +235,6 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta (long) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] -`system_annotations_retention_days`:: -(long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days] - [[ml-update-job-example]] == {api-examples-title} @@ -259,8 +255,7 @@ POST _ml/anomaly_detectors/low_request_rate/_update "renormalization_window_days": 30, "background_persist_interval": "2h", "model_snapshot_retention_days": 7, - "results_retention_days": 60, - "system_annotations_retention_days": 60 + "results_retention_days": 60 } -------------------------------------------------- // TEST[skip:setup:Kibana sample data] diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index d1f0e68f60d94..ac281304b9c61 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -1430,19 +1430,11 @@ retained. Age is calculated relative to the timestamp of the latest bucket result. If this property has a non-null value, once per day at 00:30 (server time), results that are the specified number of days older than the latest bucket result are deleted from {es}. The default value is null, which means all -results are retained. +results are retained. Annotations generated by the system also count as results +for retention purposes; they are deleted after the same number of days as +results. Annotations added by users are retained forever. end::results-retention-days[] -tag::system-annotations-retention-days[] -Advanced configuration option. The period of time (in days) that automatically -created annotations are retained. Age is calculated relative to the timestamp of -the latest bucket result. If this property has a non-null value, once per day at -00:30 (server time), annotations that are the specified number of days older -than the latest bucket result are deleted from {es}. The default value is null, -which means all annotations are retained. -User created annotations are never deleted automatically. -end::system-annotations-retention-days[] - tag::retain[] If `true`, this snapshot will not be deleted during automatic cleanup of snapshots older than `model_snapshot_retention_days`. However, this snapshot diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index b9291103b115e..1af0630af86b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -83,7 +83,6 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); - public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField MODEL_SNAPSHOT_MIN_VERSION = new ParseField("model_snapshot_min_version"); public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name"); @@ -136,7 +135,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie parser.declareString((builder, val) -> builder.setBackgroundPersistInterval( TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); - parser.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); @@ -176,7 +174,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final Map customSettings; private final String modelSnapshotId; private final Version modelSnapshotMinVersion; @@ -191,7 +188,7 @@ private Job(String jobId, String jobType, Version jobVersion, List group AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, - Long systemAnnotationsRetentionDays, Map customSettings, String modelSnapshotId, + Map customSettings, String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName, boolean deleting, boolean allowLazyOpen, Blocked blocked, DatafeedConfig datafeedConfig) { this.jobId = jobId; @@ -210,7 +207,6 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; this.modelSnapshotMinVersion = modelSnapshotMinVersion; @@ -266,11 +262,6 @@ public Job(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = null; } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } else { - systemAnnotationsRetentionDays = null; - } Map readCustomSettings = in.readMap(); customSettings = readCustomSettings == null ? null : Collections.unmodifiableMap(readCustomSettings); modelSnapshotId = in.readOptionalString(); @@ -461,10 +452,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public Map getCustomSettings() { return customSettings; } @@ -581,9 +568,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); - } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); if (out.getVersion().onOrAfter(Version.V_7_0_0)) { @@ -684,9 +668,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); if (blocked.getReason() != Blocked.Reason.NONE) { @@ -722,7 +703,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) @@ -738,7 +718,7 @@ public int hashCode() { return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -783,7 +763,6 @@ public static class Builder implements Writeable { private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private Map customSettings; private String modelSnapshotId; private Version modelSnapshotMinVersion; @@ -817,7 +796,6 @@ public Builder(Job job) { this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); - this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); this.modelSnapshotMinVersion = job.getModelSnapshotMinVersion(); @@ -861,9 +839,6 @@ public Builder(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = in.readOptionalLong(); } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } customSettings = in.readMap(); modelSnapshotId = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_7_0_0) && in.readBoolean()) { @@ -988,11 +963,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setModelSnapshotId(String modelSnapshotId) { this.modelSnapshotId = modelSnapshotId; return this; @@ -1115,9 +1085,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); - } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); if (out.getVersion().onOrAfter(Version.V_7_0_0)) { @@ -1162,7 +1129,6 @@ public boolean equals(Object o) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) @@ -1178,7 +1144,7 @@ public int hashCode() { return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription, createTime, finishedTime, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -1201,7 +1167,6 @@ public void validateInputFields() { checkValidBackgroundPersistInterval(); checkValueNotLessThan(0, RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); checkValueNotLessThan(0, RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); - checkValueNotLessThan(0, SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); if (MlStrings.isValidId(id) == false) { throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, ID.getPreferredName(), id)); @@ -1350,7 +1315,7 @@ public Job build() { id, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, Optional.ofNullable(datafeedConfig).map(DatafeedConfig.Builder::build).orElse(null)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 8b3d2f172f3e4..5d4e7b85edd75 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -53,7 +53,6 @@ public class JobUpdate implements Writeable, ToXContentObject { TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - parser.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); @@ -83,7 +82,6 @@ public class JobUpdate implements Writeable, ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final List categorizationFilters; private final PerPartitionCategorizationConfig perPartitionCategorizationConfig; private final Map customSettings; @@ -99,7 +97,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List categorizationFilters, @Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig, @Nullable Map customSettings, @Nullable String modelSnapshotId, @@ -116,7 +114,6 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.categorizationFilters = categorizationFilters; this.perPartitionCategorizationConfig = perPartitionCategorizationConfig; this.customSettings = customSettings; @@ -154,11 +151,6 @@ public JobUpdate(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = null; } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } else { - systemAnnotationsRetentionDays = null; - } if (in.readBoolean()) { categorizationFilters = in.readStringList(); } else { @@ -226,9 +218,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); - } out.writeBoolean(categorizationFilters != null); if (categorizationFilters != null) { out.writeStringCollection(categorizationFilters); @@ -316,10 +305,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public List getCategorizationFilters() { return categorizationFilters; } @@ -398,9 +383,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); } @@ -467,9 +449,6 @@ public Set getUpdateFields() { if (resultsRetentionDays != null) { updateFields.add(Job.RESULTS_RETENTION_DAYS.getPreferredName()); } - if (systemAnnotationsRetentionDays != null) { - updateFields.add(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName()); - } if (categorizationFilters != null) { updateFields.add(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName()); } @@ -557,9 +536,6 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { if (resultsRetentionDays != null) { builder.setResultsRetentionDays(resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.setSystemAnnotationsRetentionDays(systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { newAnalysisConfig.setCategorizationFilters(categorizationFilters); } @@ -612,8 +588,6 @@ && updatesDetectors(job) == false && (dailyModelSnapshotRetentionAfterDays == null || Objects.equals(dailyModelSnapshotRetentionAfterDays, job.getDailyModelSnapshotRetentionAfterDays())) && (resultsRetentionDays == null || Objects.equals(resultsRetentionDays, job.getResultsRetentionDays())) - && (systemAnnotationsRetentionDays == null - || Objects.equals(systemAnnotationsRetentionDays, job.getSystemAnnotationsRetentionDays())) && (categorizationFilters == null || Objects.equals(categorizationFilters, job.getAnalysisConfig().getCategorizationFilters())) && (perPartitionCategorizationConfig == null @@ -669,7 +643,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig) && Objects.equals(this.customSettings, that.customSettings) @@ -686,7 +659,7 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, + categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen, blocked, modelPruneWindow); } @@ -794,7 +767,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private List categorizationFilters; private PerPartitionCategorizationConfig perPartitionCategorizationConfig; private Map customSettings; @@ -865,11 +837,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setCategorizationFilters(List categorizationFilters) { this.categorizationFilters = categorizationFilters; return this; @@ -932,7 +899,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays, + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen, blocked, modelPruneWindow); } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json index 5b46c86d44202..72c468ce65286 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json @@ -291,9 +291,6 @@ "type" : "object", "enabled" : false }, - "system_annotations_retention_days" : { - "type" : "long" - }, "background_persist_interval" : { "type" : "keyword" }, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index f22191d380d40..8b95159e78ff7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -142,7 +142,6 @@ public void testConstructor_GivenEmptyJobConfiguration() { assertThat(job.getModelSnapshotRetentionDays(), equalTo(10L)); assertNull(job.getDailyModelSnapshotRetentionAfterDays()); assertNull(job.getResultsRetentionDays()); - assertNull(job.getSystemAnnotationsRetentionDays()); assertNotNull(job.allInputFields()); assertFalse(job.allInputFields().isEmpty()); assertFalse(job.allowLazyOpen()); @@ -266,21 +265,6 @@ public void testEquals_GivenDifferentResultsRetentionDays() { assertNotEquals(jobDetails1.build(), jobDetails2.build()); } - public void testEquals_GivenDifferentSystemAnnotationsRetentionDays() { - Date date = new Date(); - Job.Builder jobDetails1 = new Job.Builder("foo"); - jobDetails1.setDataDescription(new DataDescription.Builder()); - jobDetails1.setAnalysisConfig(createAnalysisConfig()); - jobDetails1.setCreateTime(date); - jobDetails1.setSystemAnnotationsRetentionDays(30L); - Job.Builder jobDetails2 = new Job.Builder("foo"); - jobDetails2.setDataDescription(new DataDescription.Builder()); - jobDetails2.setSystemAnnotationsRetentionDays(4L); - jobDetails2.setAnalysisConfig(createAnalysisConfig()); - jobDetails2.setCreateTime(date); - assertNotEquals(jobDetails1.build(), jobDetails2.build()); - } - public void testEquals_GivenDifferentCustomSettings() { Job.Builder jobDetails1 = buildJobBuilder("foo"); Map customSettings1 = new HashMap<>(); @@ -487,15 +471,6 @@ public void testVerify_GivenNegativeResultsRetentionDays() { assertEquals(errorMessage, e.getMessage()); } - public void testVerify_GivenNegativeSystemAnnotationsRetentionDays() { - String errorMessage = - Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "system_annotations_retention_days", 0, -1); - Job.Builder builder = buildJobBuilder("foo"); - builder.setSystemAnnotationsRetentionDays(-1L); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertEquals(errorMessage, e.getMessage()); - } - public void testBuilder_setsDefaultIndexName() { Job.Builder builder = buildJobBuilder("foo"); Job job = builder.build(); @@ -768,9 +743,6 @@ public static Job createRandomizedJob(DatafeedConfig.Builder datafeedBuilder) { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index e63f0d5c6f347..2671814535ed3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -105,9 +105,6 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - update.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean() && jobSupportsCategorizationFilters(job)) { update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); } @@ -258,7 +255,6 @@ public void testMergeWithJob() { updateBuilder.setAnalysisLimits(analysisLimits); updateBuilder.setBackgroundPersistInterval(TimeValue.timeValueHours(randomIntBetween(1, 24))); updateBuilder.setResultsRetentionDays(randomNonNegativeLong()); - updateBuilder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); // The createRandom() method tests the complex interactions between these next two, so this test can always update both long newModelSnapshotRetentionDays = randomNonNegativeLong(); updateBuilder.setModelSnapshotRetentionDays(newModelSnapshotRetentionDays); @@ -297,7 +293,6 @@ public void testMergeWithJob() { assertEquals(update.getBackgroundPersistInterval(), updatedJob.getBackgroundPersistInterval()); assertEquals(update.getModelSnapshotRetentionDays(), updatedJob.getModelSnapshotRetentionDays()); assertEquals(update.getResultsRetentionDays(), updatedJob.getResultsRetentionDays()); - assertEquals(update.getSystemAnnotationsRetentionDays(), updatedJob.getSystemAnnotationsRetentionDays()); assertEquals(update.getCategorizationFilters(), updatedJob.getAnalysisConfig().getCategorizationFilters()); assertEquals(update.getPerPartitionCategorizationConfig().isEnabled(), updatedJob.getAnalysisConfig().getPerPartitionCategorizationConfig().isEnabled()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index aaa48acfba83b..0abb198bc40e2 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -361,8 +361,7 @@ public void testDeleteExpiresDataDeletesAnnotations() throws Exception { Job.Builder job = new Job.Builder(jobId) - .setResultsRetentionDays(2L) - .setSystemAnnotationsRetentionDays(1L) + .setResultsRetentionDays(1L) .setAnalysisConfig( new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder().setFunction("count").build())) .setBucketSpan(TimeValue.timeValueHours(1))) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java index f5e352e3f7a94..1706206f3a9d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java @@ -61,9 +61,12 @@ public ExpiredAnnotationsRemover(OriginSettingClient client, Iterator jobIt this.threadPool = Objects.requireNonNull(threadPool); } + /** + * Annotations are retained for the same length of time as results. + */ @Override Long getRetentionDays(Job job) { - return job.getSystemAnnotationsRetentionDays(); + return job.getResultsRetentionDays(); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java index d307d7affd937..d695fd0b994bc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java @@ -65,9 +65,6 @@ protected Job.Builder createTestInstance() { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java index aa8e166cdc16d..c5d311a8bfb8c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java @@ -88,8 +88,8 @@ public void testRemove_GivenJobsWithAndWithoutRetentionPolicy() { List jobs = Arrays.asList( JobTests.buildJobBuilder("none").build(), - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build()); + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build()); createExpiredAnnotationsRemover(jobs.iterator()).remove(1.0f, listener, () -> false); @@ -106,8 +106,8 @@ public void testRemove_GivenTimeout() { givenBucket(new Bucket("id_not_important", new Date(), 60)); List jobs = Arrays.asList( - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build() + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build() ); final int timeoutAfter = randomIntBetween(0, 1); @@ -125,8 +125,8 @@ public void testRemove_GivenClientRequestsFailed() { List jobs = Arrays.asList( JobTests.buildJobBuilder("none").build(), - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build()); + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build()); createExpiredAnnotationsRemover(jobs.iterator()).remove(1.0f, listener, () -> false); assertThat(capturedDeleteByQueryRequests.size(), equalTo(1)); @@ -141,7 +141,7 @@ public void testCalcCutoffEpochMs() { Date latest = new Date(); givenBucket(new Bucket(jobId, latest, 60)); - List jobs = Collections.singletonList(JobTests.buildJobBuilder(jobId).setSystemAnnotationsRetentionDays(1L).build()); + List jobs = Collections.singletonList(JobTests.buildJobBuilder(jobId).setResultsRetentionDays(1L).build()); ActionListener cutoffListener = mock(ActionListener.class); createExpiredAnnotationsRemover(jobs.iterator()).calcCutoffEpochMs(jobId, 1L, cutoffListener); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml index 8d33d24756f51..eee082acd47f2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml @@ -18,7 +18,6 @@ setup: "time_format":"yyyy-MM-dd HH:mm:ssX" }, "results_retention_days" : 1, - "system_annotations_retention_days" : 1, "model_snapshot_retention_days" : 1 } @@ -39,7 +38,6 @@ setup: "time_format":"yyyy-MM-dd HH:mm:ssX" }, "results_retention_days" : 1, - "system_annotations_retention_days" : 1, "model_snapshot_retention_days" : 1 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml index 9658d621c6b93..605c779051e93 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -379,7 +379,6 @@ "model_snapshot_retention_days": 3, "daily_model_snapshot_retention_after_days": 2, "results_retention_days": 4, - "system_annotations_retention_days": 5, "custom_settings": { "setting1": "custom1", "setting2": "custom2" @@ -435,7 +434,6 @@ "background_persist_interval": "3h", "model_snapshot_retention_days": 30, "results_retention_days": 40, - "system_annotations_retention_days": 50, "custom_settings": { "setting3": "custom3" } @@ -458,7 +456,6 @@ - match: { results_retention_days: 40 } - match: { analysis_config.bucket_span: "5m" } - match: { analysis_config.model_prune_window: "60d" } - - match: { system_annotations_retention_days: 50 } - do: catch: "/Cannot update analysis_limits while the job is open/"