From 484ce9f882ff24bbc928559d64a13349aac9632a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 4 Aug 2021 17:42:31 +0100 Subject: [PATCH 1/2] [ML] Use results retention time for deleting system annotations In #75617 a new setting, system_annotations_retention_days, was added to control how long system annotations are retained for. We now feel that this setting is redundant and that system annotations should be retained for the same period as results. This is intuitive and defensible, as system annotations can be considered a type of result. Backport of #76096 --- .../client/ml/job/config/Job.java | 25 ++--------- .../client/ml/job/config/JobUpdate.java | 30 ++----------- .../client/ml/job/config/JobTests.java | 3 -- .../client/ml/job/config/JobUpdateTests.java | 3 -- .../anomaly-detection/apis/put-job.asciidoc | 4 -- .../apis/update-job.asciidoc | 7 +-- docs/reference/ml/ml-shared.asciidoc | 14 ++---- .../xpack/core/ml/job/config/Job.java | 45 ++++--------------- .../xpack/core/ml/job/config/JobUpdate.java | 39 ++-------------- .../xpack/core/ml/config_index_mappings.json | 3 -- .../xpack/core/ml/job/config/JobTests.java | 28 ------------ .../core/ml/job/config/JobUpdateTests.java | 5 --- .../ml/integration/DeleteExpiredDataIT.java | 3 +- .../retention/ExpiredAnnotationsRemover.java | 5 ++- .../xpack/ml/job/config/JobBuilderTests.java | 3 -- .../ExpiredAnnotationsRemoverTests.java | 14 +++--- .../test/ml/delete_expired_data.yml | 2 - .../rest-api-spec/test/ml/jobs_crud.yml | 3 -- 18 files changed, 34 insertions(+), 202 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java index 0e7b0bae1e688..674e5bf672084 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java @@ -57,7 +57,6 @@ public class Job implements ToXContentObject { public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); - public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name"); public static final ParseField DELETING = new ParseField("deleting"); @@ -88,7 +87,6 @@ public class Job implements ToXContentObject { PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval( TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); - PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); @@ -116,7 +114,6 @@ public class Job implements ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final Map customSettings; private final String modelSnapshotId; private final String resultsIndexName; @@ -130,7 +127,7 @@ private Job(String jobId, String jobType, List groups, String descriptio AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, - Long systemAnnotationsRetentionDays, Map customSettings, String modelSnapshotId, String resultsIndexName, + Map customSettings, String modelSnapshotId, String resultsIndexName, Boolean deleting, Boolean allowLazyOpen, Blocked blocked, DatafeedConfig datafeedConfig) { this.jobId = jobId; @@ -148,7 +145,6 @@ private Job(String jobId, String jobType, List groups, String descriptio this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; this.resultsIndexName = resultsIndexName; @@ -276,10 +272,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public Map getCustomSettings() { return customSettings; } @@ -354,9 +346,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (customSettings != null) { builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); } @@ -411,7 +400,6 @@ public boolean equals(Object other) { && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.resultsIndexName, that.resultsIndexName) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.deleting, that.deleting) && Objects.equals(this.allowLazyOpen, that.allowLazyOpen) && Objects.equals(this.blocked, that.blocked) @@ -423,7 +411,7 @@ public int hashCode() { return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -453,7 +441,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private Map customSettings; private String modelSnapshotId; private String resultsIndexName; @@ -485,7 +472,6 @@ public Builder(Job job) { this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); - this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); this.resultsIndexName = job.getResultsIndexNameNoPrefix(); @@ -579,11 +565,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setModelSnapshotId(String modelSnapshotId) { this.modelSnapshotId = modelSnapshotId; return this; @@ -626,7 +607,7 @@ public Job build() { id, jobType, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, + customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig == null ? null : datafeedConfig.build()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java index 1f90be3950855..0d3152d72885f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java @@ -40,7 +40,6 @@ public class JobUpdate implements ToXContentObject { TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - PARSER.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS); PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); @@ -63,7 +62,6 @@ public class JobUpdate implements ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final List categorizationFilters; private final PerPartitionCategorizationConfig perPartitionCategorizationConfig; private final Map customSettings; @@ -74,7 +72,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List categorizationFilters, @Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig, @Nullable Map customSettings, @Nullable Boolean allowLazyOpen, @Nullable TimeValue modelPruneWindow) { @@ -89,7 +87,6 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.categorizationFilters = categorizationFilters; this.perPartitionCategorizationConfig = perPartitionCategorizationConfig; this.customSettings = customSettings; @@ -137,10 +134,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public List getCategorizationFilters() { return categorizationFilters; } @@ -195,9 +188,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); } @@ -240,7 +230,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig) && Objects.equals(this.customSettings, that.customSettings) @@ -252,7 +241,7 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, + categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, modelPruneWindow); } @@ -348,7 +337,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private List categorizationFilters; private PerPartitionCategorizationConfig perPartitionCategorizationConfig; private Map customSettings; @@ -484,18 +472,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - /** - * Advanced configuration option. The number of days for which job annotations are retained - * - * Updates the {@link Job#systemAnnotationsRetentionDays} setting - * - * @param systemAnnotationsRetentionDays number of days to keep results. - */ - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - /** * Sets the categorization filters on the {@link Job} * @@ -546,7 +522,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays, + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, allowLazyOpen, modelPruneWindow); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java index fd8fe0e656976..7e88c51092be6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java @@ -147,9 +147,6 @@ public static Job.Builder createRandomizedJobBuilder() { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java index 5049317e10162..8731e83d7800c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java @@ -65,9 +65,6 @@ public static JobUpdate createRandom(String jobId) { if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - update.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); } diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index 72c88ec937622..401656da05e53 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -379,10 +379,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-index-name] (Optional, long) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] -`system_annotations_retention_days`:: -(Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days] - [[ml-put-job-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index d3c24210eef3d..26b5ccf0cf434 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -235,10 +235,6 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta (long) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] -`system_annotations_retention_days`:: -(long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=system-annotations-retention-days] - [[ml-update-job-example]] == {api-examples-title} @@ -259,8 +255,7 @@ POST _ml/anomaly_detectors/low_request_rate/_update "renormalization_window_days": 30, "background_persist_interval": "2h", "model_snapshot_retention_days": 7, - "results_retention_days": 60, - "system_annotations_retention_days": 60 + "results_retention_days": 60 } -------------------------------------------------- // TEST[skip:setup:Kibana sample data] diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index d1f0e68f60d94..ac281304b9c61 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -1430,19 +1430,11 @@ retained. Age is calculated relative to the timestamp of the latest bucket result. If this property has a non-null value, once per day at 00:30 (server time), results that are the specified number of days older than the latest bucket result are deleted from {es}. The default value is null, which means all -results are retained. +results are retained. Annotations generated by the system also count as results +for retention purposes; they are deleted after the same number of days as +results. Annotations added by users are retained forever. end::results-retention-days[] -tag::system-annotations-retention-days[] -Advanced configuration option. The period of time (in days) that automatically -created annotations are retained. Age is calculated relative to the timestamp of -the latest bucket result. If this property has a non-null value, once per day at -00:30 (server time), annotations that are the specified number of days older -than the latest bucket result are deleted from {es}. The default value is null, -which means all annotations are retained. -User created annotations are never deleted automatically. -end::system-annotations-retention-days[] - tag::retain[] If `true`, this snapshot will not be deleted during automatic cleanup of snapshots older than `model_snapshot_retention_days`. However, this snapshot diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index b9291103b115e..a02225aad68a6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -83,7 +83,6 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS = new ParseField("daily_model_snapshot_retention_after_days"); public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); - public static final ParseField SYSTEM_ANNOTATIONS_RETENTION_DAYS = new ParseField("system_annotations_retention_days"); public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); public static final ParseField MODEL_SNAPSHOT_MIN_VERSION = new ParseField("model_snapshot_min_version"); public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name"); @@ -136,7 +135,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie parser.declareString((builder, val) -> builder.setBackgroundPersistInterval( TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL); parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); - parser.declareLong(Builder::setSystemAnnotationsRetentionDays, SYSTEM_ANNOTATIONS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT); @@ -176,7 +174,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final Map customSettings; private final String modelSnapshotId; private final Version modelSnapshotMinVersion; @@ -191,7 +188,7 @@ private Job(String jobId, String jobType, Version jobVersion, List group AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval, Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays, - Long systemAnnotationsRetentionDays, Map customSettings, String modelSnapshotId, + Map customSettings, String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName, boolean deleting, boolean allowLazyOpen, Blocked blocked, DatafeedConfig datafeedConfig) { this.jobId = jobId; @@ -210,7 +207,6 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; this.modelSnapshotMinVersion = modelSnapshotMinVersion; @@ -266,11 +262,6 @@ public Job(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = null; } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } else { - systemAnnotationsRetentionDays = null; - } Map readCustomSettings = in.readMap(); customSettings = readCustomSettings == null ? null : Collections.unmodifiableMap(readCustomSettings); modelSnapshotId = in.readOptionalString(); @@ -461,10 +452,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public Map getCustomSettings() { return customSettings; } @@ -581,9 +568,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); - } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); if (out.getVersion().onOrAfter(Version.V_7_0_0)) { @@ -684,9 +668,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); if (blocked.getReason() != Blocked.Reason.NONE) { @@ -722,7 +703,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) @@ -738,7 +718,7 @@ public int hashCode() { return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -783,7 +763,6 @@ public static class Builder implements Writeable { private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private Map customSettings; private String modelSnapshotId; private Version modelSnapshotMinVersion; @@ -817,7 +796,6 @@ public Builder(Job job) { this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays(); this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays(); this.resultsRetentionDays = job.getResultsRetentionDays(); - this.systemAnnotationsRetentionDays = job.getSystemAnnotationsRetentionDays(); this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings()); this.modelSnapshotId = job.getModelSnapshotId(); this.modelSnapshotMinVersion = job.getModelSnapshotMinVersion(); @@ -861,9 +839,6 @@ public Builder(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = in.readOptionalLong(); } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } customSettings = in.readMap(); modelSnapshotId = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_7_0_0) && in.readBoolean()) { @@ -988,11 +963,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setModelSnapshotId(String modelSnapshotId) { this.modelSnapshotId = modelSnapshotId; return this; @@ -1115,8 +1085,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); +<<<<<<< HEAD if (out.getVersion().onOrAfter(Version.V_7_15_0)) { out.writeOptionalLong(systemAnnotationsRetentionDays); +======= + if (out.getVersion().onOrAfter(Version.V_7_15_0) && out.getVersion().before(Version.V_8_0_0)) { + out.writeOptionalLong(resultsRetentionDays); +>>>>>>> 7ac5ea3... [ML] Use results retention time for deleting system annotations (#76096) } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); @@ -1162,7 +1137,6 @@ public boolean equals(Object o) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) @@ -1178,7 +1152,7 @@ public int hashCode() { return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription, createTime, finishedTime, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, datafeedConfig); } @@ -1201,7 +1175,6 @@ public void validateInputFields() { checkValidBackgroundPersistInterval(); checkValueNotLessThan(0, RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); checkValueNotLessThan(0, RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); - checkValueNotLessThan(0, SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); if (MlStrings.isValidId(id) == false) { throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, ID.getPreferredName(), id)); @@ -1350,7 +1323,7 @@ public Job build() { id, jobType, jobVersion, groups, description, createTime, finishedTime, analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, + customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen, blocked, Optional.ofNullable(datafeedConfig).map(DatafeedConfig.Builder::build).orElse(null)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 8b3d2f172f3e4..5d4e7b85edd75 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -53,7 +53,6 @@ public class JobUpdate implements Writeable, ToXContentObject { TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - parser.declareLong(Builder::setSystemAnnotationsRetentionDays, Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS); parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); @@ -83,7 +82,6 @@ public class JobUpdate implements Writeable, ToXContentObject { private final Long modelSnapshotRetentionDays; private final Long dailyModelSnapshotRetentionAfterDays; private final Long resultsRetentionDays; - private final Long systemAnnotationsRetentionDays; private final List categorizationFilters; private final PerPartitionCategorizationConfig perPartitionCategorizationConfig; private final Map customSettings; @@ -99,7 +97,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval, @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, - @Nullable Long systemAnnotationsRetentionDays, @Nullable Long modelSnapshotRetentionDays, + @Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays, @Nullable List categorizationFilters, @Nullable PerPartitionCategorizationConfig perPartitionCategorizationConfig, @Nullable Map customSettings, @Nullable String modelSnapshotId, @@ -116,7 +114,6 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays; this.resultsRetentionDays = resultsRetentionDays; - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; this.categorizationFilters = categorizationFilters; this.perPartitionCategorizationConfig = perPartitionCategorizationConfig; this.customSettings = customSettings; @@ -154,11 +151,6 @@ public JobUpdate(StreamInput in) throws IOException { dailyModelSnapshotRetentionAfterDays = null; } resultsRetentionDays = in.readOptionalLong(); - if (in.getVersion().onOrAfter(Version.V_7_15_0)) { - systemAnnotationsRetentionDays = in.readOptionalLong(); - } else { - systemAnnotationsRetentionDays = null; - } if (in.readBoolean()) { categorizationFilters = in.readStringList(); } else { @@ -226,9 +218,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); - } out.writeBoolean(categorizationFilters != null); if (categorizationFilters != null) { out.writeStringCollection(categorizationFilters); @@ -316,10 +305,6 @@ public Long getResultsRetentionDays() { return resultsRetentionDays; } - public Long getSystemAnnotationsRetentionDays() { - return systemAnnotationsRetentionDays; - } - public List getCategorizationFilters() { return categorizationFilters; } @@ -398,9 +383,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (resultsRetentionDays != null) { builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.field(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName(), systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); } @@ -467,9 +449,6 @@ public Set getUpdateFields() { if (resultsRetentionDays != null) { updateFields.add(Job.RESULTS_RETENTION_DAYS.getPreferredName()); } - if (systemAnnotationsRetentionDays != null) { - updateFields.add(Job.SYSTEM_ANNOTATIONS_RETENTION_DAYS.getPreferredName()); - } if (categorizationFilters != null) { updateFields.add(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName()); } @@ -557,9 +536,6 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { if (resultsRetentionDays != null) { builder.setResultsRetentionDays(resultsRetentionDays); } - if (systemAnnotationsRetentionDays != null) { - builder.setSystemAnnotationsRetentionDays(systemAnnotationsRetentionDays); - } if (categorizationFilters != null) { newAnalysisConfig.setCategorizationFilters(categorizationFilters); } @@ -612,8 +588,6 @@ && updatesDetectors(job) == false && (dailyModelSnapshotRetentionAfterDays == null || Objects.equals(dailyModelSnapshotRetentionAfterDays, job.getDailyModelSnapshotRetentionAfterDays())) && (resultsRetentionDays == null || Objects.equals(resultsRetentionDays, job.getResultsRetentionDays())) - && (systemAnnotationsRetentionDays == null - || Objects.equals(systemAnnotationsRetentionDays, job.getSystemAnnotationsRetentionDays())) && (categorizationFilters == null || Objects.equals(categorizationFilters, job.getAnalysisConfig().getCategorizationFilters())) && (perPartitionCategorizationConfig == null @@ -669,7 +643,6 @@ public boolean equals(Object other) { && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) && Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays) && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) - && Objects.equals(this.systemAnnotationsRetentionDays, that.systemAnnotationsRetentionDays) && Objects.equals(this.categorizationFilters, that.categorizationFilters) && Objects.equals(this.perPartitionCategorizationConfig, that.perPartitionCategorizationConfig) && Objects.equals(this.customSettings, that.customSettings) @@ -686,7 +659,7 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays, - systemAnnotationsRetentionDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, + categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen, blocked, modelPruneWindow); } @@ -794,7 +767,6 @@ public static class Builder { private Long modelSnapshotRetentionDays; private Long dailyModelSnapshotRetentionAfterDays; private Long resultsRetentionDays; - private Long systemAnnotationsRetentionDays; private List categorizationFilters; private PerPartitionCategorizationConfig perPartitionCategorizationConfig; private Map customSettings; @@ -865,11 +837,6 @@ public Builder setResultsRetentionDays(Long resultsRetentionDays) { return this; } - public Builder setSystemAnnotationsRetentionDays(Long systemAnnotationsRetentionDays) { - this.systemAnnotationsRetentionDays = systemAnnotationsRetentionDays; - return this; - } - public Builder setCategorizationFilters(List categorizationFilters) { this.categorizationFilters = categorizationFilters; return this; @@ -932,7 +899,7 @@ public Builder setModelPruneWindow(TimeValue modelPruneWindow) { public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, - renormalizationWindowDays, resultsRetentionDays, systemAnnotationsRetentionDays, modelSnapshotRetentionDays, + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, categorizationFilters, perPartitionCategorizationConfig, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen, blocked, modelPruneWindow); } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json index 5b46c86d44202..72c468ce65286 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json @@ -291,9 +291,6 @@ "type" : "object", "enabled" : false }, - "system_annotations_retention_days" : { - "type" : "long" - }, "background_persist_interval" : { "type" : "keyword" }, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index f22191d380d40..8b95159e78ff7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -142,7 +142,6 @@ public void testConstructor_GivenEmptyJobConfiguration() { assertThat(job.getModelSnapshotRetentionDays(), equalTo(10L)); assertNull(job.getDailyModelSnapshotRetentionAfterDays()); assertNull(job.getResultsRetentionDays()); - assertNull(job.getSystemAnnotationsRetentionDays()); assertNotNull(job.allInputFields()); assertFalse(job.allInputFields().isEmpty()); assertFalse(job.allowLazyOpen()); @@ -266,21 +265,6 @@ public void testEquals_GivenDifferentResultsRetentionDays() { assertNotEquals(jobDetails1.build(), jobDetails2.build()); } - public void testEquals_GivenDifferentSystemAnnotationsRetentionDays() { - Date date = new Date(); - Job.Builder jobDetails1 = new Job.Builder("foo"); - jobDetails1.setDataDescription(new DataDescription.Builder()); - jobDetails1.setAnalysisConfig(createAnalysisConfig()); - jobDetails1.setCreateTime(date); - jobDetails1.setSystemAnnotationsRetentionDays(30L); - Job.Builder jobDetails2 = new Job.Builder("foo"); - jobDetails2.setDataDescription(new DataDescription.Builder()); - jobDetails2.setSystemAnnotationsRetentionDays(4L); - jobDetails2.setAnalysisConfig(createAnalysisConfig()); - jobDetails2.setCreateTime(date); - assertNotEquals(jobDetails1.build(), jobDetails2.build()); - } - public void testEquals_GivenDifferentCustomSettings() { Job.Builder jobDetails1 = buildJobBuilder("foo"); Map customSettings1 = new HashMap<>(); @@ -487,15 +471,6 @@ public void testVerify_GivenNegativeResultsRetentionDays() { assertEquals(errorMessage, e.getMessage()); } - public void testVerify_GivenNegativeSystemAnnotationsRetentionDays() { - String errorMessage = - Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "system_annotations_retention_days", 0, -1); - Job.Builder builder = buildJobBuilder("foo"); - builder.setSystemAnnotationsRetentionDays(-1L); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertEquals(errorMessage, e.getMessage()); - } - public void testBuilder_setsDefaultIndexName() { Job.Builder builder = buildJobBuilder("foo"); Job job = builder.build(); @@ -768,9 +743,6 @@ public static Job createRandomizedJob(DatafeedConfig.Builder datafeedBuilder) { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index e63f0d5c6f347..2671814535ed3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -105,9 +105,6 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - update.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean() && jobSupportsCategorizationFilters(job)) { update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); } @@ -258,7 +255,6 @@ public void testMergeWithJob() { updateBuilder.setAnalysisLimits(analysisLimits); updateBuilder.setBackgroundPersistInterval(TimeValue.timeValueHours(randomIntBetween(1, 24))); updateBuilder.setResultsRetentionDays(randomNonNegativeLong()); - updateBuilder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); // The createRandom() method tests the complex interactions between these next two, so this test can always update both long newModelSnapshotRetentionDays = randomNonNegativeLong(); updateBuilder.setModelSnapshotRetentionDays(newModelSnapshotRetentionDays); @@ -297,7 +293,6 @@ public void testMergeWithJob() { assertEquals(update.getBackgroundPersistInterval(), updatedJob.getBackgroundPersistInterval()); assertEquals(update.getModelSnapshotRetentionDays(), updatedJob.getModelSnapshotRetentionDays()); assertEquals(update.getResultsRetentionDays(), updatedJob.getResultsRetentionDays()); - assertEquals(update.getSystemAnnotationsRetentionDays(), updatedJob.getSystemAnnotationsRetentionDays()); assertEquals(update.getCategorizationFilters(), updatedJob.getAnalysisConfig().getCategorizationFilters()); assertEquals(update.getPerPartitionCategorizationConfig().isEnabled(), updatedJob.getAnalysisConfig().getPerPartitionCategorizationConfig().isEnabled()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index aaa48acfba83b..0abb198bc40e2 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -361,8 +361,7 @@ public void testDeleteExpiresDataDeletesAnnotations() throws Exception { Job.Builder job = new Job.Builder(jobId) - .setResultsRetentionDays(2L) - .setSystemAnnotationsRetentionDays(1L) + .setResultsRetentionDays(1L) .setAnalysisConfig( new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder().setFunction("count").build())) .setBucketSpan(TimeValue.timeValueHours(1))) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java index f5e352e3f7a94..1706206f3a9d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java @@ -61,9 +61,12 @@ public ExpiredAnnotationsRemover(OriginSettingClient client, Iterator jobIt this.threadPool = Objects.requireNonNull(threadPool); } + /** + * Annotations are retained for the same length of time as results. + */ @Override Long getRetentionDays(Job job) { - return job.getSystemAnnotationsRetentionDays(); + return job.getResultsRetentionDays(); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java index d307d7affd937..d695fd0b994bc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java @@ -65,9 +65,6 @@ protected Job.Builder createTestInstance() { if (randomBoolean()) { builder.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { - builder.setSystemAnnotationsRetentionDays(randomNonNegativeLong()); - } if (randomBoolean()) { builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java index aa8e166cdc16d..c5d311a8bfb8c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemoverTests.java @@ -88,8 +88,8 @@ public void testRemove_GivenJobsWithAndWithoutRetentionPolicy() { List jobs = Arrays.asList( JobTests.buildJobBuilder("none").build(), - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build()); + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build()); createExpiredAnnotationsRemover(jobs.iterator()).remove(1.0f, listener, () -> false); @@ -106,8 +106,8 @@ public void testRemove_GivenTimeout() { givenBucket(new Bucket("id_not_important", new Date(), 60)); List jobs = Arrays.asList( - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build() + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build() ); final int timeoutAfter = randomIntBetween(0, 1); @@ -125,8 +125,8 @@ public void testRemove_GivenClientRequestsFailed() { List jobs = Arrays.asList( JobTests.buildJobBuilder("none").build(), - JobTests.buildJobBuilder("annotations-1").setSystemAnnotationsRetentionDays(10L).build(), - JobTests.buildJobBuilder("annotations-2").setSystemAnnotationsRetentionDays(20L).build()); + JobTests.buildJobBuilder("annotations-1").setResultsRetentionDays(10L).build(), + JobTests.buildJobBuilder("annotations-2").setResultsRetentionDays(20L).build()); createExpiredAnnotationsRemover(jobs.iterator()).remove(1.0f, listener, () -> false); assertThat(capturedDeleteByQueryRequests.size(), equalTo(1)); @@ -141,7 +141,7 @@ public void testCalcCutoffEpochMs() { Date latest = new Date(); givenBucket(new Bucket(jobId, latest, 60)); - List jobs = Collections.singletonList(JobTests.buildJobBuilder(jobId).setSystemAnnotationsRetentionDays(1L).build()); + List jobs = Collections.singletonList(JobTests.buildJobBuilder(jobId).setResultsRetentionDays(1L).build()); ActionListener cutoffListener = mock(ActionListener.class); createExpiredAnnotationsRemover(jobs.iterator()).calcCutoffEpochMs(jobId, 1L, cutoffListener); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml index 8d33d24756f51..eee082acd47f2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/delete_expired_data.yml @@ -18,7 +18,6 @@ setup: "time_format":"yyyy-MM-dd HH:mm:ssX" }, "results_retention_days" : 1, - "system_annotations_retention_days" : 1, "model_snapshot_retention_days" : 1 } @@ -39,7 +38,6 @@ setup: "time_format":"yyyy-MM-dd HH:mm:ssX" }, "results_retention_days" : 1, - "system_annotations_retention_days" : 1, "model_snapshot_retention_days" : 1 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml index 9658d621c6b93..605c779051e93 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -379,7 +379,6 @@ "model_snapshot_retention_days": 3, "daily_model_snapshot_retention_after_days": 2, "results_retention_days": 4, - "system_annotations_retention_days": 5, "custom_settings": { "setting1": "custom1", "setting2": "custom2" @@ -435,7 +434,6 @@ "background_persist_interval": "3h", "model_snapshot_retention_days": 30, "results_retention_days": 40, - "system_annotations_retention_days": 50, "custom_settings": { "setting3": "custom3" } @@ -458,7 +456,6 @@ - match: { results_retention_days: 40 } - match: { analysis_config.bucket_span: "5m" } - match: { analysis_config.model_prune_window: "60d" } - - match: { system_annotations_retention_days: 50 } - do: catch: "/Cannot update analysis_limits while the job is open/" From fa791787ba2d0760bb2fec76f5f2a8dff0eeefaf Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 4 Aug 2021 17:59:49 +0100 Subject: [PATCH 2/2] Fix one more merge clash --- .../org/elasticsearch/xpack/core/ml/job/config/Job.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index a02225aad68a6..1af0630af86b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -1085,14 +1085,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays); } out.writeOptionalLong(resultsRetentionDays); -<<<<<<< HEAD - if (out.getVersion().onOrAfter(Version.V_7_15_0)) { - out.writeOptionalLong(systemAnnotationsRetentionDays); -======= - if (out.getVersion().onOrAfter(Version.V_7_15_0) && out.getVersion().before(Version.V_8_0_0)) { - out.writeOptionalLong(resultsRetentionDays); ->>>>>>> 7ac5ea3... [ML] Use results retention time for deleting system annotations (#76096) - } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); if (out.getVersion().onOrAfter(Version.V_7_0_0)) {