diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java index 9b759599dda3c..f3049be2b5e5b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java @@ -60,8 +60,6 @@ public class AnalysisConfig implements ToXContentObject { public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); public static final ParseField DETECTORS = new ParseField("detectors"); public static final ParseField INFLUENCERS = new ParseField("influencers"); - public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); - public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); @SuppressWarnings("unchecked") @@ -84,8 +82,6 @@ public class AnalysisConfig implements ToXContentObject { builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY); PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME); PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS); - PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); - PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW); PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS); } @@ -100,14 +96,11 @@ public class AnalysisConfig implements ToXContentObject { private final String summaryCountFieldName; private final List detectors; private final List influencers; - private final Boolean overlappingBuckets; - private final Long resultFinalizationWindow; private final Boolean multivariateByFields; private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List categorizationFilters, CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName, - List detectors, List influencers, Boolean overlappingBuckets, Long resultFinalizationWindow, - Boolean multivariateByFields) { + List detectors, List influencers, Boolean multivariateByFields) { this.detectors = Collections.unmodifiableList(detectors); this.bucketSpan = bucketSpan; this.latency = latency; @@ -116,8 +109,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters); this.summaryCountFieldName = summaryCountFieldName; this.influencers = Collections.unmodifiableList(influencers); - this.overlappingBuckets = overlappingBuckets; - this.resultFinalizationWindow = resultFinalizationWindow; this.multivariateByFields = multivariateByFields; } @@ -177,14 +168,6 @@ public List getInfluencers() { return influencers; } - public Boolean getOverlappingBuckets() { - return overlappingBuckets; - } - - public Long getResultFinalizationWindow() { - return resultFinalizationWindow; - } - public Boolean getMultivariateByFields() { return multivariateByFields; } @@ -255,12 +238,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.field(INFLUENCERS.getPreferredName(), influencers); - if (overlappingBuckets != null) { - builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets); - } - if (resultFinalizationWindow != null) { - builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow); - } if (multivariateByFields != null) { builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields); } @@ -287,8 +264,6 @@ public boolean equals(Object object) { Objects.equals(summaryCountFieldName, that.summaryCountFieldName) && Objects.equals(detectors, that.detectors) && Objects.equals(influencers, that.influencers) && - Objects.equals(overlappingBuckets, that.overlappingBuckets) && - Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) && Objects.equals(multivariateByFields, that.multivariateByFields); } @@ -296,8 +271,7 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash( bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency, - summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, - multivariateByFields); + summaryCountFieldName, detectors, influencers, multivariateByFields); } public static Builder builder(List detectors) { @@ -314,8 +288,6 @@ public static class Builder { private CategorizationAnalyzerConfig categorizationAnalyzerConfig; private String summaryCountFieldName; private List influencers = new ArrayList<>(); - private Boolean overlappingBuckets; - private Long resultFinalizationWindow; private Boolean multivariateByFields; public Builder(List detectors) { @@ -332,8 +304,6 @@ public Builder(AnalysisConfig analysisConfig) { this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig; this.summaryCountFieldName = analysisConfig.summaryCountFieldName; this.influencers = new ArrayList<>(analysisConfig.influencers); - this.overlappingBuckets = analysisConfig.overlappingBuckets; - this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; this.multivariateByFields = analysisConfig.multivariateByFields; } @@ -391,16 +361,6 @@ public Builder setInfluencers(List influencers) { return this; } - public Builder setOverlappingBuckets(Boolean overlappingBuckets) { - this.overlappingBuckets = overlappingBuckets; - return this; - } - - public Builder setResultFinalizationWindow(Long resultFinalizationWindow) { - this.resultFinalizationWindow = resultFinalizationWindow; - return this; - } - public Builder setMultivariateByFields(Boolean multivariateByFields) { this.multivariateByFields = multivariateByFields; return this; @@ -409,8 +369,7 @@ public Builder setMultivariateByFields(Boolean multivariateByFields) { public AnalysisConfig build() { return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, - latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, - resultFinalizationWindow, multivariateByFields); + latency, summaryCountFieldName, detectors, influencers, multivariateByFields); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java index 7b76688f4d31b..3914697b768ba 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java @@ -89,12 +89,6 @@ public static AnalysisConfig.Builder createRandomized() { if (randomBoolean()) { builder.setMultivariateByFields(randomBoolean()); } - if (randomBoolean()) { - builder.setOverlappingBuckets(randomBoolean()); - } - if (randomBoolean()) { - builder.setResultFinalizationWindow(randomNonNegativeLong()); - } builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java index 9068ffda4de55..e3b86704d0e73 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java @@ -61,15 +61,11 @@ public class AnalysisConfig implements ToXContentObject, Writeable { private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); private static final ParseField DETECTORS = new ParseField("detectors"); private static final ParseField INFLUENCERS = new ParseField("influencers"); - private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); - private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); public static final String ML_CATEGORY_FIELD = "mlcategory"; public static final Set AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD)); - public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L; - // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); public static final ConstructingObjectParser STRICT_PARSER = createParser(false); @@ -94,8 +90,6 @@ private static ConstructingObjectParser createPars builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY); parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME); parser.declareStringArray(Builder::setInfluencers, INFLUENCERS); - parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); - parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW); parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS); return parser; @@ -112,14 +106,11 @@ private static ConstructingObjectParser createPars private final String summaryCountFieldName; private final List detectors; private final List influencers; - private final Boolean overlappingBuckets; - private final Long resultFinalizationWindow; private final Boolean multivariateByFields; private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List categorizationFilters, CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName, - List detectors, List influencers, Boolean overlappingBuckets, Long resultFinalizationWindow, - Boolean multivariateByFields) { + List detectors, List influencers, Boolean multivariateByFields) { this.detectors = detectors; this.bucketSpan = bucketSpan; this.latency = latency; @@ -128,8 +119,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters); this.summaryCountFieldName = summaryCountFieldName; this.influencers = Collections.unmodifiableList(influencers); - this.overlappingBuckets = overlappingBuckets; - this.resultFinalizationWindow = resultFinalizationWindow; this.multivariateByFields = multivariateByFields; } @@ -146,8 +135,13 @@ public AnalysisConfig(StreamInput in) throws IOException { summaryCountFieldName = in.readOptionalString(); detectors = Collections.unmodifiableList(in.readList(Detector::new)); influencers = Collections.unmodifiableList(in.readList(StreamInput::readString)); - overlappingBuckets = in.readOptionalBoolean(); - resultFinalizationWindow = in.readOptionalLong(); + + // BWC for result_finalization_window and overlapping_buckets + // TODO Remove in 7.0.0 + if (in.getVersion().before(Version.CURRENT)) {//setting to current before backport + in.readOptionalBoolean(); + in.readOptionalLong(); + } multivariateByFields = in.readOptionalBoolean(); // BWC for removed multiple_bucket_spans @@ -185,8 +179,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(summaryCountFieldName); out.writeList(detectors); out.writeStringList(influencers); - out.writeOptionalBoolean(overlappingBuckets); - out.writeOptionalLong(resultFinalizationWindow); + + // BWC for result_finalization_window and overlapping_buckets + // TODO Remove in 7.0.0 + if (out.getVersion().before(Version.CURRENT)) { //setting to current before backport + out.writeOptionalBoolean(null); + out.writeOptionalLong(null); + } out.writeOptionalBoolean(multivariateByFields); // BWC for removed multiple_bucket_spans @@ -291,14 +290,6 @@ public Set extractReferencedFilters() { .flatMap(Set::stream).collect(Collectors.toSet()); } - public Boolean getOverlappingBuckets() { - return overlappingBuckets; - } - - public Long getResultFinalizationWindow() { - return resultFinalizationWindow; - } - public Boolean getMultivariateByFields() { return multivariateByFields; } @@ -394,12 +385,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.field(INFLUENCERS.getPreferredName(), influencers); - if (overlappingBuckets != null) { - builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets); - } - if (resultFinalizationWindow != null) { - builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow); - } if (multivariateByFields != null) { builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields); } @@ -420,8 +405,6 @@ public boolean equals(Object o) { Objects.equals(summaryCountFieldName, that.summaryCountFieldName) && Objects.equals(detectors, that.detectors) && Objects.equals(influencers, that.influencers) && - Objects.equals(overlappingBuckets, that.overlappingBuckets) && - Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) && Objects.equals(multivariateByFields, that.multivariateByFields); } @@ -429,9 +412,7 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash( bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency, - summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, - multivariateByFields - ); + summaryCountFieldName, detectors, influencers, multivariateByFields); } public static class Builder { @@ -446,8 +427,6 @@ public static class Builder { private CategorizationAnalyzerConfig categorizationAnalyzerConfig; private String summaryCountFieldName; private List influencers = new ArrayList<>(); - private Boolean overlappingBuckets; - private Long resultFinalizationWindow; private Boolean multivariateByFields; public Builder(List detectors) { @@ -464,8 +443,6 @@ public Builder(AnalysisConfig analysisConfig) { this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig; this.summaryCountFieldName = analysisConfig.summaryCountFieldName; this.influencers = new ArrayList<>(analysisConfig.influencers); - this.overlappingBuckets = analysisConfig.overlappingBuckets; - this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; this.multivariateByFields = analysisConfig.multivariateByFields; } @@ -517,14 +494,6 @@ public void setInfluencers(List influencers) { this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName()); } - public void setOverlappingBuckets(Boolean overlappingBuckets) { - this.overlappingBuckets = overlappingBuckets; - } - - public void setResultFinalizationWindow(Long resultFinalizationWindow) { - this.resultFinalizationWindow = resultFinalizationWindow; - } - public void setMultivariateByFields(Boolean multivariateByFields) { this.multivariateByFields = multivariateByFields; } @@ -536,7 +505,6 @@ public void setMultivariateByFields(Boolean multivariateByFields) { *
  • Check that if non-null Latency is <= MAX_LATENCY
    • *
  • Check there is at least one detector configured
    • *
  • Check all the detectors are configured correctly
    • - *
  • Check that OVERLAPPING_BUCKETS is set appropriately
    • *
  • Check that MULTIPLE_BUCKETSPANS are set appropriately
    • *
  • If Per Partition normalization is configured at least one detector * must have a partition field and no influences can be used
    • @@ -555,17 +523,13 @@ public AnalysisConfig build() { verifyMlCategoryIsUsedWhenCategorizationFieldNameIsSet(); verifyCategorizationAnalyzer(); verifyCategorizationFilters(); - checkFieldIsNotNegativeIfSpecified(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow); verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet(); - overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors); - verifyNoInconsistentNestedFieldNames(); return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, - latency, summaryCountFieldName, detectors, influencers, overlappingBuckets, - resultFinalizationWindow, multivariateByFields); + latency, summaryCountFieldName, detectors, influencers, multivariateByFields); } private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() { @@ -576,13 +540,6 @@ private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() { } } - private static void checkFieldIsNotNegativeIfSpecified(String fieldName, Long value) { - if (value != null && value < 0) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, fieldName, 0, value); - throw ExceptionsHelper.badRequestException(msg); - } - } - private void verifyDetectorAreDefined() { if (detectors == null || detectors.isEmpty()) { throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS)); @@ -697,25 +654,5 @@ private static boolean isValidRegex(String exp) { return false; } } - - private static Boolean verifyOverlappingBucketsConfig(Boolean overlappingBuckets, List detectors) { - // If any detector function is rare/freq_rare, mustn't use overlapping buckets - boolean mustNotUse = false; - - List illegalFunctions = new ArrayList<>(); - for (Detector d : detectors) { - if (Detector.NO_OVERLAPPING_BUCKETS_FUNCTIONS.contains(d.getFunction())) { - illegalFunctions.add(d.getFunction()); - mustNotUse = true; - } - } - - if (Boolean.TRUE.equals(overlappingBuckets) && mustNotUse) { - throw ExceptionsHelper.badRequestException( - Messages.getMessage(Messages.JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION, illegalFunctions.toString())); - } - - return overlappingBuckets; - } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java index d53e4cb74126d..f34739eefe2d5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java @@ -184,25 +184,6 @@ private static ObjectParser createParser(boolean ignoreUnknownFie DetectorFunction.HIGH_NON_ZERO_COUNT ); - /** - * The set of functions that must not be used with overlapping buckets - */ - public static final EnumSet NO_OVERLAPPING_BUCKETS_FUNCTIONS = EnumSet.of( - DetectorFunction.RARE, - DetectorFunction.FREQ_RARE - ); - - /** - * The set of functions that should not be used with overlapping buckets - * as they gain no benefit but have overhead - */ - public static final EnumSet OVERLAPPING_BUCKETS_FUNCTIONS_NOT_NEEDED = EnumSet.of( - DetectorFunction.MIN, - DetectorFunction.MAX, - DetectorFunction.TIME_OF_DAY, - DetectorFunction.TIME_OF_WEEK - ); - /** * Functions that do not support rule conditions: *
      diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 4792180ec51e9..e8b48d5750f96 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -140,8 +140,6 @@ public final class Messages { public static final String JOB_CONFIG_NO_DETECTORS = "No detectors configured"; public static final String JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION = "over_field_name cannot be used with function ''{0}''"; - public static final String JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION = - "Overlapping buckets cannot be used with function ''{0}''"; public static final String JOB_CONFIG_UNKNOWN_FUNCTION = "Unknown function ''{0}''"; public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED = "Invalid update value for analysis_limits: model_memory_limit cannot be decreased below current usage; " + diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java index c95403a112d58..0e38be0c04df1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java @@ -90,12 +90,6 @@ public static AnalysisConfig.Builder createRandomized() { if (randomBoolean()) { builder.setMultivariateByFields(randomBoolean()); } - if (randomBoolean()) { - builder.setOverlappingBuckets(randomBoolean()); - } - if (randomBoolean()) { - builder.setResultFinalizationWindow(randomNonNegativeLong()); - } builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); @@ -564,94 +558,6 @@ public void testVerify_GivenFieldIsControlField() { RecordWriter.CONTROL_FIELD_NAME), e.getMessage()); } - public void testVerify_OverlappingBuckets() { - List detectors; - Detector detector; - - boolean onByDefault = false; - - // Uncomment this when overlappingBuckets turned on by default - if (onByDefault) { - // Test overlappingBuckets unset - AnalysisConfig.Builder analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - detector = new Detector.Builder("mean", "value").build(); - detectors.add(detector); - analysisConfig.setDetectors(detectors); - AnalysisConfig ac = analysisConfig.build(); - assertTrue(ac.getOverlappingBuckets()); - - // Test overlappingBuckets unset - analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - detector = new Detector.Builder("rare", "value").build(); - detectors.add(detector); - analysisConfig.setDetectors(detectors); - ac = analysisConfig.build(); - assertFalse(ac.getOverlappingBuckets()); - - // Test overlappingBuckets unset - analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - detector = new Detector.Builder("min", "value").build(); - detectors.add(detector); - detector = new Detector.Builder("max", "value").build(); - detectors.add(detector); - analysisConfig.setDetectors(detectors); - ac = analysisConfig.build(); - assertFalse(ac.getOverlappingBuckets()); - } - - // Test overlappingBuckets set - AnalysisConfig.Builder analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - Detector.Builder builder = new Detector.Builder("rare", null); - builder.setByFieldName("value"); - detectors.add(builder.build()); - analysisConfig.setOverlappingBuckets(false); - analysisConfig.setDetectors(detectors); - assertFalse(analysisConfig.build().getOverlappingBuckets()); - - // Test overlappingBuckets set - analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - analysisConfig.setOverlappingBuckets(true); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - builder = new Detector.Builder("rare", null); - builder.setByFieldName("value"); - detectors.add(builder.build()); - analysisConfig.setDetectors(detectors); - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, analysisConfig::build); - assertEquals("Overlapping buckets cannot be used with function '[rare]'", e.getMessage()); - - // Test overlappingBuckets set - analysisConfig = createValidConfig(); - analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(5000L)); - analysisConfig.setOverlappingBuckets(false); - detectors = new ArrayList<>(); - detector = new Detector.Builder("count", null).build(); - detectors.add(detector); - detector = new Detector.Builder("mean", "value").build(); - detectors.add(detector); - analysisConfig.setDetectors(detectors); - AnalysisConfig ac = analysisConfig.build(); - assertFalse(ac.getOverlappingBuckets()); - } - public void testVerify_GivenMetricAndSummaryCountField() { Detector d = new Detector.Builder("metric", "my_metric").build(); AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d)); @@ -728,7 +634,7 @@ private static AnalysisConfig.Builder createValidCategorizationConfig() { @Override protected AnalysisConfig mutateInstance(AnalysisConfig instance) { AnalysisConfig.Builder builder = new AnalysisConfig.Builder(instance); - switch (between(0, 10)) { + switch (between(0, 8)) { case 0: List detectors = new ArrayList<>(instance.getDetectors()); Detector.Builder detector = new Detector.Builder(); @@ -806,20 +712,6 @@ protected AnalysisConfig mutateInstance(AnalysisConfig instance) { builder.setInfluencers(influencers); break; case 8: - if (instance.getOverlappingBuckets() == null) { - builder.setOverlappingBuckets(randomBoolean()); - } else { - builder.setOverlappingBuckets(instance.getOverlappingBuckets() == false); - } - break; - case 9: - if (instance.getResultFinalizationWindow() == null) { - builder.setResultFinalizationWindow(between(1, 100) * 1000L); - } else { - builder.setResultFinalizationWindow(instance.getResultFinalizationWindow() + (between(1, 100) * 1000)); - } - break; - case 10: if (instance.getMultivariateByFields() == null) { builder.setMultivariateByFields(randomBoolean()); } else { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java index 3d5533fed08ed..4cbeaf1dc482c 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java @@ -43,7 +43,6 @@ public void test() throws Exception { AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( Collections.singletonList(new Detector.Builder("max", "value").build())); analysisConfig.setBucketSpan(TimeValue.timeValueSeconds(BUCKET_SPAN_SECONDS)); - analysisConfig.setOverlappingBuckets(true); DataDescription.Builder dataDescription = new DataDescription.Builder(); dataDescription.setTimeFormat("epoch"); Job.Builder job = new Job.Builder(JOB_ID); @@ -77,10 +76,9 @@ public void test() throws Exception { // We might need to retry this while waiting for a refresh assertBusy(() -> { List firstInterimBuckets = getInterimResults(job.getId()); - assertThat("interim buckets were: " + firstInterimBuckets, firstInterimBuckets.size(), equalTo(2)); - assertThat(firstInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400039000000L)); - assertThat(firstInterimBuckets.get(1).getTimestamp().getTime(), equalTo(1400040000000L)); - assertThat(firstInterimBuckets.get(1).getRecords().get(0).getActual().get(0), equalTo(16.0)); + assertThat("interim buckets were: " + firstInterimBuckets, firstInterimBuckets.size(), equalTo(1)); + assertThat(firstInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400040000000L)); + assertThat(firstInterimBuckets.get(0).getRecords().get(0).getActual().get(0), equalTo(16.0)); }); // push 1 more record, flush (with interim), check same interim result @@ -90,9 +88,7 @@ public void test() throws Exception { assertBusy(() -> { List secondInterimBuckets = getInterimResults(job.getId()); - assertThat(secondInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400039000000L)); - assertThat(secondInterimBuckets.get(1).getTimestamp().getTime(), equalTo(1400040000000L)); - assertThat(secondInterimBuckets.get(1).getRecords().get(0).getActual().get(0), equalTo(16.0)); + assertThat(secondInterimBuckets.get(0).getTimestamp().getTime(), equalTo(1400040000000L)); }); // push rest of data, close, verify no interim results @@ -103,11 +99,11 @@ public void test() throws Exception { // Verify interim results have been replaced with finalized results GetBucketsAction.Request bucketRequest = new GetBucketsAction.Request(job.getId()); - bucketRequest.setTimestamp("1400039500000"); + bucketRequest.setTimestamp("1400040000000"); bucketRequest.setExpand(true); List bucket = client().execute(GetBucketsAction.INSTANCE, bucketRequest).get().getBuckets().results(); assertThat(bucket.size(), equalTo(1)); - assertThat(bucket.get(0).getRecords().get(0).getActual().get(0), equalTo(14.0)); + assertThat(bucket.get(0).getRecords().get(0).getActual().get(0), equalTo(16.0)); } private String createData(int halfBuckets) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java index dbc565fc50c12..5465a11d149ab 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java @@ -68,7 +68,6 @@ public class AutodetectBuilder { private static final String MODEL_PLOT_CONFIG_ARG = "--modelplotconfig="; private static final String FIELD_CONFIG_ARG = "--fieldconfig="; static final String LATENCY_ARG = "--latency="; - static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow="; static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields"; static final String PERSIST_INTERVAL_ARG = "--persistInterval="; static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval="; @@ -202,15 +201,7 @@ List buildAutodetectCommand() { if (analysisConfig != null) { addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command); addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command); - addIfNotNull(analysisConfig.getSummaryCountFieldName(), - SUMMARY_COUNT_FIELD_ARG, command); - if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) { - Long window = analysisConfig.getResultFinalizationWindow(); - if (window == null) { - window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW; - } - command.add(RESULT_FINALIZATION_WINDOW_ARG + window); - } + addIfNotNull(analysisConfig.getSummaryCountFieldName(), SUMMARY_COUNT_FIELD_ARG, command); if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) { command.add(MULTIVARIATE_BY_FIELDS_ARG); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java index 9ef56d927f553..fedd3b320a39e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java @@ -54,8 +54,8 @@ public void testBuildAutodetectCommand() { acBuilder.setBucketSpan(TimeValue.timeValueSeconds(120)); acBuilder.setLatency(TimeValue.timeValueSeconds(360)); acBuilder.setSummaryCountFieldName("summaryField"); - acBuilder.setOverlappingBuckets(true); acBuilder.setMultivariateByFields(true); + job.setAnalysisConfig(acBuilder); DataDescription.Builder dd = new DataDescription.Builder(); @@ -65,12 +65,11 @@ public void testBuildAutodetectCommand() { job.setDataDescription(dd); List command = autodetectBuilder(job.build()).buildAutodetectCommand(); - assertEquals(12, command.size()); + assertEquals(11, command.size()); assertTrue(command.contains(AutodetectBuilder.AUTODETECT_PATH)); assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "120")); assertTrue(command.contains(AutodetectBuilder.LATENCY_ARG + "360")); assertTrue(command.contains(AutodetectBuilder.SUMMARY_COUNT_FIELD_ARG + "summaryField")); - assertTrue(command.contains(AutodetectBuilder.RESULT_FINALIZATION_WINDOW_ARG + "2")); assertTrue(command.contains(AutodetectBuilder.MULTIVARIATE_BY_FIELDS_ARG)); assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG)); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index ba0f4d5091e0f..39cd2a1ed3ab5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -1,4 +1,9 @@ --- +setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" +--- "Test get old cluster job": - do: xpack.ml.get_jobs: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml index 0ec288f90973c..486dd6400e7af 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml @@ -1,4 +1,9 @@ --- +setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" +--- "Test old cluster datafeed": - do: xpack.ml.get_datafeeds: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index 3a3334f6907e9..dd44bac660b09 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -1,4 +1,9 @@ --- +setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" +--- "Put job on the old cluster and post some data": - do: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml index c1317bdf3d660..afd4ee2bf1fda 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml @@ -1,4 +1,9 @@ --- +setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" +--- "Put job and datafeed in old cluster": - do: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml index bb47524b41d87..74244deb8448a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml @@ -1,4 +1,7 @@ setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" - do: cluster.health: wait_for_status: green diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml index 6b4c963dd533b..9fbae269e4937 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml @@ -1,4 +1,7 @@ setup: + - skip: + version: "all" + reason: "Temporarily disabled while backporting https://github.com/elastic/elasticsearch/pull/35645" - do: cluster.health: wait_for_status: green